diff --git a/CHANGELOG.md b/CHANGELOG.md index d569bb6..90eaad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The minimum supported Rust version (MSRV) is now 1.64. - Manual (and badly designed) threads have been replaced by async. +- Randomized early delay, for spacing out renewals when dealing with a lot of certificates. ## [0.21.0] - 2022-12-19 diff --git a/Cargo.lock b/Cargo.lock index 049da4b..0c7e16e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,7 @@ dependencies = [ "log", "nix", "nom", + "rand", "serde", "serde_json", "tinytemplate", @@ -536,6 +537,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "glob" version = "0.3.1" @@ -915,6 +927,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.56" @@ -939,6 +957,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.2.16" diff --git a/acmed/Cargo.toml b/acmed/Cargo.toml index df2fe0a..a6d097f 100644 --- a/acmed/Cargo.toml +++ b/acmed/Cargo.toml @@ -35,6 +35,7 @@ serde_json = "1.0" tinytemplate = "1.2" toml = "0.7" tokio = { version = "1", features = ["full"] } +rand = "0.8.5" [target.'cfg(unix)'.dependencies] nix = "0.26" diff --git a/acmed/src/certificate.rs b/acmed/src/certificate.rs index 739936a..89f4410 100644 --- a/acmed/src/certificate.rs +++ b/acmed/src/certificate.rs @@ -6,6 +6,7 @@ use crate::storage::{certificate_files_exists, get_certificate, FileManager}; use acme_common::crypto::{HashFunction, KeyType, SubjectAttribute, X509Certificate}; use acme_common::error::Error; use log::{debug, info, trace, warn}; +use rand::{thread_rng, Rng}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::time::Duration; @@ -22,6 +23,7 @@ pub struct Certificate { pub hooks: Vec, pub crt_name: String, pub env: HashMap, + pub random_early_renew: Duration, pub renew_delay: Duration, pub file_manager: FileManager, } @@ -70,14 +72,16 @@ impl Certificate { Err(format!("{identifier}: identifier not found").into()) } - fn is_expiring(&self, cert: &X509Certificate) -> Result { + fn renew_in(&self, cert: &X509Certificate) -> Result { let expires_in = cert.expires_in()?; self.debug(&format!( "certificate expires in {} days ({} days delay)", expires_in.as_secs() / 86400, self.renew_delay.as_secs() / 86400, )); - Ok(expires_in <= self.renew_delay) + Ok(expires_in + .saturating_sub(self.renew_delay) + .saturating_sub(thread_rng().gen_range(Duration::ZERO..self.random_early_renew))) } fn has_missing_identifiers(&self, cert: &X509Certificate) -> bool { @@ -110,33 +114,22 @@ impl Certificate { .join(",") } - pub async fn should_renew(&self) -> Result { + pub async fn schedule_renewal(&self) -> Result { self.debug(&format!( "checking for renewal (identifiers: {})", self.identifier_list() )); if !certificate_files_exists(&self.file_manager) { self.debug("certificate does not exist: requesting one"); - return Ok(true); + return Ok(Duration::ZERO); } let cert = get_certificate(&self.file_manager).await?; - let renew_ident = self.has_missing_identifiers(&cert); - if renew_ident { + if self.has_missing_identifiers(&cert) { self.debug("the current certificate doesn't include all the required identifiers"); + return Ok(Duration::ZERO); } - let renew_exp = self.is_expiring(&cert)?; - if renew_exp { - self.debug("the certificate is expiring"); - } - let renew = renew_ident || renew_exp; - - if renew { - self.debug("the certificate will be renewed now"); - } else { - self.debug("the certificate will not be renewed now"); - } - Ok(renew) + self.renew_in(&cert) } pub async fn call_challenge_hooks( diff --git a/acmed/src/config.rs b/acmed/src/config.rs index a1aa52f..032654d 100644 --- a/acmed/src/config.rs +++ b/acmed/src/config.rs @@ -186,11 +186,19 @@ pub struct GlobalOptions { pub pk_file_group: Option, pub pk_file_mode: Option, pub pk_file_user: Option, + pub random_early_renew: Option, pub renew_delay: Option, pub root_certificates: Option>, } impl GlobalOptions { + pub fn get_random_early_renew(&self) -> Result { + match &self.random_early_renew { + Some(d) => parse_duration(d), + None => Ok(Duration::new(crate::DEFAULT_CERT_RANDOM_EARLY_RENEW, 0)), + } + } + pub fn get_renew_delay(&self) -> Result { match &self.renew_delay { Some(d) => parse_duration(d), @@ -211,6 +219,7 @@ impl GlobalOptions { pub struct Endpoint { pub file_name_format: Option, pub name: String, + pub random_early_renew: Option, #[serde(default)] pub rate_limits: Vec, pub renew_delay: Option, @@ -220,6 +229,16 @@ pub struct Endpoint { } impl Endpoint { + pub fn get_random_early_renew(&self, cnf: &Config) -> Result { + match &self.random_early_renew { + Some(d) => parse_duration(d), + None => match &cnf.global { + Some(g) => g.get_random_early_renew(), + None => Ok(Duration::new(crate::DEFAULT_CERT_RANDOM_EARLY_RENEW, 0)), + }, + } + } + pub fn get_renew_delay(&self, cnf: &Config) -> Result { match &self.renew_delay { Some(d) => parse_duration(d), @@ -437,6 +456,7 @@ pub struct Certificate { pub key_type: Option, pub kp_reuse: Option, pub name: Option, + pub random_early_renew: Option, pub renew_delay: Option, #[serde(default)] pub subject_attributes: SubjectAttributes, @@ -538,6 +558,16 @@ impl Certificate { Ok(res) } + pub fn get_random_early_renew(&self, cnf: &Config) -> Result { + match &self.random_early_renew { + Some(d) => parse_duration(d), + None => { + let endpoint = self.do_get_endpoint(cnf)?; + endpoint.get_random_early_renew(cnf) + } + } + } + pub fn get_renew_delay(&self, cnf: &Config) -> Result { match &self.renew_delay { Some(d) => parse_duration(d), diff --git a/acmed/src/main.rs b/acmed/src/main.rs index a933867..ba4f3ff 100644 --- a/acmed/src/main.rs +++ b/acmed/src/main.rs @@ -33,12 +33,12 @@ pub const DEFAULT_CERT_DIR: &str = env!("ACMED_DEFAULT_CERT_DIR"); pub const DEFAULT_CERT_FORMAT: &str = env!("ACMED_DEFAULT_CERT_FORMAT"); pub const DEFAULT_CONFIG_FILE: &str = env!("ACMED_DEFAULT_CONFIG_FILE"); pub const DEFAULT_PID_FILE: &str = env!("ACMED_DEFAULT_PID_FILE"); -pub const DEFAULT_SLEEP_TIME: u64 = 3600; pub const DEFAULT_POOL_TIME: u64 = 5000; pub const DEFAULT_CSR_DIGEST: HashFunction = HashFunction::Sha256; pub const DEFAULT_CERT_KEY_TYPE: KeyType = KeyType::Rsa2048; pub const DEFAULT_CERT_FILE_MODE: u32 = 0o644; -pub const DEFAULT_CERT_RENEW_DELAY: u64 = 1_814_400; // 1_814_400 is 3 weeks (3 * 7 * 24 * 60 * 60) +pub const DEFAULT_CERT_RANDOM_EARLY_RENEW: u64 = 0; // default to not renewing early +pub const DEFAULT_CERT_RENEW_DELAY: u64 = 30 * 24 * 60 * 60; // 30 days pub const DEFAULT_PK_FILE_MODE: u32 = 0o600; pub const DEFAULT_ACCOUNT_FILE_MODE: u32 = 0o600; pub const DEFAULT_KP_REUSE: bool = false; diff --git a/acmed/src/main_event_loop.rs b/acmed/src/main_event_loop.rs index 85456c2..9d95c69 100644 --- a/acmed/src/main_event_loop.rs +++ b/acmed/src/main_event_loop.rs @@ -116,6 +116,7 @@ impl MainEventLoop { .collect(), crt_name, env: crt.env.to_owned(), + random_early_renew: crt.get_random_early_renew(&cnf)?, renew_delay: crt.get_renew_delay(&cnf)?, file_manager: fm, }; @@ -179,15 +180,23 @@ async fn renew_certificate( account_s: AccountSync, endpoint_s: EndpointSync, ) -> (&mut Certificate, AccountSync, EndpointSync) { + let backoff = [60, 10 * 60, 100 * 60, 24 * 60 * 60]; + let mut scheduling_retries = 0; loop { - match certificate.should_renew().await { - Ok(true) => break, - Ok(false) => {} + match certificate.schedule_renewal().await { + Ok(duration) => { + sleep(duration).await; + break; + } Err(e) => { certificate.warn(&e.message); + sleep(Duration::from_secs( + backoff[scheduling_retries.min(backoff.len() - 1)], + )) + .await; + scheduling_retries += 1; } } - sleep(Duration::from_secs(crate::DEFAULT_SLEEP_TIME)).await; } let (status, is_success) = match request_certificate(certificate, account_s.clone(), endpoint_s.clone()).await { diff --git a/man/en/acmed.toml.5 b/man/en/acmed.toml.5 index 81d6c41..35b6ba6 100644 --- a/man/en/acmed.toml.5 +++ b/man/en/acmed.toml.5 @@ -206,6 +206,11 @@ Name of the certificate. Must be unique unless the key type is different. Will b and .Sq / characters will be replaced by an underscore. Default is the first identifier. +.It Cm random_early_renew Ar string +Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when +you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the +.Sx TIME PERIODS +section. Default is the value defined in the associated endpoint. .It Cm renew_delay Ar string Period of time between the certificate renewal and its expiration date. The format is described in the .Sx TIME PERIODS @@ -246,6 +251,11 @@ element. The name the endpoint is registered under. Must be unique. .It Cm rate_limits Ar array Array containing the names of the HTTPS rate limits to apply. +.It Cm random_early_renew Ar string +Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when +you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the +.Sx TIME PERIODS +section. Default is the value defined in the global section. .It Cm renew_delay Ar string Period of time between the certificate renewal and its expiration date. The format is described in the .Sx TIME PERIODS @@ -297,10 +307,15 @@ for more details. Specify the user who will own newly-created private-key files. See .Xr chown 2 for more details. +.It Cm random_early_renew Ar string +Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when +you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the +.Sx TIME PERIODS +section. By default, this is disabled, or rather, the time frame is set to 0. .It Cm renew_delay Ar string Period of time between the certificate renewal and its expiration date. The format is described in the .Sx TIME PERIODS -section. Default is 3w. +section. Default is 30d. .It Cm root_certificates Ar array Array containing the path to root certificates that should be added to the trust store. .El