Merge pull request #79 from jcgruenhage/renewal-scheduling

Scheduling renewals

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - The minimum supported Rust version (MSRV) is now 1.64.
 - Manual (and badly designed) threads have been replaced by async.
+- Randomized early delay, for spacing out renewals when dealing with a lot of certificates.
 
 
 ## [0.21.0] - 2022-12-19

Cargo.lock

@@ -38,6 +38,7 @@ dependencies = [
  "log",
  "nix",
  "nom",
+ "rand",
  "serde",
  "serde_json",
  "tinytemplate",
@@ -536,6 +537,17 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "glob"
 version = "0.3.1"
@@ -915,6 +927,12 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.56"
@@ -939,6 +957,36 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"

acmed/Cargo.toml

@@ -35,6 +35,7 @@ serde_json = "1.0"
 tinytemplate = "1.2"
 toml = "0.7"
 tokio = { version = "1", features = ["full"] }
+rand = "0.8.5"
 
 [target.'cfg(unix)'.dependencies]
 nix = "0.26"

acmed/src/certificate.rs

@@ -6,6 +6,7 @@ use crate::storage::{certificate_files_exists, get_certificate, FileManager};
 use acme_common::crypto::{HashFunction, KeyType, SubjectAttribute, X509Certificate};
 use acme_common::error::Error;
 use log::{debug, info, trace, warn};
+use rand::{thread_rng, Rng};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 use std::time::Duration;
@@ -22,6 +23,7 @@ pub struct Certificate {
 	pub hooks: Vec<Hook>,
 	pub crt_name: String,
 	pub env: HashMap<String, String>,
+	pub random_early_renew: Duration,
 	pub renew_delay: Duration,
 	pub file_manager: FileManager,
 }
@@ -70,14 +72,16 @@ impl Certificate {
 		Err(format!("{identifier}: identifier not found").into())
 	}
 
-	fn is_expiring(&self, cert: &X509Certificate) -> Result<bool, Error> {
+	fn renew_in(&self, cert: &X509Certificate) -> Result<Duration, Error> {
 		let expires_in = cert.expires_in()?;
 		self.debug(&format!(
 			"certificate expires in {} days ({} days delay)",
 			expires_in.as_secs() / 86400,
 			self.renew_delay.as_secs() / 86400,
 		));
-		Ok(expires_in <= self.renew_delay)
+		Ok(expires_in
+			.saturating_sub(self.renew_delay)
+			.saturating_sub(thread_rng().gen_range(Duration::ZERO..self.random_early_renew)))
 	}
 
 	fn has_missing_identifiers(&self, cert: &X509Certificate) -> bool {
@@ -110,33 +114,22 @@ impl Certificate {
 			.join(",")
 	}
 
-	pub async fn should_renew(&self) -> Result<bool, Error> {
+	pub async fn schedule_renewal(&self) -> Result<Duration, Error> {
 		self.debug(&format!(
 			"checking for renewal (identifiers: {})",
 			self.identifier_list()
 		));
 		if !certificate_files_exists(&self.file_manager) {
 			self.debug("certificate does not exist: requesting one");
-			return Ok(true);
+			return Ok(Duration::ZERO);
 		}
 		let cert = get_certificate(&self.file_manager).await?;
 
-		let renew_ident = self.has_missing_identifiers(&cert);
-		if renew_ident {
+		if self.has_missing_identifiers(&cert) {
 			self.debug("the current certificate doesn't include all the required identifiers");
+			return Ok(Duration::ZERO);
 		}
-		let renew_exp = self.is_expiring(&cert)?;
-		if renew_exp {
-			self.debug("the certificate is expiring");
-		}
-		let renew = renew_ident || renew_exp;
-
-		if renew {
-			self.debug("the certificate will be renewed now");
-		} else {
-			self.debug("the certificate will not be renewed now");
-		}
-		Ok(renew)
+		self.renew_in(&cert)
 	}
 
 	pub async fn call_challenge_hooks(

acmed/src/config.rs

@@ -186,11 +186,19 @@ pub struct GlobalOptions {
 	pub pk_file_group: Option<String>,
 	pub pk_file_mode: Option<u32>,
 	pub pk_file_user: Option<String>,
+	pub random_early_renew: Option<String>,
 	pub renew_delay: Option<String>,
 	pub root_certificates: Option<Vec<String>>,
 }
 
 impl GlobalOptions {
+	pub fn get_random_early_renew(&self) -> Result<Duration, Error> {
+		match &self.random_early_renew {
+			Some(d) => parse_duration(d),
+			None => Ok(Duration::new(crate::DEFAULT_CERT_RANDOM_EARLY_RENEW, 0)),
+		}
+	}
+
 	pub fn get_renew_delay(&self) -> Result<Duration, Error> {
 		match &self.renew_delay {
 			Some(d) => parse_duration(d),
@@ -211,6 +219,7 @@ impl GlobalOptions {
 pub struct Endpoint {
 	pub file_name_format: Option<String>,
 	pub name: String,
+	pub random_early_renew: Option<String>,
 	#[serde(default)]
 	pub rate_limits: Vec<String>,
 	pub renew_delay: Option<String>,
@@ -220,6 +229,16 @@ pub struct Endpoint {
 }
 
 impl Endpoint {
+	pub fn get_random_early_renew(&self, cnf: &Config) -> Result<Duration, Error> {
+		match &self.random_early_renew {
+			Some(d) => parse_duration(d),
+			None => match &cnf.global {
+				Some(g) => g.get_random_early_renew(),
+				None => Ok(Duration::new(crate::DEFAULT_CERT_RANDOM_EARLY_RENEW, 0)),
+			},
+		}
+	}
+
 	pub fn get_renew_delay(&self, cnf: &Config) -> Result<Duration, Error> {
 		match &self.renew_delay {
 			Some(d) => parse_duration(d),
@@ -437,6 +456,7 @@ pub struct Certificate {
 	pub key_type: Option<String>,
 	pub kp_reuse: Option<bool>,
 	pub name: Option<String>,
+	pub random_early_renew: Option<String>,
 	pub renew_delay: Option<String>,
 	#[serde(default)]
 	pub subject_attributes: SubjectAttributes,
@@ -538,6 +558,16 @@ impl Certificate {
 		Ok(res)
 	}
 
+	pub fn get_random_early_renew(&self, cnf: &Config) -> Result<Duration, Error> {
+		match &self.random_early_renew {
+			Some(d) => parse_duration(d),
+			None => {
+				let endpoint = self.do_get_endpoint(cnf)?;
+				endpoint.get_random_early_renew(cnf)
+			}
+		}
+	}
+
 	pub fn get_renew_delay(&self, cnf: &Config) -> Result<Duration, Error> {
 		match &self.renew_delay {
 			Some(d) => parse_duration(d),

acmed/src/main.rs

@@ -33,12 +33,12 @@ pub const DEFAULT_CERT_DIR: &str = env!("ACMED_DEFAULT_CERT_DIR");
 pub const DEFAULT_CERT_FORMAT: &str = env!("ACMED_DEFAULT_CERT_FORMAT");
 pub const DEFAULT_CONFIG_FILE: &str = env!("ACMED_DEFAULT_CONFIG_FILE");
 pub const DEFAULT_PID_FILE: &str = env!("ACMED_DEFAULT_PID_FILE");
-pub const DEFAULT_SLEEP_TIME: u64 = 3600;
 pub const DEFAULT_POOL_TIME: u64 = 5000;
 pub const DEFAULT_CSR_DIGEST: HashFunction = HashFunction::Sha256;
 pub const DEFAULT_CERT_KEY_TYPE: KeyType = KeyType::Rsa2048;
 pub const DEFAULT_CERT_FILE_MODE: u32 = 0o644;
-pub const DEFAULT_CERT_RENEW_DELAY: u64 = 1_814_400; // 1_814_400 is 3 weeks (3 * 7 * 24 * 60 * 60)
+pub const DEFAULT_CERT_RANDOM_EARLY_RENEW: u64 = 0; // default to not renewing early
+pub const DEFAULT_CERT_RENEW_DELAY: u64 = 30 * 24 * 60 * 60; // 30 days
 pub const DEFAULT_PK_FILE_MODE: u32 = 0o600;
 pub const DEFAULT_ACCOUNT_FILE_MODE: u32 = 0o600;
 pub const DEFAULT_KP_REUSE: bool = false;

acmed/src/main_event_loop.rs

@@ -116,6 +116,7 @@ impl MainEventLoop {
 					.collect(),
 				crt_name,
 				env: crt.env.to_owned(),
+				random_early_renew: crt.get_random_early_renew(&cnf)?,
 				renew_delay: crt.get_renew_delay(&cnf)?,
 				file_manager: fm,
 			};
@@ -179,15 +180,23 @@ async fn renew_certificate(
 	account_s: AccountSync,
 	endpoint_s: EndpointSync,
 ) -> (&mut Certificate, AccountSync, EndpointSync) {
+	let backoff = [60, 10 * 60, 100 * 60, 24 * 60 * 60];
+	let mut scheduling_retries = 0;
 	loop {
-		match certificate.should_renew().await {
-			Ok(true) => break,
-			Ok(false) => {}
+		match certificate.schedule_renewal().await {
+			Ok(duration) => {
+				sleep(duration).await;
+				break;
+			}
 			Err(e) => {
 				certificate.warn(&e.message);
+				sleep(Duration::from_secs(
+					backoff[scheduling_retries.min(backoff.len() - 1)],
+				))
+				.await;
+				scheduling_retries += 1;
 			}
 		}
-		sleep(Duration::from_secs(crate::DEFAULT_SLEEP_TIME)).await;
 	}
 	let (status, is_success) =
 		match request_certificate(certificate, account_s.clone(), endpoint_s.clone()).await {

man/en/acmed.toml.5

@@ -206,6 +206,11 @@ Name of the certificate. Must be unique unless the key type is different. Will b
 and
 .Sq /
 characters will be replaced by an underscore. Default is the first identifier.
+.It Cm random_early_renew Ar string
+Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when
+you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the
+.Sx TIME PERIODS
+section. Default is the value defined in the associated endpoint.
 .It Cm renew_delay Ar string
 Period of time between the certificate renewal and its expiration date. The format is described in the
 .Sx TIME PERIODS
@@ -246,6 +251,11 @@ element.
 The name the endpoint is registered under. Must be unique.
 .It Cm rate_limits Ar array
 Array containing the names of the HTTPS rate limits to apply.
+.It Cm random_early_renew Ar string
+Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when
+you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the
+.Sx TIME PERIODS
+section. Default is the value defined in the global section.
 .It Cm renew_delay Ar string
 Period of time between the certificate renewal and its expiration date. The format is described in the
 .Sx TIME PERIODS
@@ -297,10 +307,15 @@ for more details.
 Specify the user who will own newly-created private-key files. See
 .Xr chown 2
 for more details.
+.It Cm random_early_renew Ar string
+Period of time before the usual certificate renewal, in which the certificate will renew at a random time. This is useful for when
+you want to even out your certificate orders when you're dealing with very large numbers of certificates. The format is described in the
+.Sx TIME PERIODS
+section. By default, this is disabled, or rather, the time frame is set to 0.
 .It Cm renew_delay Ar string
 Period of time between the certificate renewal and its expiration date. The format is described in the
 .Sx TIME PERIODS
-section. Default is 3w.
+section. Default is 30d.
 .It Cm root_certificates Ar array
 Array containing the path to root certificates that should be added to the trust store.
 .El