diff --git a/CHANGELOG.md b/CHANGELOG.md index 378c5b7..f71c5b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Hooks now have the optional `allow_failure` field. - In hooks, the `stdin_str` has been added in replacement of the previous `stdin` behavior. +- HTTPS request rate limits. ### Changed - Certificates are renewed in parallel. diff --git a/README.md b/README.md index 3308c46..a27cc89 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ The Automatic Certificate Management Environment (ACME), is an internet standard - Nice and simple configuration file - Run as a deamon: no need to set-up timers, crontab or other time-triggered process - Retry of HTTPS request rejected with a badNonce or other recoverable errors -- Optional private-key reuse (useful for [HPKP](https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning)) +- Customizable HTTPS requests rate limits. +- Optional key pair reuse (useful for [HPKP](https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning)) - For a given certificate, each domain names may be validated using a different challenge. - A standalone server dedicated to the tls-alpn-01 challenge validation diff --git a/acme_common/src/error.rs b/acme_common/src/error.rs index 637d57e..9e9ffd6 100644 --- a/acme_common/src/error.rs +++ b/acme_common/src/error.rs @@ -51,6 +51,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: std::sync::mpsc::RecvError) -> Self { + format!("MSPC receiver error: {}", error).into() + } +} + impl From for Error { fn from(error: syslog::Error) -> Self { format!("syslog error: {}", error).into() diff --git a/acmed/Cargo.toml b/acmed/Cargo.toml index 012df8b..d3dd731 100644 --- a/acmed/Cargo.toml +++ b/acmed/Cargo.toml @@ -17,6 +17,7 @@ clap = "2.32" handlebars = "2.0.0-beta.1" http_req = "0.4" log = "0.4" +nom = "5.0.0-beta2" openssl = "0.10" openssl-sys = "0.9" serde = { version = "1.0", features = ["derive"] } diff --git a/acmed/config/acmed.toml b/acmed/config/acmed.toml index ca1784c..ad01320 100644 --- a/acmed/config/acmed.toml +++ b/acmed/config/acmed.toml @@ -2,12 +2,19 @@ include = [ "default_hooks.toml" ] +[[rate-limit]] +name = "LE min" +number = 20 +period = "1s" + [[endpoint]] name = "letsencrypt v2 prod" url = "https://acme-v02.api.letsencrypt.org/directory" +rate_limits = ["LE min"] tos_agreed = false [[endpoint]] name = "letsencrypt v2 staging" url = "https://acme-staging-v02.api.letsencrypt.org/directory" +rate_limits = ["LE min"] tos_agreed = false diff --git a/acmed/src/acme_proto/http.rs b/acmed/src/acme_proto/http.rs index ab6c737..2dd874f 100644 --- a/acmed/src/acme_proto/http.rs +++ b/acmed/src/acme_proto/http.rs @@ -1,7 +1,8 @@ use crate::acme_proto::structs::{AcmeError, ApiError, Directory, HttpApiError}; use crate::certificate::Certificate; +use crate::rate_limits; use acme_common::error::Error; -use http_req::request::{Method, Request}; +use http_req::request::{self, Method}; use http_req::response::Response; use http_req::uri::Uri; use std::path::Path; @@ -11,6 +12,12 @@ use std::{thread, time}; const CONTENT_TYPE_JOSE: &str = "application/jose+json"; const CONTENT_TYPE_JSON: &str = "application/json"; +struct Request<'a> { + r: request::Request<'a>, + uri: &'a Uri, + method: Method, +} + struct DummyString { pub content: String, } @@ -25,13 +32,7 @@ impl FromStr for DummyString { } } -fn new_request<'a>( - cert: &Certificate, - root_certs: &'a [String], - uri: &'a Uri, - method: Method, -) -> Request<'a> { - cert.debug(&format!("{}: {}", method, uri)); +fn new_request<'a>(root_certs: &'a [String], uri: &'a Uri, method: Method) -> Request<'a> { let useragent = format!( "{}/{} ({}) {}", crate::APP_NAME, @@ -39,27 +40,31 @@ fn new_request<'a>( env!("ACMED_TARGET"), env!("ACMED_HTTP_LIB_AGENT") ); - let mut rb = Request::new(uri); + let mut rb = request::Request::new(uri); for file_name in root_certs.iter() { rb.root_cert_file_pem(&Path::new(file_name)); } - rb.method(method); + rb.method(method.to_owned()); rb.header("User-Agent", &useragent); // TODO: allow to configure the language rb.header("Accept-Language", "en-US,en;q=0.5"); - rb + Request { r: rb, method, uri } } -fn send_request(request: &Request) -> Result<(Response, String), Error> { +fn send_request(cert: &Certificate, request: &Request) -> Result<(Response, String), Error> { let mut buffer = Vec::new(); - let res = request.send(&mut buffer)?; + cert.https_throttle + .send(rate_limits::Request::HttpsRequest) + .unwrap(); + cert.debug(&format!("{}: {}", request.method, request.uri)); + let res = request.r.send(&mut buffer)?; let res_str = String::from_utf8(buffer)?; Ok((res, res_str)) } fn send_request_retry(cert: &Certificate, request: &Request) -> Result<(Response, String), Error> { for _ in 0..crate::DEFAULT_HTTP_FAIL_NB_RETRY { - let (res, res_body) = send_request(request)?; + let (res, res_body) = send_request(cert, request)?; match check_response(&res, &res_body) { Ok(()) => { return Ok((res, res_body)); @@ -110,14 +115,14 @@ fn post_jose_type( accept_type: &str, ) -> Result<(Response, String), Error> { let uri = url.parse::()?; - let mut request = new_request(cert, root_certs, &uri, Method::POST); - request.header("Content-Type", CONTENT_TYPE_JOSE); - request.header("Content-Length", &data.len().to_string()); - request.header("Accept", accept_type); - request.body(data); + let mut request = new_request(root_certs, &uri, Method::POST); + request.r.header("Content-Type", CONTENT_TYPE_JOSE); + request.r.header("Content-Length", &data.len().to_string()); + request.r.header("Accept", accept_type); + request.r.body(data); let rstr = String::from_utf8_lossy(data); cert.trace(&format!("request body: {}", rstr)); - let (res, res_body) = send_request(&request)?; + let (res, res_body) = send_request(cert, &request)?; cert.trace(&format!("response body: {}", res_body)); Ok((res, res_body)) } @@ -287,8 +292,8 @@ pub fn get_directory( url: &str, ) -> Result { let uri = url.parse::()?; - let mut request = new_request(cert, root_certs, &uri, Method::GET); - request.header("Accept", CONTENT_TYPE_JSON); + let mut request = new_request(root_certs, &uri, Method::GET); + request.r.header("Accept", CONTENT_TYPE_JSON); let (r, s) = send_request_retry(cert, &request)?; check_response(&r, &s)?; Directory::from_str(&s) @@ -296,7 +301,7 @@ pub fn get_directory( pub fn get_nonce(cert: &Certificate, root_certs: &[String], url: &str) -> Result { let uri = url.parse::()?; - let request = new_request(cert, root_certs, &uri, Method::HEAD); + let request = new_request(root_certs, &uri, Method::HEAD); let (res, res_body) = send_request_retry(cert, &request)?; check_response(&res, &res_body)?; nonce_from_response(cert, &res) diff --git a/acmed/src/certificate.rs b/acmed/src/certificate.rs index 77a6e73..76104df 100644 --- a/acmed/src/certificate.rs +++ b/acmed/src/certificate.rs @@ -7,6 +7,7 @@ use log::{debug, info, trace, warn}; use openssl::x509::X509; use std::collections::{HashMap, HashSet}; use std::fmt; +use std::sync::mpsc::SyncSender; use time::{strptime, Duration}; #[derive(Clone, Debug)] @@ -49,6 +50,7 @@ pub struct Certificate { pub kp_reuse: bool, pub remote_url: String, pub tos_agreed: bool, + pub https_throttle: SyncSender, pub hooks: Vec, pub account_directory: String, pub crt_directory: String, @@ -245,8 +247,10 @@ impl Certificate { mod tests { use super::{Algorithm, Certificate}; use std::collections::HashMap; + use std::sync::mpsc::sync_channel; fn get_dummy_certificate() -> Certificate { + let (https_throttle, _) = sync_channel(0); Certificate { account: crate::config::Account { name: String::new(), @@ -257,6 +261,7 @@ mod tests { kp_reuse: false, remote_url: String::new(), tos_agreed: false, + https_throttle, hooks: Vec::new(), account_directory: String::new(), crt_directory: String::new(), diff --git a/acmed/src/config.rs b/acmed/src/config.rs index f41de6f..f8cb492 100644 --- a/acmed/src/config.rs +++ b/acmed/src/config.rs @@ -1,13 +1,15 @@ use crate::certificate::Algorithm; use crate::hooks; +use crate::rate_limits; use acme_common::error::Error; use log::info; use serde::Deserialize; use std::collections::HashMap; -use std::fmt; use std::fs::{self, File}; use std::io::prelude::*; use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::{fmt, thread}; macro_rules! set_cfg_attr { ($to: expr, $from: expr) => { @@ -42,6 +44,8 @@ pub struct Config { pub global: Option, #[serde(default)] pub endpoint: Vec, + #[serde(default, rename = "rate-limit")] + pub rate_limit: Vec, #[serde(default)] pub hook: Vec, #[serde(default)] @@ -55,6 +59,15 @@ pub struct Config { } impl Config { + fn get_rate_limit(&self, name: &str) -> Result<(usize, String), Error> { + for rl in self.rate_limit.iter() { + if rl.name == name { + return Ok((rl.number, rl.period.to_owned())); + } + } + Err(format!("{}: rate limit not found", name).into()) + } + pub fn get_account_dir(&self) -> String { let account_dir = match &self.global { Some(g) => match &g.accounts_directory { @@ -167,6 +180,27 @@ pub struct Endpoint { pub name: String, pub url: String, pub tos_agreed: bool, + #[serde(default)] + pub rate_limits: Vec, +} + +impl Endpoint { + fn is_used(&self, config: &Config) -> bool { + for crt in config.certificate.iter() { + if crt.endpoint == self.name { + return true; + } + } + false + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RateLimit { + pub name: String, + pub number: usize, + pub period: String, } #[derive(Deserialize)] @@ -304,6 +338,11 @@ impl Certificate { Ok(ep.url) } + pub fn get_endpoint_name(&self, cnf: &Config) -> Result { + let ep = self.get_endpoint(cnf)?; + Ok(ep.name) + } + pub fn get_tos_agreement(&self, cnf: &Config) -> Result { let ep = self.get_endpoint(cnf)?; Ok(ep.tos_agreed) @@ -368,6 +407,7 @@ fn read_cnf(path: &PathBuf) -> Result { let cnf_path = get_cnf_path(path, cnf_name); let mut add_cnf = read_cnf(&cnf_path)?; config.endpoint.append(&mut add_cnf.endpoint); + config.rate_limit.append(&mut add_cnf.rate_limit); config.hook.append(&mut add_cnf.hook); config.group.append(&mut add_cnf.group); config.account.append(&mut add_cnf.account); @@ -407,6 +447,24 @@ fn dispatch_global_env_vars(config: &mut Config) { } } +pub fn init_rate_limits( + config: &Config, +) -> Result>, Error> { + let mut corresp = HashMap::new(); + for endpoint in config.endpoint.iter() { + if endpoint.is_used(config) { + let mut limits = Vec::new(); + for l in endpoint.rate_limits.iter() { + limits.push(config.get_rate_limit(l)?); + } + let mut rl = rate_limits::RateLimit::new(&limits)?; + corresp.insert(endpoint.name.to_owned(), rl.get_sender()); + thread::spawn(move || rl.run()); + } + } + Ok(corresp) +} + pub fn from_file(file_name: &str) -> Result { let path = PathBuf::from(file_name); let mut config = read_cnf(&path)?; diff --git a/acmed/src/main.rs b/acmed/src/main.rs index 5dc4dbf..f2ee323 100644 --- a/acmed/src/main.rs +++ b/acmed/src/main.rs @@ -8,6 +8,7 @@ mod certificate; mod config; mod hooks; mod main_event_loop; +mod rate_limits; mod storage; pub const APP_NAME: &str = "ACMEd"; @@ -30,6 +31,8 @@ pub const DEFAULT_POOL_WAIT_SEC: u64 = 5; pub const DEFAULT_HTTP_FAIL_NB_RETRY: usize = 10; pub const DEFAULT_HTTP_FAIL_WAIT_SEC: u64 = 1; pub const DEFAULT_HOOK_ALLOW_FAILURE: bool = false; +pub const MAX_RATE_LIMIT_SLEEP_MILISEC: u64 = 3_600_000; +pub const MIN_RATE_LIMIT_SLEEP_MILISEC: u64 = 100; fn main() { let full_version = format!( diff --git a/acmed/src/main_event_loop.rs b/acmed/src/main_event_loop.rs index 398ab10..2b948e1 100644 --- a/acmed/src/main_event_loop.rs +++ b/acmed/src/main_event_loop.rs @@ -31,9 +31,20 @@ pub struct MainEventLoop { impl MainEventLoop { pub fn new(config_file: &str, root_certs: &[&str]) -> Result { let cnf = config::from_file(config_file)?; + let rate_limits_corresp = config::init_rate_limits(&cnf)?; let mut certs = Vec::new(); for (i, crt) in cnf.certificate.iter().enumerate() { + let ep_name = crt.get_endpoint_name(&cnf)?; + let https_throttle = rate_limits_corresp + .get(&ep_name) + .ok_or_else(|| { + Error::from(format!( + "{}: rate limit not found for this endpoint", + ep_name + )) + })? + .to_owned(); let cert = Certificate { account: crt.get_account(&cnf)?, domains: crt.domains.to_owned(), @@ -41,6 +52,7 @@ impl MainEventLoop { kp_reuse: crt.get_kp_reuse(), remote_url: crt.get_remote_url(&cnf)?, tos_agreed: crt.get_tos_agreement(&cnf)?, + https_throttle, hooks: crt.get_hooks(&cnf)?, account_directory: cnf.get_account_dir(), crt_directory: crt.get_crt_dir(&cnf), diff --git a/acmed/src/rate_limits.rs b/acmed/src/rate_limits.rs new file mode 100644 index 0000000..ecf028a --- /dev/null +++ b/acmed/src/rate_limits.rs @@ -0,0 +1,188 @@ +use acme_common::error::Error; +use nom::bytes::complete::take_while_m_n; +use nom::character::complete::digit1; +use nom::combinator::map_res; +use nom::multi::fold_many1; +use nom::IResult; +use std::cmp; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +pub enum Request { + HttpsRequest, +} + +pub struct RateLimit { + limits: Vec<(usize, Duration)>, + sender: mpsc::SyncSender, + receiver: mpsc::Receiver, + log: Vec, +} + +impl RateLimit { + pub fn new(limits: &[(usize, String)]) -> Result { + let mut max_size = 0; + let mut parsed_limits = Vec::new(); + for (nb, raw_duration) in limits.iter() { + if *nb > max_size { + max_size = *nb; + } + let parsed_duration = parse_duration(raw_duration)?; + parsed_limits.push((*nb, parsed_duration)); + } + parsed_limits.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap()); + parsed_limits.reverse(); + let (sender, receiver) = mpsc::sync_channel::(0); + Ok(RateLimit { + limits: parsed_limits, + sender, + receiver, + log: Vec::with_capacity(max_size), + }) + } + + pub fn get_sender(&self) -> mpsc::SyncSender { + self.sender.clone() + } + + pub fn run(&mut self) -> Result<(), Error> { + let sleep_duration = self.get_sleep_duration(); + loop { + self.prune_log(); + if self.request_allowed() { + match self.receiver.recv()? { + Request::HttpsRequest => { + if !self.limits.is_empty() { + self.log.push(Instant::now()); + } + } + } + } else { + // TODO: find a better sleep duration + thread::sleep(sleep_duration); + } + } + } + + fn get_sleep_duration(&self) -> Duration { + let (nb_req, min_duration) = match self.limits.last() { + Some((n, d)) => (*n as u64, *d), + None => { + return Duration::from_millis(0); + } + }; + let nb_mili = match min_duration.as_secs() { + 0 | 1 => crate::MIN_RATE_LIMIT_SLEEP_MILISEC, + n => { + let a = n * 200 / nb_req; + let a = cmp::min(a, crate::MAX_RATE_LIMIT_SLEEP_MILISEC); + cmp::max(a, crate::MIN_RATE_LIMIT_SLEEP_MILISEC) + } + }; + Duration::from_millis(nb_mili) + } + + fn request_allowed(&self) -> bool { + for (max_allowed, duration) in self.limits.iter() { + let max_date = Instant::now() - *duration; + let nb_req = self.log.iter().filter(move |x| **x > max_date).count(); + if nb_req >= *max_allowed { + return false; + } + } + true + } + + fn prune_log(&mut self) { + if let Some((_, max_limit)) = self.limits.first() { + let prune_date = Instant::now() - *max_limit; + self.log.retain(move |&d| d > prune_date); + } + } +} + +fn is_duration_chr(c: char) -> bool { + c == 's' || c == 'm' || c == 'h' || c == 'd' || c == 'w' +} + +fn get_multiplicator(input: &str) -> IResult<&str, u64> { + let (input, nb) = take_while_m_n(1, 1, is_duration_chr)(input)?; + let mult = match nb.chars().nth(0) { + Some('s') => 1, + Some('m') => 60, + Some('h') => 3_600, + Some('d') => 86_400, + Some('w') => 604_800, + _ => 0, + }; + Ok((input, mult)) +} + +fn get_duration_part(input: &str) -> IResult<&str, Duration> { + let (input, nb) = map_res(digit1, |s: &str| s.parse::())(input)?; + let (input, mult) = get_multiplicator(input)?; + Ok((input, Duration::from_secs(nb * mult))) +} + +fn get_duration(input: &str) -> IResult<&str, Duration> { + fold_many1( + get_duration_part, + Duration::new(0, 0), + |mut acc: Duration, item| { + acc += item; + acc + }, + )(input) +} + +fn parse_duration(input: &str) -> Result { + match get_duration(input) { + Ok((r, d)) => match r.len() { + 0 => Ok(d), + _ => Err(format!("{}: invalid duration", input).into()), + }, + Err(_) => Err(format!("{}: invalid duration", input).into()), + } +} + +#[cfg(test)] +mod tests { + use super::{parse_duration, RateLimit}; + + #[test] + fn test_rate_limit_build() { + let l = vec![ + (5, String::from("5s")), + (12, String::from("2m")), + (8, String::from("5m")), + (1, String::from("1s")), + (2, String::from("1m")), + ]; + let rl = RateLimit::new(l.as_slice()).unwrap(); + let ref_t = (8_usize, parse_duration("5m").unwrap()); + assert_eq!(rl.limits.first(), Some(&ref_t)); + assert_eq!(rl.log.len(), 0); + assert_eq!(rl.log.capacity(), 12); + } + + #[test] + fn test_parse_duration() { + let lst = [ + ("42s", 42), + ("21m", 1_260), + ("3h", 10_800), + ("2d", 172_800), + ("1w", 604_800), + ("42m30s", 2_550), + ("30s42m", 2_550), + ("3h5m12s", 11_112), + ("40s2s", 42), + ]; + for (fmt, ref_sec) in lst.iter() { + let d = parse_duration(fmt); + assert!(d.is_ok()); + assert_eq!(d.unwrap().as_secs(), *ref_sec); + } + } +} diff --git a/man/en/acmed.toml.5 b/man/en/acmed.toml.5 index b566746..a9be258 100644 --- a/man/en/acmed.toml.5 +++ b/man/en/acmed.toml.5 @@ -63,6 +63,18 @@ Specify the group who will own newly-created private-key files. See .Xr chown 2 for more details. .El +.It Ic rate-limit +Array of table where each element defines a HTTPS rate limit. +.Bl -tag +.It Cm name Ar string +The name the rate limit is registered under. Must be unique. +.It Cm number Ar integer +Number of requests authorized withing the time period. +.It Cm period Ar string +Period of time during which a maximal number of requests is authorized. The format is described in the +.Sx TIME PERIODS +section. +.El .It Ic endpoint Array of table where each element defines a Certificate Authority .Pq CA @@ -70,11 +82,13 @@ which may be used to request certificates. .Bl -tag .It Cm name Ar string The name the endpoint is registered under. Must be unique. -.It Cm url Ar string -The endpoint's directory URL. +.It Cm rate_limits Ar array +Array containing the names of the HTTPS rate limits to apply. .It Cm tos_agreed Ar boolean Set whether or not the user agrees to the Terms Of Service .Pq TOS . +.It Cm url Ar string +The endpoint's directory URL. .El .It Ic hook Array of table where each element defines a command that will be launched at a defined point. See section @@ -442,6 +456,39 @@ If is not specified, it will be set to .Pa /run . .El +.Sh TIME PERIODS +ACMEd uses its own time period format, which is vaguely inspired by the ISO 8601 one. Periods are formatted as +.Ar PM[PM...] +where +.Ar M +is case sensitive character representing a length and +.Ar P +is an integer representing a multiplayer for the following length. The authorized length are: +.Bl -dash -compact +.It +.Ar s : +second +.It +.Ar m : +minute +.It +.Ar h : +hour +.It +.Ar d : +day +.It +.Ar w : +week +.El +The +.Ar PM +couples can be specified multiple times and in any order. +.Pp +For example, +.Dq 1d42s and +.Dq 40s20h4h2s +both represents a period of one day and forty-two seconds. .Sh FILES .Bl -tag .It Pa /etc/acmed/acmed.toml