// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use rustfs_config::{ DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY, }; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; use rustls::sign::CertifiedKey; use std::collections::HashMap; use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; use std::io::{self, Error}; use std::sync::{Arc, RwLock}; use std::time::Duration; use tokio::sync::watch; use tokio::task::JoinHandle; use tokio::time::MissedTickBehavior; use tracing::{debug, info, warn}; #[derive(Debug)] struct ResolverState { cert_resolver: ResolvesServerCertUsingSni, default_cert: Option>, cert_count: usize, fingerprint: u64, } impl ResolverState { fn load_from_directory(cert_dir: &str) -> io::Result { let cert_key_pairs = rustfs_utils::load_all_certs_from_directory( rustfs_utils::CertDirectoryLoadOptions::builder(cert_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(), )?; if cert_key_pairs.is_empty() { return Err(Error::other("No valid certificates found in directory")); } Self::from_cert_key_pairs(cert_key_pairs) } fn from_cert_key_pairs( cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, ) -> io::Result { let cert_count = cert_key_pairs.len(); let mut cert_resolver = ResolvesServerCertUsingSni::new(); let mut default_cert = None; let mut entries = cert_key_pairs.into_iter().collect::>(); entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain)); let fingerprint = fingerprint_tls_entries(&entries); for (domain, (certs, key)) in entries { let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) .map_err(|e| Error::other(format!("unsupported private key type for {domain}: {e:?}")))?; let certified_key = CertifiedKey::new(certs, signing_key); if domain.as_str() == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { cert_resolver .add(&domain, certified_key) .map_err(|e| Error::other(format!("failed to add certificate for {domain}: {e:?}")))?; } } Ok(Self { cert_resolver, default_cert, cert_count, fingerprint, }) } } fn fingerprint_tls_entries(entries: &[(String, (Vec>, PrivateKeyDer<'static>))]) -> u64 { let mut hasher = DefaultHasher::new(); for (domain, (certs, key)) in entries { hasher.write_usize(domain.len()); hasher.write(domain.as_bytes()); hasher.write_usize(certs.len()); for cert in certs { hasher.write_usize(cert.as_ref().len()); hasher.write(cert.as_ref()); } hasher.write_usize(key.secret_der().len()); hasher.write(key.secret_der()); } hasher.finish() } #[derive(Debug)] pub(crate) struct ReloadableCertResolver { current: RwLock, } impl ReloadableCertResolver { pub(crate) fn load_from_directory(cert_dir: &str) -> io::Result> { let state = ResolverState::load_from_directory(cert_dir)?; Ok(Arc::new(Self { current: RwLock::new(state), })) } pub(crate) fn reload_from_directory(&self, cert_dir: &str) -> io::Result> { let new_state = ResolverState::load_from_directory(cert_dir)?; match self.current.write() { Ok(mut guard) => { if guard.fingerprint == new_state.fingerprint { return Ok(None); } let cert_count = new_state.cert_count; *guard = new_state; Ok(Some(cert_count)) } Err(poisoned) => { let mut guard = poisoned.into_inner(); if guard.fingerprint == new_state.fingerprint { return Ok(None); } let cert_count = new_state.cert_count; *guard = new_state; Ok(Some(cert_count)) } } } } impl ResolvesServerCert for ReloadableCertResolver { fn resolve(&self, client_hello: ClientHello) -> Option> { let guard = match self.current.read() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; guard .cert_resolver .resolve(client_hello) .or_else(|| guard.default_cert.clone()) } } pub(crate) fn spawn_cert_reload_loop( protocol: &'static str, cert_dir: String, resolver: Arc, mut shutdown_rx: watch::Receiver, ) -> Option> { let enabled = rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); if !enabled { debug!( protocol, "TLS certificate hot reload is disabled (set {}=1 to enable)", ENV_TLS_RELOAD_ENABLE ); return None; } let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); info!( protocol, cert_dir = %cert_dir, "TLS certificate hot reload enabled, checking every {}s", interval_secs ); Some(tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); interval.tick().await; loop { tokio::select! { changed = shutdown_rx.changed() => { match changed { Ok(()) => { if *shutdown_rx.borrow() { info!(protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped"); break; } continue; } Err(_) => { info!( protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped because the shutdown channel closed" ); break; } } } _ = interval.tick() => {} } match resolver.reload_from_directory(&cert_dir) { Ok(Some(cert_count)) => { info!( protocol, cert_dir = %cert_dir, cert_count, "TLS certificates reloaded successfully" ); } Ok(None) => { debug!(protocol, cert_dir = %cert_dir, "TLS certificate material unchanged; skipping reload"); } Err(e) => { warn!( protocol, cert_dir = %cert_dir, "TLS certificate reload failed (will retry): {}", e ); } } } })) } #[cfg(test)] mod tests { use super::*; use rcgen::generate_simple_self_signed; use std::fs; use tempfile::TempDir; fn cert_key_pair(san: &str) -> (Vec>, PrivateKeyDer<'static>) { let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap(); ( vec![cert.cert.der().clone()], PrivateKeyDer::try_from(cert.signing_key.serialize_der()).unwrap(), ) } fn clone_cert_key_pair( cert_key_pair: &(Vec>, PrivateKeyDer<'static>), ) -> (Vec>, PrivateKeyDer<'static>) { (cert_key_pair.0.clone(), cert_key_pair.1.clone_key()) } fn write_default_cert(dir: &std::path::Path, san: &str) { let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap(); fs::write(dir.join(RUSTFS_TLS_CERT), cert.cert.pem()).unwrap(); fs::write(dir.join(RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).unwrap(); } #[test] fn reload_from_directory_replaces_default_certificate() { let temp_dir = TempDir::new().unwrap(); write_default_cert(temp_dir.path(), "localhost"); let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); let before = { let guard = resolver.current.read().unwrap(); guard.default_cert.as_ref().unwrap().clone() }; write_default_cert(temp_dir.path(), "rotated.local"); let cert_count = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); assert_eq!(cert_count, Some(1)); let after = { let guard = resolver.current.read().unwrap(); guard.default_cert.as_ref().unwrap().clone() }; assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref()); } #[test] fn reload_from_directory_skips_when_material_is_unchanged() { let temp_dir = TempDir::new().unwrap(); write_default_cert(temp_dir.path(), "localhost"); let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); let outcome = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); assert_eq!(outcome, None); } #[test] fn resolver_state_fingerprint_is_stable_across_domain_ordering() { let default_cert = cert_key_pair("localhost"); let api_cert = cert_key_pair("api.example.com"); let web_cert = cert_key_pair("web.example.com"); let mut first = HashMap::new(); first.insert("default".to_string(), clone_cert_key_pair(&default_cert)); first.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert)); first.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert)); let mut second = HashMap::new(); second.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert)); second.insert("default".to_string(), clone_cert_key_pair(&default_cert)); second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert)); let first_state = ResolverState::from_cert_key_pairs(first).unwrap(); let second_state = ResolverState::from_cert_key_pairs(second).unwrap(); assert_eq!(first_state.cert_count, 3); assert_eq!(second_state.cert_count, 3); assert_eq!(first_state.fingerprint, second_state.fingerprint); } }