// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory}; use crate::config::TlsReloadOptions; use crate::error::TlsRuntimeError; use crate::material::{ServerTlsMaterial, server_material_fingerprint}; use crate::metrics::{record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped}; use crate::source::TlsSource; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; use rustls::sign::CertifiedKey; use std::collections::HashMap; use std::io; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; use tokio::sync::watch; use tokio::task::JoinHandle; use tokio::time::MissedTickBehavior; use tracing::{debug, info, warn}; #[derive(Debug)] struct ResolverState { cert_resolver: ResolvesServerCertUsingSni, default_cert: Option>, cert_count: usize, fingerprint: crate::fingerprint::TlsFingerprint, } impl ResolverState { fn load_from_source(source: &TlsSource) -> Result { let base_dir = source.validate_directory()?; let cert_key_pairs = load_all_certs_from_directory( CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) .build(), )?; if cert_key_pairs.is_empty() { return Err(TlsRuntimeError::Material("No valid certificates found in directory".to_string())); } Self::from_cert_key_pairs(cert_key_pairs) } fn from_cert_key_pairs( cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, ) -> Result { let cert_count = cert_key_pairs.len(); let mut cert_resolver = ResolvesServerCertUsingSni::new(); let mut default_cert = None; let mut entries = cert_key_pairs.into_iter().collect::>(); entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain)); let material = ServerTlsMaterial::MultiCert { cert_key_pairs: entries .iter() .map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key()))) .collect(), }; let fingerprint = server_material_fingerprint(&material); for (domain, (certs, key)) in entries { let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) .map_err(|e| io::Error::other(format!("unsupported private key type for {domain}: {e:?}")))?; let certified_key = CertifiedKey::new(certs, signing_key); if domain.as_str() == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { cert_resolver .add(&domain, certified_key) .map_err(|e| io::Error::other(format!("failed to add certificate for {domain}: {e:?}")))?; } } Ok(Self { cert_resolver, default_cert, cert_count, fingerprint, }) } } #[derive(Debug)] pub struct ReloadableServerCertResolver { source: TlsSource, current: RwLock, generation: AtomicU64, } impl ReloadableServerCertResolver { pub fn load_from_source(source: TlsSource) -> Result, TlsRuntimeError> { let state = ResolverState::load_from_source(&source)?; record_tls_generation("server_resolver", 1); Ok(Arc::new(Self { source, current: RwLock::new(state), generation: AtomicU64::new(1), })) } pub fn load_from_directory(cert_dir: &str) -> Result, TlsRuntimeError> { Self::load_from_source(TlsSource::from_directory(cert_dir)) } pub fn reload(&self) -> Result, TlsRuntimeError> { let new_state = ResolverState::load_from_source(&self.source)?; match self.current.write() { Ok(mut guard) => { if guard.fingerprint == new_state.fingerprint { return Ok(None); } let cert_count = new_state.cert_count; *guard = new_state; self.generation.fetch_add(1, Ordering::Relaxed); Ok(Some(cert_count)) } Err(poisoned) => { let mut guard = poisoned.into_inner(); if guard.fingerprint == new_state.fingerprint { return Ok(None); } let cert_count = new_state.cert_count; *guard = new_state; self.generation.fetch_add(1, Ordering::Relaxed); Ok(Some(cert_count)) } } } pub fn generation(&self) -> u64 { self.generation.load(Ordering::Relaxed) } } impl ResolvesServerCert for ReloadableServerCertResolver { fn resolve(&self, client_hello: ClientHello<'_>) -> Option> { let guard = match self.current.read() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; guard .cert_resolver .resolve(client_hello) .or_else(|| guard.default_cert.clone()) } } pub fn spawn_server_cert_reload_loop( protocol: &'static str, resolver: Arc, options: TlsReloadOptions, mut shutdown_rx: watch::Receiver, ) -> Option> { if !options.enabled { debug!(protocol, "TLS certificate hot reload is disabled"); return None; } info!( protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload enabled, checking every {}s", options.interval.as_secs() ); Some(tokio::spawn(async move { let mut interval = tokio::time::interval(options.interval); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); interval.tick().await; loop { tokio::select! { changed = shutdown_rx.changed() => { match changed { Ok(()) => { if *shutdown_rx.borrow() { info!(protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped"); break; } continue; } Err(_) => { info!( protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped because the shutdown channel closed" ); break; } } } _ = interval.tick() => {} } match resolver.reload() { Ok(Some(cert_count)) => { record_tls_reload_result(protocol, "ok", None, Some(resolver.generation())); info!( protocol, cert_dir = %resolver.source.base_dir.display(), cert_count, "TLS certificates reloaded successfully" ); } Ok(None) => { record_tls_reload_skipped(protocol, "unchanged"); debug!( protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate material unchanged; skipping reload" ); } Err(e) => { record_tls_publication_fail(protocol); warn!( protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate reload failed (will retry): {}", e ); } } } })) } #[cfg(test)] mod tests { use super::*; use rcgen::generate_simple_self_signed; use std::fs; use tempfile::TempDir; fn cert_key_pair(san: &str) -> (Vec>, PrivateKeyDer<'static>) { let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate"); ( vec![cert.cert.der().clone()], PrivateKeyDer::try_from(cert.signing_key.serialize_der()).expect("key should convert"), ) } fn clone_cert_key_pair( cert_key_pair: &(Vec>, PrivateKeyDer<'static>), ) -> (Vec>, PrivateKeyDer<'static>) { (cert_key_pair.0.clone(), cert_key_pair.1.clone_key()) } fn write_default_cert(dir: &std::path::Path, san: &str) { let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate"); fs::write(dir.join(rustfs_config::RUSTFS_TLS_CERT), cert.cert.pem()).expect("cert should write"); fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).expect("key should write"); } #[test] fn reload_replaces_default_certificate() { let temp_dir = TempDir::new().expect("tempdir should create"); write_default_cert(temp_dir.path(), "localhost"); let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8")) .expect("resolver should load"); let before = { let guard = resolver.current.read().expect("lock should acquire"); guard.default_cert.as_ref().expect("default cert should exist").clone() }; write_default_cert(temp_dir.path(), "rotated.local"); let cert_count = resolver.reload().expect("reload should succeed"); assert_eq!(cert_count, Some(1)); let after = { let guard = resolver.current.read().expect("lock should acquire"); guard.default_cert.as_ref().expect("default cert should exist").clone() }; assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref()); } #[test] fn reload_skips_when_material_is_unchanged() { let temp_dir = TempDir::new().expect("tempdir should create"); write_default_cert(temp_dir.path(), "localhost"); let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8")) .expect("resolver should load"); let outcome = resolver.reload().expect("reload should succeed"); assert_eq!(outcome, None); } #[test] fn resolver_state_fingerprint_is_stable_across_domain_ordering() { let default_cert = cert_key_pair("localhost"); let api_cert = cert_key_pair("api.example.com"); let web_cert = cert_key_pair("web.example.com"); let mut first = HashMap::new(); first.insert("default".to_string(), clone_cert_key_pair(&default_cert)); first.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert)); first.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert)); let mut second = HashMap::new(); second.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert)); second.insert("default".to_string(), clone_cert_key_pair(&default_cert)); second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert)); let first_state = ResolverState::from_cert_key_pairs(first).expect("first state should build"); let second_state = ResolverState::from_cert_key_pairs(second).expect("second state should build"); assert_eq!(first_state.cert_count, 3); assert_eq!(second_state.cert_count, 3); assert_eq!(first_state.fingerprint, second_state.fingerprint); } }