fix(protocols): add hot reload for WebDAV FTPS and SFTP (#2922)

Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-08-30 08:49:26 +00:00 · 2026-05-12 00:25:20 +08:00
parent 8ef2c0a5e1
commit 6185d38911
10 changed files with 586 additions and 46 deletions
@@ -0,0 +1,279 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::{
+    DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, RUSTFS_TLS_CERT,
+    RUSTFS_TLS_KEY,
+};
+use rustls::pki_types::{CertificateDer, PrivateKeyDer};
+use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
+use rustls::sign::CertifiedKey;
+use std::collections::HashMap;
+use std::collections::hash_map::DefaultHasher;
+use std::hash::Hasher;
+use std::io::{self, Error};
+use std::sync::{Arc, RwLock};
+use std::time::Duration;
+use tokio::sync::watch;
+use tokio::task::JoinHandle;
+use tokio::time::MissedTickBehavior;
+use tracing::{debug, info, warn};
+
+#[derive(Debug)]
+struct ResolverState {
+    cert_resolver: ResolvesServerCertUsingSni,
+    default_cert: Option<Arc<CertifiedKey>>,
+    cert_count: usize,
+    fingerprint: u64,
+}
+
+impl ResolverState {
+    fn load_from_directory(cert_dir: &str) -> io::Result<Self> {
+        let cert_key_pairs = rustfs_utils::load_all_certs_from_directory(
+            rustfs_utils::CertDirectoryLoadOptions::builder(cert_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(),
+        )?;
+        if cert_key_pairs.is_empty() {
+            return Err(Error::other("No valid certificates found in directory"));
+        }
+
+        Self::from_cert_key_pairs(cert_key_pairs)
+    }
+
+    fn from_cert_key_pairs(
+        cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
+    ) -> io::Result<Self> {
+        let cert_count = cert_key_pairs.len();
+        let mut cert_resolver = ResolvesServerCertUsingSni::new();
+        let mut default_cert = None;
+        let mut entries = cert_key_pairs.into_iter().collect::<Vec<_>>();
+        entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain));
+        let fingerprint = fingerprint_tls_entries(&entries);
+
+        for (domain, (certs, key)) in entries {
+            let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
+                .map_err(|e| Error::other(format!("unsupported private key type for {domain}: {e:?}")))?;
+            let certified_key = CertifiedKey::new(certs, signing_key);
+
+            if domain.as_str() == "default" {
+                default_cert = Some(Arc::new(certified_key.clone()));
+            } else {
+                cert_resolver
+                    .add(&domain, certified_key)
+                    .map_err(|e| Error::other(format!("failed to add certificate for {domain}: {e:?}")))?;
+            }
+        }
+
+        Ok(Self {
+            cert_resolver,
+            default_cert,
+            cert_count,
+            fingerprint,
+        })
+    }
+}
+
+fn fingerprint_tls_entries(entries: &[(String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>))]) -> u64 {
+    let mut hasher = DefaultHasher::new();
+
+    for (domain, (certs, key)) in entries {
+        hasher.write_usize(domain.len());
+        hasher.write(domain.as_bytes());
+        hasher.write_usize(certs.len());
+        for cert in certs {
+            hasher.write_usize(cert.as_ref().len());
+            hasher.write(cert.as_ref());
+        }
+        hasher.write_usize(key.secret_der().len());
+        hasher.write(key.secret_der());
+    }
+
+    hasher.finish()
+}
+
+#[derive(Debug)]
+pub(crate) struct ReloadableCertResolver {
+    current: RwLock<ResolverState>,
+}
+
+impl ReloadableCertResolver {
+    pub(crate) fn load_from_directory(cert_dir: &str) -> io::Result<Arc<Self>> {
+        let state = ResolverState::load_from_directory(cert_dir)?;
+        Ok(Arc::new(Self {
+            current: RwLock::new(state),
+        }))
+    }
+
+    pub(crate) fn reload_from_directory(&self, cert_dir: &str) -> io::Result<Option<usize>> {
+        let new_state = ResolverState::load_from_directory(cert_dir)?;
+
+        match self.current.write() {
+            Ok(mut guard) => {
+                if guard.fingerprint == new_state.fingerprint {
+                    return Ok(None);
+                }
+                let cert_count = new_state.cert_count;
+                *guard = new_state;
+                Ok(Some(cert_count))
+            }
+            Err(poisoned) => {
+                let mut guard = poisoned.into_inner();
+                if guard.fingerprint == new_state.fingerprint {
+                    return Ok(None);
+                }
+                let cert_count = new_state.cert_count;
+                *guard = new_state;
+                Ok(Some(cert_count))
+            }
+        }
+    }
+}
+
+impl ResolvesServerCert for ReloadableCertResolver {
+    fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
+        let guard = match self.current.read() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+
+        guard
+            .cert_resolver
+            .resolve(client_hello)
+            .or_else(|| guard.default_cert.clone())
+    }
+}
+
+pub(crate) fn spawn_cert_reload_loop(
+    protocol: &'static str,
+    cert_dir: String,
+    resolver: Arc<ReloadableCertResolver>,
+    mut shutdown_rx: watch::Receiver<bool>,
+) -> Option<JoinHandle<()>> {
+    let enabled = rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE);
+    if !enabled {
+        debug!(
+            protocol,
+            "TLS certificate hot reload is disabled (set {}=1 to enable)", ENV_TLS_RELOAD_ENABLE
+        );
+        return None;
+    }
+
+    let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5);
+    info!(
+        protocol,
+        cert_dir = %cert_dir,
+        "TLS certificate hot reload enabled, checking every {}s",
+        interval_secs
+    );
+
+    Some(tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
+        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+        interval.tick().await;
+
+        loop {
+            tokio::select! {
+                changed = shutdown_rx.changed() => {
+                    match changed {
+                        Ok(()) => {
+                            if *shutdown_rx.borrow() {
+                                info!(protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped");
+                                break;
+                            }
+                            continue;
+                        }
+                        Err(_) => {
+                            info!(
+                                protocol,
+                                cert_dir = %cert_dir,
+                                "TLS certificate hot reload task stopped because the shutdown channel closed"
+                            );
+                            break;
+                        }
+                    }
+                }
+                _ = interval.tick() => {}
+            }
+
+            match resolver.reload_from_directory(&cert_dir) {
+                Ok(Some(cert_count)) => {
+                    info!(
+                        protocol,
+                        cert_dir = %cert_dir,
+                        cert_count,
+                        "TLS certificates reloaded successfully"
+                    );
+                }
+                Ok(None) => {
+                    debug!(protocol, cert_dir = %cert_dir, "TLS certificate material unchanged; skipping reload");
+                }
+                Err(e) => {
+                    warn!(
+                        protocol,
+                        cert_dir = %cert_dir,
+                        "TLS certificate reload failed (will retry): {}",
+                        e
+                    );
+                }
+            }
+        }
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rcgen::generate_simple_self_signed;
+    use std::fs;
+    use tempfile::TempDir;
+
+    fn write_default_cert(dir: &std::path::Path, san: &str) {
+        let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap();
+        fs::write(dir.join(RUSTFS_TLS_CERT), cert.cert.pem()).unwrap();
+        fs::write(dir.join(RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).unwrap();
+    }
+
+    #[test]
+    fn reload_from_directory_replaces_default_certificate() {
+        let temp_dir = TempDir::new().unwrap();
+        write_default_cert(temp_dir.path(), "localhost");
+
+        let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
+        let before = {
+            let guard = resolver.current.read().unwrap();
+            guard.default_cert.as_ref().unwrap().clone()
+        };
+
+        write_default_cert(temp_dir.path(), "rotated.local");
+
+        let cert_count = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
+        assert_eq!(cert_count, Some(1));
+
+        let after = {
+            let guard = resolver.current.read().unwrap();
+            guard.default_cert.as_ref().unwrap().clone()
+        };
+
+        assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref());
+    }
+
+    #[test]
+    fn reload_from_directory_skips_when_material_is_unchanged() {
+        let temp_dir = TempDir::new().unwrap();
+        write_default_cert(temp_dir.path(), "localhost");
+
+        let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
+        let outcome = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
+        assert_eq!(outcome, None);
+    }
+}