refactor(tls): centralize runtime foundation (#3065)

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* feat(tls-runtime): add TLS debug state and admin handler

* refactor(tls-runtime): unify TLS debug consumer status view

* fix(tls): address PR3065 review feedback

* refactor(tls): align debug status payload types

* refactor(targets): harden TLS hot reload paths

* fix(targets): resolve review-4348251652 findings

* fix(targets): finalize tls runtime review follow-ups

* fix(targets): harden tls reload and review follow-ups

* fix(targets): align tls reload handling across targets

* fix(targets): finalize tls reload state and metrics updates

* chore(deps): trim unused TLS deps

* style(targets): normalize TLS reload formatting

* refactor(targets): introduce tls runtime adapter path

* chore: update workspace manifests for tls refactor

* fix(tls): stabilize material reload and audit workflow

* fix(targets): refresh tls fingerprint flow across sinks

* fix(tls): align runtime coordinator and http reader updates

* fix(sftp): simplify protocol error mapping

* fix(tls): harmonize material loading behavior

* fix(server): finalize tls material wiring in startup flow

* fix(protos): tighten tls generation cache and deps

crates/protocols/src/ftps/server.rs

@@ -17,12 +17,14 @@ use super::driver::FtpsDriver;
 use crate::common::client::s3::StorageBackend;
 use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
 use crate::constants::{network::DEFAULT_SOURCE_IP, paths::ROOT_PATH};
-use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
 use libunftp::options::FtpsRequired;
+use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
+use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
 use std::fmt::{Debug, Display, Formatter};
 use std::net::IpAddr;
 use std::path::Path;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::broadcast;
 use tokio::sync::watch;
 use tracing::{debug, error, info, warn};
@@ -68,6 +70,14 @@ impl<S> FtpsServer<S>
 where
     S: StorageBackend + Clone + Send + Sync + 'static + Debug,
 {
+    fn tls_reload_options() -> TlsReloadOptions {
+        TlsReloadOptions {
+            enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
+            interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
+            ..TlsReloadOptions::default()
+        }
+    }
+
     /// Create a new FTPS server
     pub async fn new(config: FtpsConfig, storage: S) -> Result<Self, FtpsInitError> {
         config.validate().await?;
@@ -114,9 +124,14 @@ where
             if let Some(cert_dir) = &self.config.cert_dir {
                 debug!("Enabling FTPS with multi-certificate support from directory: {}", cert_dir);
 
-                let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
+                let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
                     .map_err(|e| FtpsInitError::InvalidConfig(format!("Failed to create certificate resolver: {}", e)))?;
-                let _reload_task = spawn_cert_reload_loop("ftps", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
+                let _reload_task = spawn_server_cert_reload_loop(
+                    "ftps",
+                    resolver.clone(),
+                    Self::tls_reload_options(),
+                    reload_shutdown_rx.clone(),
+                );
 
                 // Build ServerConfig with SNI support
                 let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();

crates/protocols/src/lib.rs

@@ -17,9 +17,6 @@
 pub mod common;
 pub mod constants;
 
-#[cfg(any(feature = "ftps", feature = "webdav"))]
-mod tls_hot_reload;
-
 #[cfg(feature = "ftps")]
 pub mod ftps;
 

crates/protocols/src/sftp/errors.rs

@@ -18,6 +18,7 @@
 
 use super::constants::{http_error_codes, s3_error_codes};
 use russh_sftp::protocol::{Status, StatusCode};
+use russh_sftp::server::StatusReply;
 use s3s::{S3Error, S3ErrorCode};
 use std::{any::Any, fmt::Display};
 
@@ -31,6 +32,12 @@ impl From<SftpError> for StatusCode {
     }
 }
 
+impl From<SftpError> for StatusReply {
+    fn from(err: SftpError) -> Self {
+        StatusReply::new(err.0)
+    }
+}
+
 impl SftpError {
     pub(super) fn code(code: StatusCode) -> Self {
         Self(code)

crates/protocols/src/tls_hot_reload.rs

@@ -1,317 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use rustfs_config::{
-    DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, RUSTFS_TLS_CERT,
-    RUSTFS_TLS_KEY,
-};
-use rustls::pki_types::{CertificateDer, PrivateKeyDer};
-use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
-use rustls::sign::CertifiedKey;
-use std::collections::HashMap;
-use std::collections::hash_map::DefaultHasher;
-use std::hash::Hasher;
-use std::io::{self, Error};
-use std::sync::{Arc, RwLock};
-use std::time::Duration;
-use tokio::sync::watch;
-use tokio::task::JoinHandle;
-use tokio::time::MissedTickBehavior;
-use tracing::{debug, info, warn};
-
-#[derive(Debug)]
-struct ResolverState {
-    cert_resolver: ResolvesServerCertUsingSni,
-    default_cert: Option<Arc<CertifiedKey>>,
-    cert_count: usize,
-    fingerprint: u64,
-}
-
-impl ResolverState {
-    fn load_from_directory(cert_dir: &str) -> io::Result<Self> {
-        let cert_key_pairs = rustfs_utils::load_all_certs_from_directory(
-            rustfs_utils::CertDirectoryLoadOptions::builder(cert_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(),
-        )?;
-        if cert_key_pairs.is_empty() {
-            return Err(Error::other("No valid certificates found in directory"));
-        }
-
-        Self::from_cert_key_pairs(cert_key_pairs)
-    }
-
-    fn from_cert_key_pairs(
-        cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
-    ) -> io::Result<Self> {
-        let cert_count = cert_key_pairs.len();
-        let mut cert_resolver = ResolvesServerCertUsingSni::new();
-        let mut default_cert = None;
-        let mut entries = cert_key_pairs.into_iter().collect::<Vec<_>>();
-        entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain));
-        let fingerprint = fingerprint_tls_entries(&entries);
-
-        for (domain, (certs, key)) in entries {
-            let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
-                .map_err(|e| Error::other(format!("unsupported private key type for {domain}: {e:?}")))?;
-            let certified_key = CertifiedKey::new(certs, signing_key);
-
-            if domain.as_str() == "default" {
-                default_cert = Some(Arc::new(certified_key.clone()));
-            } else {
-                cert_resolver
-                    .add(&domain, certified_key)
-                    .map_err(|e| Error::other(format!("failed to add certificate for {domain}: {e:?}")))?;
-            }
-        }
-
-        Ok(Self {
-            cert_resolver,
-            default_cert,
-            cert_count,
-            fingerprint,
-        })
-    }
-}
-
-fn fingerprint_tls_entries(entries: &[(String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>))]) -> u64 {
-    let mut hasher = DefaultHasher::new();
-
-    for (domain, (certs, key)) in entries {
-        hasher.write_usize(domain.len());
-        hasher.write(domain.as_bytes());
-        hasher.write_usize(certs.len());
-        for cert in certs {
-            hasher.write_usize(cert.as_ref().len());
-            hasher.write(cert.as_ref());
-        }
-        hasher.write_usize(key.secret_der().len());
-        hasher.write(key.secret_der());
-    }
-
-    hasher.finish()
-}
-
-#[derive(Debug)]
-pub(crate) struct ReloadableCertResolver {
-    current: RwLock<ResolverState>,
-}
-
-impl ReloadableCertResolver {
-    pub(crate) fn load_from_directory(cert_dir: &str) -> io::Result<Arc<Self>> {
-        let state = ResolverState::load_from_directory(cert_dir)?;
-        Ok(Arc::new(Self {
-            current: RwLock::new(state),
-        }))
-    }
-
-    pub(crate) fn reload_from_directory(&self, cert_dir: &str) -> io::Result<Option<usize>> {
-        let new_state = ResolverState::load_from_directory(cert_dir)?;
-
-        match self.current.write() {
-            Ok(mut guard) => {
-                if guard.fingerprint == new_state.fingerprint {
-                    return Ok(None);
-                }
-                let cert_count = new_state.cert_count;
-                *guard = new_state;
-                Ok(Some(cert_count))
-            }
-            Err(poisoned) => {
-                let mut guard = poisoned.into_inner();
-                if guard.fingerprint == new_state.fingerprint {
-                    return Ok(None);
-                }
-                let cert_count = new_state.cert_count;
-                *guard = new_state;
-                Ok(Some(cert_count))
-            }
-        }
-    }
-}
-
-impl ResolvesServerCert for ReloadableCertResolver {
-    fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
-        let guard = match self.current.read() {
-            Ok(guard) => guard,
-            Err(poisoned) => poisoned.into_inner(),
-        };
-
-        guard
-            .cert_resolver
-            .resolve(client_hello)
-            .or_else(|| guard.default_cert.clone())
-    }
-}
-
-pub(crate) fn spawn_cert_reload_loop(
-    protocol: &'static str,
-    cert_dir: String,
-    resolver: Arc<ReloadableCertResolver>,
-    mut shutdown_rx: watch::Receiver<bool>,
-) -> Option<JoinHandle<()>> {
-    let enabled = rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE);
-    if !enabled {
-        debug!(
-            protocol,
-            "TLS certificate hot reload is disabled (set {}=1 to enable)", ENV_TLS_RELOAD_ENABLE
-        );
-        return None;
-    }
-
-    let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5);
-    info!(
-        protocol,
-        cert_dir = %cert_dir,
-        "TLS certificate hot reload enabled, checking every {}s",
-        interval_secs
-    );
-
-    Some(tokio::spawn(async move {
-        let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
-        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
-        interval.tick().await;
-
-        loop {
-            tokio::select! {
-                changed = shutdown_rx.changed() => {
-                    match changed {
-                        Ok(()) => {
-                            if *shutdown_rx.borrow() {
-                                info!(protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped");
-                                break;
-                            }
-                            continue;
-                        }
-                        Err(_) => {
-                            info!(
-                                protocol,
-                                cert_dir = %cert_dir,
-                                "TLS certificate hot reload task stopped because the shutdown channel closed"
-                            );
-                            break;
-                        }
-                    }
-                }
-                _ = interval.tick() => {}
-            }
-
-            match resolver.reload_from_directory(&cert_dir) {
-                Ok(Some(cert_count)) => {
-                    info!(
-                        protocol,
-                        cert_dir = %cert_dir,
-                        cert_count,
-                        "TLS certificates reloaded successfully"
-                    );
-                }
-                Ok(None) => {
-                    debug!(protocol, cert_dir = %cert_dir, "TLS certificate material unchanged; skipping reload");
-                }
-                Err(e) => {
-                    warn!(
-                        protocol,
-                        cert_dir = %cert_dir,
-                        "TLS certificate reload failed (will retry): {}",
-                        e
-                    );
-                }
-            }
-        }
-    }))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rcgen::generate_simple_self_signed;
-    use std::fs;
-    use tempfile::TempDir;
-
-    fn cert_key_pair(san: &str) -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
-        let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap();
-        (
-            vec![cert.cert.der().clone()],
-            PrivateKeyDer::try_from(cert.signing_key.serialize_der()).unwrap(),
-        )
-    }
-
-    fn clone_cert_key_pair(
-        cert_key_pair: &(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>),
-    ) -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
-        (cert_key_pair.0.clone(), cert_key_pair.1.clone_key())
-    }
-
-    fn write_default_cert(dir: &std::path::Path, san: &str) {
-        let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap();
-        fs::write(dir.join(RUSTFS_TLS_CERT), cert.cert.pem()).unwrap();
-        fs::write(dir.join(RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).unwrap();
-    }
-
-    #[test]
-    fn reload_from_directory_replaces_default_certificate() {
-        let temp_dir = TempDir::new().unwrap();
-        write_default_cert(temp_dir.path(), "localhost");
-
-        let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
-        let before = {
-            let guard = resolver.current.read().unwrap();
-            guard.default_cert.as_ref().unwrap().clone()
-        };
-
-        write_default_cert(temp_dir.path(), "rotated.local");
-
-        let cert_count = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
-        assert_eq!(cert_count, Some(1));
-
-        let after = {
-            let guard = resolver.current.read().unwrap();
-            guard.default_cert.as_ref().unwrap().clone()
-        };
-
-        assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref());
-    }
-
-    #[test]
-    fn reload_from_directory_skips_when_material_is_unchanged() {
-        let temp_dir = TempDir::new().unwrap();
-        write_default_cert(temp_dir.path(), "localhost");
-
-        let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
-        let outcome = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
-        assert_eq!(outcome, None);
-    }
-
-    #[test]
-    fn resolver_state_fingerprint_is_stable_across_domain_ordering() {
-        let default_cert = cert_key_pair("localhost");
-        let api_cert = cert_key_pair("api.example.com");
-        let web_cert = cert_key_pair("web.example.com");
-
-        let mut first = HashMap::new();
-        first.insert("default".to_string(), clone_cert_key_pair(&default_cert));
-        first.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert));
-        first.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert));
-
-        let mut second = HashMap::new();
-        second.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert));
-        second.insert("default".to_string(), clone_cert_key_pair(&default_cert));
-        second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert));
-
-        let first_state = ResolverState::from_cert_key_pairs(first).unwrap();
-        let second_state = ResolverState::from_cert_key_pairs(second).unwrap();
-
-        assert_eq!(first_state.cert_count, 3);
-        assert_eq!(second_state.cert_count, 3);
-        assert_eq!(first_state.fingerprint, second_state.fingerprint);
-    }
-}

crates/protocols/src/webdav/server.rs

@@ -16,7 +16,6 @@ use super::config::{WebDavConfig, WebDavInitError};
 use super::driver::WebDavDriver;
 use crate::common::client::s3::StorageBackend;
 use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
-use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
 use bytes::Bytes;
 use dav_server::DavHandler;
 use dav_server::fakels::FakeLs;
@@ -25,10 +24,13 @@ use hyper::server::conn::http1;
 use hyper::service::service_fn;
 use hyper::{Request, Response, StatusCode};
 use hyper_util::rt::TokioIo;
+use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
+use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
 use rustls::ServerConfig;
 use std::convert::Infallible;
 use std::net::IpAddr;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::net::TcpListener;
 use tokio::sync::{broadcast, watch};
 use tokio_rustls::TlsAcceptor;
@@ -49,6 +51,14 @@ impl<S> WebDavServer<S>
 where
     S: StorageBackend + Clone + Send + Sync + 'static + std::fmt::Debug,
 {
+    fn tls_reload_options() -> TlsReloadOptions {
+        TlsReloadOptions {
+            enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
+            interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
+            ..TlsReloadOptions::default()
+        }
+    }
+
     /// Create a new WebDAV server
     pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
         config.validate().await?;
@@ -68,10 +78,14 @@ where
             if let Some(cert_dir) = &self.config.cert_dir {
                 debug!("Enabling WebDAV TLS with certificates from: {}", cert_dir);
 
-                let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
+                let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
                     .map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?;
-                let _reload_task =
-                    spawn_cert_reload_loop("webdav", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
+                let _reload_task = spawn_server_cert_reload_loop(
+                    "webdav",
+                    resolver.clone(),
+                    Self::tls_reload_options(),
+                    reload_shutdown_rx.clone(),
+                );
 
                 let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();