Restore globals and add unified TLS/mTLS loading from RUSTFS_TLS_PATH (#1309)

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

rustfs/Cargo.toml

@@ -65,7 +65,6 @@ rustfs-zip = { workspace = true }
 # Async Runtime and Networking
 async-trait = { workspace = true }
 axum.workspace = true
-axum-extra = { workspace = true }
 axum-server = { workspace = true }
 futures.workspace = true
 futures-util.workspace = true
@@ -95,6 +94,7 @@ serde_urlencoded = { workspace = true }
 # Cryptography and Security
 rustls = { workspace = true }
 subtle = { workspace = true }
+rustls-pemfile = { workspace = true }
 
 # Time and Date
 chrono = { workspace = true }

rustfs/src/main.rs

@@ -95,7 +95,9 @@ async fn async_main() -> Result<()> {
 
     // Store in global storage
     match set_global_guard(guard).map_err(Error::other) {
-        Ok(_) => (),
+        Ok(_) => {
+            info!(target: "rustfs::main", "Global observability guard set successfully.");
+        }
         Err(e) => {
             error!("Failed to set global observability guard: {}", e);
             return Err(e);
@@ -110,7 +112,15 @@ async fn async_main() -> Result<()> {
 
     // Initialize TLS if a certificate path is provided
     if let Some(tls_path) = &opt.tls_path {
-        init_cert(tls_path).await
+        match init_cert(tls_path).await {
+            Ok(_) => {
+                info!(target: "rustfs::main", "TLS initialized successfully with certs from {}", tls_path);
+            }
+            Err(e) => {
+                error!("Failed to initialize TLS from {}: {}", tls_path, e);
+                return Err(Error::other(e));
+            }
+        }
     }
 
     // Run parameters

rustfs/src/server/cert.rs

@@ -12,34 +12,129 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_common::set_global_root_cert;
+use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert};
 use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_PUBLIC_CERT, RUSTFS_TLS_CERT};
+use rustls::pki_types::{CertificateDer, PrivateKeyDer};
+use std::path::{Path, PathBuf};
 use tracing::{debug, info};
 
-/// Initialize TLS certificates for inter-node communication.
-/// This function attempts to load certificates from the specified `tls_path`.
-/// It looks for `rustfs_cert.pem`, `public.crt`, and `ca.crt` files.
-/// Additionally, it tries to load system root certificates from common locations
-/// to ensure trust for public CAs when mixing self-signed and public certificates.
-/// If any certificates are found, they are set as the global root certificates.
-pub(crate) async fn init_cert(tls_path: &str) {
+#[derive(Debug)]
+pub enum RustFSError {
+    Cert(String),
+}
+
+impl std::fmt::Display for RustFSError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RustFSError::Cert(msg) => write!(f, "Certificate error: {}", msg),
+        }
+    }
+}
+
+impl std::error::Error for RustFSError {}
+
+/// Parse PEM-encoded certificates into DER format.
+/// Returns a vector of DER-encoded certificates.
+///
+/// # Arguments
+/// * `pem` - A byte slice containing the PEM-encoded certificates.
+///
+/// # Returns
+/// A vector of `CertificateDer` containing the DER-encoded certificates.
+///
+/// # Errors
+/// Returns `RustFSError` if parsing fails.
+fn parse_pem_certs(pem: &[u8]) -> Result<Vec<CertificateDer<'static>>, RustFSError> {
+    let mut out = Vec::new();
+    let mut reader = std::io::Cursor::new(pem);
+    for item in rustls_pemfile::certs(&mut reader) {
+        let c = item.map_err(|e| RustFSError::Cert(format!("parse cert pem: {e}")))?;
+        out.push(c);
+    }
+    Ok(out)
+}
+
+/// Parse a PEM-encoded private key into DER format.
+/// Supports PKCS#8 and RSA private keys.
+///
+/// # Arguments
+/// * `pem` - A byte slice containing the PEM-encoded private key.
+///
+/// # Returns
+/// A `PrivateKeyDer` containing the DER-encoded private key.
+///
+/// # Errors
+/// Returns `RustFSError` if parsing fails or no key is found.
+fn parse_pem_private_key(pem: &[u8]) -> Result<PrivateKeyDer<'static>, RustFSError> {
+    let mut reader = std::io::Cursor::new(pem);
+    let key = rustls_pemfile::private_key(&mut reader).map_err(|e| RustFSError::Cert(format!("parse private key pem: {e}")))?;
+    key.ok_or_else(|| RustFSError::Cert("no private key found in PEM".into()))
+}
+
+/// Helper function to read a file and return its contents.
+/// Returns the file contents as a vector of bytes.
+/// # Errors
+/// Returns `RustFSError` if reading fails.
+async fn read_file(path: &PathBuf, desc: &str) -> Result<Vec<u8>, RustFSError> {
+    tokio::fs::read(path)
+        .await
+        .map_err(|e| RustFSError::Cert(format!("read {} {:?}: {e}", desc, path)))
+}
+
+/// Initialize TLS material for both server and outbound client connections.
+///
+/// Loads roots from:
+/// - `${RUSTFS_TLS_PATH}/ca.crt` (or `tls/ca.crt`)
+/// - `${RUSTFS_TLS_PATH}/public.crt` (optional additional root bundle)
+/// - system roots if `RUSTFS_TRUST_SYSTEM_CA=true` (default: false)
+/// - if `RUSTFS_TRUST_LEAF_CERT_AS_CA=true`, also loads leaf cert(s) from
+///   `${RUSTFS_TLS_PATH}/rustfs_cert.pem` into the root store.
+///
+/// Loads mTLS client identity (optional) from:
+/// - `${RUSTFS_TLS_PATH}/client_cert.pem`
+/// - `${RUSTFS_TLS_PATH}/client_key.pem`
+///
+/// Environment overrides:
+/// - RUSTFS_TLS_PATH
+/// - RUSTFS_MTLS_CLIENT_CERT
+/// - RUSTFS_MTLS_CLIENT_KEY
+pub(crate) async fn init_cert(tls_path: &str) -> Result<(), RustFSError> {
+    if tls_path.is_empty() {
+        info!("No TLS path configured; skipping certificate initialization");
+        return Ok(());
+    }
+    let tls_dir = PathBuf::from(tls_path);
+
+    // Load root certificates
+    load_root_certs(&tls_dir).await?;
+
+    // Load optional mTLS identity
+    load_mtls_identity(&tls_dir).await?;
+
+    Ok(())
+}
+
+/// Load root certificates from various sources.
+async fn load_root_certs(tls_dir: &Path) -> Result<(), RustFSError> {
     let mut cert_data = Vec::new();
 
-    // Try rustfs_cert.pem (custom cert name)
-    walk_dir(std::path::PathBuf::from(tls_path), RUSTFS_TLS_CERT, &mut cert_data).await;
+    let trust_leaf_as_ca =
+        rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_LEAF_CERT_AS_CA, rustfs_config::DEFAULT_TRUST_LEAF_CERT_AS_CA);
+    if trust_leaf_as_ca {
+        walk_dir(tls_dir.to_path_buf(), RUSTFS_TLS_CERT, &mut cert_data).await;
+        info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA");
+    }
 
-    // Try public.crt (common CA name)
-    let public_cert_path = std::path::Path::new(tls_path).join(RUSTFS_PUBLIC_CERT);
+    // Try public.crt and ca.crt
+    let public_cert_path = tls_dir.join(RUSTFS_PUBLIC_CERT);
     load_cert_file(public_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
 
-    // Try ca.crt (common CA name)
-    let ca_cert_path = std::path::Path::new(tls_path).join(RUSTFS_CA_CERT);
+    let ca_cert_path = tls_dir.join(RUSTFS_CA_CERT);
     load_cert_file(ca_cert_path.to_str().unwrap_or_default(), &mut cert_data, "CA certificate").await;
 
+    // Load system root certificates if enabled
     let trust_system_ca = rustfs_utils::get_env_bool(rustfs_config::ENV_TRUST_SYSTEM_CA, rustfs_config::DEFAULT_TRUST_SYSTEM_CA);
-    if !trust_system_ca {
-        // Attempt to load system root certificates to maintain trust for public CAs
-        // This is important when mixing self-signed internal certs with public external certs
+    if trust_system_ca {
         let system_ca_paths = [
             "/etc/ssl/certs/ca-certificates.crt",                  // Debian/Ubuntu/Alpine
             "/etc/pki/tls/certs/ca-bundle.crt",                    // Fedora/RHEL/CentOS
@@ -57,7 +152,7 @@ pub(crate) async fn init_cert(tls_path: &str) {
             if load_cert_file(path, &mut cert_data, "system root certificates").await {
                 system_cert_loaded = true;
                 info!("Loaded system root certificates from {}", path);
-                break; // Stop after finding the first valid bundle
+                break;
             }
         }
 
@@ -67,10 +162,51 @@ pub(crate) async fn init_cert(tls_path: &str) {
     } else {
         info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA");
     }
+
     if !cert_data.is_empty() {
         set_global_root_cert(cert_data).await;
         info!("Configured custom root certificates for inter-node communication");
     }
+
+    Ok(())
+}
+
+/// Load optional mTLS identity.
+async fn load_mtls_identity(tls_dir: &Path) -> Result<(), RustFSError> {
+    let client_cert_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_CERT) {
+        Some(p) => PathBuf::from(p),
+        None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME),
+    };
+
+    let client_key_path = match rustfs_utils::get_env_opt_str(rustfs_config::ENV_MTLS_CLIENT_KEY) {
+        Some(p) => PathBuf::from(p),
+        None => tls_dir.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME),
+    };
+
+    if client_cert_path.exists() && client_key_path.exists() {
+        let cert_bytes = read_file(&client_cert_path, "client cert").await?;
+        let key_bytes = read_file(&client_key_path, "client key").await?;
+
+        // Validate parse-ability early; store as PEM bytes for tonic.
+        parse_pem_certs(&cert_bytes)?;
+        parse_pem_private_key(&key_bytes)?;
+
+        let identity_pem = MtlsIdentityPem {
+            cert_pem: cert_bytes,
+            key_pem: key_bytes,
+        };
+
+        set_global_mtls_identity(Some(identity_pem)).await;
+        info!("Loaded mTLS client identity cert={:?} key={:?}", client_cert_path, client_key_path);
+    } else {
+        set_global_mtls_identity(None).await;
+        info!(
+            "mTLS client identity not configured (missing {:?} and/or {:?}); proceeding with server-only TLS",
+            client_cert_path, client_key_path
+        );
+    }
+
+    Ok(())
 }
 
 /// Helper function to load a certificate file and append to cert_data.
@@ -114,7 +250,7 @@ async fn load_if_matches(entry: &tokio::fs::DirEntry, cert_name: &str, cert_data
 /// - `path`: The starting directory path to search for certificates.
 /// - `cert_name`: The name of the certificate file to look for.
 /// - `cert_data`: A mutable vector to append loaded certificate data.
-async fn walk_dir(path: std::path::PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
+async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
     if let Ok(mut rd) = tokio::fs::read_dir(&path).await {
         while let Ok(Some(entry)) = rd.next_entry().await {
             if let Ok(ft) = entry.file_type().await {

rustfs/src/server/http.rs

@@ -431,11 +431,11 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
         debug!("TLS path is not provided or does not exist, starting with HTTP");
         return Ok(None);
     }
-
     debug!("Found TLS directory, checking for certificates");
 
     // Make sure to use a modern encryption suite
     let _ = rustls::crypto::ring::default_provider().install_default();
+    let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?;
 
     // 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
     if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) {
@@ -446,9 +446,15 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
             let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?;
 
             // Configure the server to enable SNI support
-            let mut server_config = ServerConfig::builder()
-                .with_no_client_auth()
-                .with_cert_resolver(Arc::new(resolver));
+            let mut server_config = if let Some(verifier) = mtls_verifier.clone() {
+                ServerConfig::builder()
+                    .with_client_cert_verifier(verifier)
+                    .with_cert_resolver(Arc::new(resolver))
+            } else {
+                ServerConfig::builder()
+                    .with_no_client_auth()
+                    .with_cert_resolver(Arc::new(resolver))
+            };
 
             // Configure ALPN protocol priority
             server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
@@ -470,10 +476,17 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
         let certs = rustfs_utils::load_certs(&cert_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
         let key = rustfs_utils::load_private_key(&key_path).map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
 
-        let mut server_config = ServerConfig::builder()
-            .with_no_client_auth()
-            .with_single_cert(certs, key)
-            .map_err(|e| rustfs_utils::certs_error(e.to_string()))?;
+        let mut server_config = if let Some(verifier) = mtls_verifier {
+            ServerConfig::builder()
+                .with_client_cert_verifier(verifier)
+                .with_single_cert(certs, key)
+                .map_err(|e| rustfs_utils::certs_error(e.to_string()))?
+        } else {
+            ServerConfig::builder()
+                .with_no_client_auth()
+                .with_single_cert(certs, key)
+                .map_err(|e| rustfs_utils::certs_error(e.to_string()))?
+        };
 
         // Configure ALPN protocol priority
         server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];