Restore globals and add unified TLS/mTLS loading from RUSTFS_TLS_PATH (#1309)

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/common/src/globals.rs

@@ -25,18 +25,42 @@ pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLoc
 pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
 pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
 pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
+pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
 
+/// Set the global RustFS address used for gRPC connections.
+///
+/// # Arguments
+/// * `addr` - A string slice representing the RustFS address (e.g., "https://node1:9000").
 pub async fn set_global_addr(addr: &str) {
     *GLOBAL_RUSTFS_ADDR.write().await = addr.to_string();
 }
 
+/// Set the global root CA certificate for outbound gRPC clients.
+/// This certificate is used to validate server TLS certificates.
+/// When set to None, clients use the system default root CAs.
+///
+/// # Arguments
+/// * `cert` - A vector of bytes representing the PEM-encoded root CA certificate.
 pub async fn set_global_root_cert(cert: Vec<u8>) {
     *GLOBAL_ROOT_CERT.write().await = Some(cert);
 }
 
+/// Set the global mTLS identity (cert+key PEM) for outbound gRPC clients.
+/// When set, clients will present this identity to servers requesting/requiring mTLS.
+/// When None, clients proceed with standard server-authenticated TLS.
+///
+/// # Arguments
+/// * `identity` - An optional MtlsIdentityPem struct containing the cert and key PEM.
+pub async fn set_global_mtls_identity(identity: Option<MtlsIdentityPem>) {
+    *GLOBAL_MTLS_IDENTITY.write().await = identity;
+}
+
 /// Evict a stale/dead connection from the global connection cache.
 /// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off).
 /// By removing the cached connection, subsequent requests will establish a fresh connection.
+///
+/// # Arguments
+/// * `addr` - The address of the connection to evict.
 pub async fn evict_connection(addr: &str) {
     let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
     if removed.is_some() {
@@ -45,6 +69,12 @@ pub async fn evict_connection(addr: &str) {
 }
 
 /// Check if a connection exists in the cache for the given address.
+///
+/// # Arguments
+/// * `addr` - The address to check.
+///
+/// # Returns
+/// * `bool` - True if a cached connection exists, false otherwise.
 pub async fn has_cached_connection(addr: &str) -> bool {
     GLOBAL_CONN_MAP.read().await.contains_key(addr)
 }
@@ -58,3 +88,12 @@ pub async fn clear_all_connections() {
         tracing::warn!("Cleared {} cached connections from global map", count);
     }
 }
+/// Optional client identity (cert+key PEM) for outbound mTLS.
+///
+/// When present, gRPC clients will present this identity to servers requesting/requiring mTLS.
+/// When absent, clients proceed with standard server-authenticated TLS.
+#[derive(Clone, Debug)]
+pub struct MtlsIdentityPem {
+    pub cert_pem: Vec<u8>,
+    pub key_pem: Vec<u8>,
+}

crates/config/src/constants/tls.rs

@@ -35,3 +35,52 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
 /// By default, RustFS does not trust system CA certificates.
 /// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
 pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;
+
+/// Environment variable to trust leaf certificates as CA
+/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation.
+/// By default, this is disabled.
+/// To enable, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1
+pub const ENV_TRUST_LEAF_CERT_AS_CA: &str = "RUSTFS_TRUST_LEAF_CERT_AS_CA";
+
+/// Default value for trusting leaf certificates as CA
+/// By default, RustFS does not trust leaf certificates as CA.
+/// To change this behavior, set the environment variable RUSTFS_TRUST_LEAF_CERT_AS_CA=1
+pub const DEFAULT_TRUST_LEAF_CERT_AS_CA: bool = false;
+
+/// Default filename for client CA certificate
+/// client_ca.crt (CA bundle for verifying client certificates in server mTLS)
+pub const RUSTFS_CLIENT_CA_CERT_FILENAME: &str = "client_ca.crt";
+
+/// Environment variable for client certificate file path
+/// RUSTFS_MTLS_CLIENT_CERT
+/// Specifies the file path to the client certificate used for mTLS authentication.
+/// If not set, RustFS will look for the default filename "client_cert.pem" in the current directory.
+/// To set, use the environment variable RUSTFS_MTLS_CLIENT_CERT=/path/to/client_cert.pem
+pub const ENV_MTLS_CLIENT_CERT: &str = "RUSTFS_MTLS_CLIENT_CERT";
+
+/// Default filename for client certificate
+/// client_cert.pem
+pub const RUSTFS_CLIENT_CERT_FILENAME: &str = "client_cert.pem";
+
+/// Environment variable for client private key file path
+/// RUSTFS_MTLS_CLIENT_KEY
+/// Specifies the file path to the client private key used for mTLS authentication.
+/// If not set, RustFS will look for the default filename "client_key.pem" in the current directory.
+/// To set, use the environment variable RUSTFS_MTLS_CLIENT_KEY=/path/to/client_key.pem
+pub const ENV_MTLS_CLIENT_KEY: &str = "RUSTFS_MTLS_CLIENT_KEY";
+
+/// Default filename for client private key
+/// client_key.pem
+pub const RUSTFS_CLIENT_KEY_FILENAME: &str = "client_key.pem";
+
+/// RUSTFS_SERVER_MTLS_ENABLE
+/// Environment variable to enable server mTLS
+/// When set to "1", RustFS server will require client certificates for authentication.
+/// By default, this is disabled.
+/// To enable, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
+pub const ENV_SERVER_MTLS_ENABLE: &str = "RUSTFS_SERVER_MTLS_ENABLE";
+
+/// Default value for enabling server mTLS
+/// By default, RustFS server mTLS is disabled.
+/// To change this behavior, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
+pub const DEFAULT_SERVER_MTLS_ENABLE: bool = false;

crates/ecstore/src/client/transition_api.rs

@@ -132,6 +132,25 @@ pub enum BucketLookupType {
     BucketLookupPath,
 }
 
+fn load_root_store_from_tls_path() -> Option<rustls::RootCertStore> {
+    // Load the root certificate bundle from the path specified by the
+    // RUSTFS_TLS_PATH environment variable.
+    let tp = std::env::var("RUSTFS_TLS_PATH").ok()?;
+    let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT);
+    if !ca.exists() {
+        return None;
+    }
+
+    let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?;
+    let mut store = rustls::RootCertStore::empty();
+    for der in der_list {
+        if let Err(e) = store.add(der.into()) {
+            warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display());
+        }
+    }
+    Some(store)
+}
+
 impl TransitionClient {
     pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
         let clnt = Self::private_new(endpoint, opts, tier_type).await?;
@@ -142,18 +161,22 @@ impl TransitionClient {
     async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
 
-        //#[cfg(feature = "ring")]
         let _ = rustls::crypto::ring::default_provider().install_default();
-        //#[cfg(feature = "aws-lc-rs")]
-        // let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
         let scheme = endpoint_url.scheme();
         let client;
-        let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth();
+        let tls = if let Some(store) = load_root_store_from_tls_path() {
+            rustls::ClientConfig::builder()
+                .with_root_certificates(store)
+                .with_no_client_auth()
+        } else {
+            rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth()
+        };
+
         let https = hyper_rustls::HttpsConnectorBuilder::new()
             .with_tls_config(tls)
             .https_or_http()
             .enable_http1()
+            .enable_http2()
             .build();
         client = Client::builder(TokioExecutor::new()).build(https);
 

crates/protos/src/lib.rs

@@ -16,7 +16,7 @@
 mod generated;
 
 use proto_gen::node_service::node_service_client::NodeServiceClient;
-use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_ROOT_CERT, evict_connection};
+use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection};
 use std::{error::Error, time::Duration};
 use tonic::{
     Request, Status,
@@ -83,6 +83,11 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
 
     let root_cert = GLOBAL_ROOT_CERT.read().await;
     if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
+        if root_cert.is_none() {
+            debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
+            // If no custom root cert is configured, try to use system roots.
+            connector = connector.tls_config(ClientTlsConfig::new())?;
+        }
         if let Some(cert_pem) = root_cert.as_ref() {
             let ca = Certificate::from_pem(cert_pem);
             // Derive the hostname from the HTTPS URL for TLS hostname verification.
@@ -95,7 +100,13 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
                 .next()
                 .unwrap_or("");
             let tls = if !domain.is_empty() {
-                ClientTlsConfig::new().ca_certificate(ca).domain_name(domain)
+                let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain);
+                let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await;
+                if let Some(id) = mtls_identity.as_ref() {
+                    let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
+                    cfg = cfg.identity(identity);
+                }
+                cfg
             } else {
                 // Fallback: configure TLS without explicit domain if parsing fails.
                 ClientTlsConfig::new().ca_certificate(ca)
@@ -103,12 +114,9 @@ async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
             connector = connector.tls_config(tls)?;
             debug!("Configured TLS with custom root certificate for: {}", addr);
         } else {
-            debug!("Using system root certificates for TLS: {}", addr);
-        }
-    } else {
-        // Custom root certificates are configured but will be ignored for non-HTTPS addresses.
-        if root_cert.is_some() {
-            warn!("Custom root certificates are configured but not used because the address does not use HTTPS: {addr}");
+            return Err(std::io::Error::other(
+                "HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)."
+            ).into());
         }
     }
 

crates/rio/Cargo.toml

@@ -41,7 +41,8 @@ reqwest.workspace = true
 tokio-util.workspace = true
 faster-hex.workspace = true
 futures.workspace = true
-rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
+rustfs-config = { workspace = true, features = ["constants"] }
+rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] }
 serde_json.workspace = true
 md-5 = { workspace = true }
 tracing.workspace = true

crates/rio/src/http_reader.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
 use bytes::Bytes;
 use futures::{Stream, TryStreamExt as _};
 use http::HeaderMap;
 use pin_project_lite::pin_project;
-use reqwest::{Client, Method, RequestBuilder};
+use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
 use std::error::Error as _;
 use std::io::{self, Error};
 use std::ops::Not as _;
@@ -26,21 +27,88 @@ use std::task::{Context, Poll};
 use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 use tokio::sync::mpsc;
 use tokio_util::io::StreamReader;
+use tracing::error;
 
-use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
+/// Get the TLS path from the RUSTFS_TLS_PATH environment variable.
+/// If the variable is not set, return None.
+fn tls_path() -> Option<&'static std::path::PathBuf> {
+    static TLS_PATH: LazyLock<Option<std::path::PathBuf>> = LazyLock::new(|| {
+        std::env::var("RUSTFS_TLS_PATH")
+            .ok()
+            .and_then(|s| if s.is_empty() { None } else { Some(s.into()) })
+    });
+    TLS_PATH.as_ref()
+}
+
+/// Load CA root certificates from the RUSTFS_TLS_PATH directory.
+/// The CA certificates should be in PEM format and stored in the file
+/// specified by the RUSTFS_CA_CERT constant.
+/// If the file does not exist or cannot be read, return the builder unchanged.
+fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
+    let Some(tp) = tls_path() else {
+        return builder;
+    };
+    let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
+    if !ca_path.exists() {
+        return builder;
+    }
+
+    let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else {
+        return builder;
+    };
+
+    let mut b = builder;
+    for der in certs_der {
+        if let Ok(cert) = Certificate::from_der(&der) {
+            b = b.add_root_certificate(cert);
+        }
+    }
+    b
+}
+
+/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory.
+/// The client certificate and private key should be in PEM format and stored in the files
+/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants.
+/// If the files do not exist or cannot be read, return None.
+fn load_optional_mtls_identity_from_tls_path() -> Option<Identity> {
+    let tp = tls_path()?;
+    let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?;
+    let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?;
+
+    let mut pem = Vec::with_capacity(cert.len() + key.len() + 1);
+    pem.extend_from_slice(&cert);
+    if !pem.ends_with(b"\n") {
+        pem.push(b'\n');
+    }
+    pem.extend_from_slice(&key);
+
+    match Identity::from_pem(&pem) {
+        Ok(id) => Some(id),
+        Err(e) => {
+            error!("Failed to load mTLS identity from PEM: {e}");
+            None
+        }
+    }
+}
 
 fn get_http_client() -> Client {
     // Reuse the HTTP connection pool in the global `reqwest::Client` instance
     // TODO: interact with load balancing?
     static CLIENT: LazyLock<Client> = LazyLock::new(|| {
-        Client::builder()
+        let mut builder = Client::builder()
             .connect_timeout(std::time::Duration::from_secs(5))
             .tcp_keepalive(std::time::Duration::from_secs(10))
             .http2_keep_alive_interval(std::time::Duration::from_secs(5))
             .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
-            .http2_keep_alive_while_idle(true)
-            .build()
-            .expect("Failed to create global HTTP client")
+            .http2_keep_alive_while_idle(true);
+
+        // HTTPS root trust + optional mTLS identity from RUSTFS_TLS_PATH
+        builder = load_ca_roots_from_tls_path(builder);
+        if let Some(id) = load_optional_mtls_identity_from_tls_path() {
+            builder = builder.identity(id);
+        }
+
+        builder.build().expect("Failed to create global HTTP client")
     });
     CLIENT.clone()
 }

crates/utils/src/certs.rs

@@ -12,8 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::get_env_bool;
 use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
-use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
+use rustls::RootCertStore;
+use rustls::server::danger::ClientCertVerifier;
+use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni, WebPkiClientVerifier};
 use rustls::sign::CertifiedKey;
 use rustls_pemfile::{certs, private_key};
 use rustls_pki_types::{CertificateDer, PrivateKeyDer};
@@ -48,6 +51,79 @@ pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
     Ok(certs)
 }
 
+/// Load a PEM certificate bundle and return each certificate as DER bytes.
+///
+/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that
+/// need to add root certificates one-by-one.
+///
+/// - Input: a PEM file that may contain multiple cert blocks.
+/// - Output: Vec of DER-encoded cert bytes, one per cert.
+///
+/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates
+/// to depend on rustls types.
+pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
+    let pem = fs::read(path)?;
+    let mut reader = io::BufReader::new(&pem[..]);
+
+    let certs = certs(&mut reader)
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| certs_error(format!("Failed to parse PEM certs from {path}: {e}")))?;
+
+    Ok(certs.into_iter().map(|c| c.to_vec()).collect())
+}
+
+/// Builds a WebPkiClientVerifier for mTLS if enabled via environment variable.
+///
+/// # Arguments
+/// * `tls_path` - Directory containing client CA certificates
+///
+/// # Returns
+/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found
+/// * `Ok(None)` if mTLS is disabled
+/// * `Err` if mTLS is enabled but configuration is invalid
+pub fn build_webpki_client_verifier(tls_path: &str) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
+    if !get_env_bool(rustfs_config::ENV_SERVER_MTLS_ENABLE, rustfs_config::DEFAULT_SERVER_MTLS_ENABLE) {
+        return Ok(None);
+    }
+
+    let ca_path = mtls_ca_bundle_path(tls_path).ok_or_else(|| {
+        Error::other(format!(
+            "RUSTFS_SERVER_MTLS_ENABLE=true but missing {}/client_ca.crt (or fallback {}/ca.crt)",
+            tls_path, tls_path
+        ))
+    })?;
+
+    let der_list = load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default())?;
+
+    let mut store = RootCertStore::empty();
+    for der in der_list {
+        store
+            .add(der.into())
+            .map_err(|e| Error::other(format!("Invalid client CA cert: {e}")))?;
+    }
+
+    let verifier = WebPkiClientVerifier::builder(Arc::new(store))
+        .build()
+        .map_err(|e| Error::other(format!("Build client cert verifier failed: {e}")))?;
+
+    Ok(Some(verifier))
+}
+
+/// Locate the mTLS client CA bundle in the specified TLS path
+fn mtls_ca_bundle_path(tls_path: &str) -> Option<std::path::PathBuf> {
+    use std::path::Path;
+
+    let p1 = Path::new(tls_path).join(rustfs_config::RUSTFS_CLIENT_CA_CERT_FILENAME);
+    if p1.exists() {
+        return Some(p1);
+    }
+    let p2 = Path::new(tls_path).join(rustfs_config::RUSTFS_CA_CERT);
+    if p2.exists() {
+        return Some(p2);
+    }
+    None
+}
+
 /// Load private key from file.
 /// This function loads a private key from the specified file.
 ///