refactor(tls): centralize FIPS provider policy

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.github/workflows/ci.yml

@@ -536,6 +536,15 @@ jobs:
           if [[ '${{ matrix.mode }}' == 'fips' ]]; then
             args+=(--features fips)
           fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
           tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
           if [[ '${{ matrix.mode }}' == 'fips' ]]; then
             tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/protocols/Cargo.toml

@@ -30,6 +30,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-tls-runtime?/fips"]
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/protocols/src/ftps/server.rs

@@ -110,6 +110,9 @@ where
             ftps_required = self.config.ftps_required,
             "FTPS server starting"
         );
+        if self.config.tls_enabled && self.config.cert_dir.is_some() {
+            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
+        }
         let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
 
         let storage_clone = self.storage.clone();
@@ -191,8 +194,6 @@ where
                 );
 
                 // Build ServerConfig with SNI support
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
                 let server_config = rustls::ServerConfig::builder()
                     .with_no_client_auth()
                     .with_cert_resolver(resolver);

crates/protocols/src/lib.rs

@@ -43,3 +43,70 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
 
 #[cfg(feature = "sftp")]
 pub use sftp::{SftpConfig, SftpInitError, SftpServer};
+
+#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
+mod fips_tests {
+    use crate::common::dummy_storage::DummyBackend;
+    use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
+
+    #[test]
+    fn protocols_reject_incompatible_host_provider() {
+        const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
+        if std::env::var_os(CHILD).is_some() {
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            provider.install_default().expect("install incompatible host provider");
+            let cert_dir = tempfile::tempdir().expect("certificate directory");
+            let cert_dir = cert_dir.path().to_string_lossy().into_owned();
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("test runtime");
+            runtime.block_on(async {
+                let ftps_config = FtpsConfig {
+                    cert_dir: Some(cert_dir.clone()),
+                    ..Default::default()
+                };
+                let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
+                    .await
+                    .expect("FTPS config");
+                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
+                let error = ftps
+                    .start(receiver)
+                    .await
+                    .expect_err("FTPS must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+
+                let webdav_config = WebDavConfig {
+                    bind_addr: ([127, 0, 0, 1], 0).into(),
+                    cert_dir: Some(cert_dir),
+                    ..Default::default()
+                };
+                let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
+                    .await
+                    .expect("WebDAV config");
+                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
+                let error = webdav
+                    .start(receiver)
+                    .await
+                    .expect_err("WebDAV must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            });
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("protocols_reject_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .env("NO_PROXY", "localhost,127.0.0.1,::1")
+            .env("no_proxy", "localhost,127.0.0.1,::1")
+            .output()
+            .expect("start isolated protocol TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+}

crates/protocols/src/webdav/server.rs

@@ -118,6 +118,10 @@ where
             "WebDAV server starting"
         );
 
+        if self.config.tls_enabled && self.config.cert_dir.is_some() {
+            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
+        }
+
         let listener = TcpListener::bind(self.config.bind_addr).await?;
         self.serve(listener, shutdown_rx).await
     }
@@ -155,8 +159,6 @@ where
                     reload_shutdown_rx.clone(),
                 );
 
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
                 let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
 
                 Some(TlsAcceptor::from(Arc::new(server_config)))

crates/s3-client/Cargo.toml

@@ -25,6 +25,10 @@ keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
 categories = ["web-programming", "development-tools", "network-programming"]
 documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
 
+[features]
+default = []
+fips = ["rustfs-tls-runtime/fips"]
+
 [dependencies]
 rustfs-checksums.workspace = true
 rustfs-config.workspace = true

crates/s3-client/src/transition_api.rs

@@ -81,7 +81,7 @@ use time::Duration;
 use time::OffsetDateTime;
 use tokio::io::BufReader;
 use tokio::io::{AsyncRead, AsyncReadExt};
-use tracing::{debug, error, trace, warn};
+use tracing::{error, trace, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
@@ -448,16 +448,7 @@ impl TransitionClient {
         tier_type: &str,
         timeouts: TransitionClientTimeouts,
     ) -> Result<TransitionClient, std::io::Error> {
-        if rustls::crypto::CryptoProvider::get_default().is_none() {
-            // No default provider is set yet; try to install aws-lc-rs.
-            // `install_default` can only fail if another thread races us and installs a provider
-            // between our check and this call, which is still safe to ignore.
-            if rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() {
-                debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install");
-            }
-        } else {
-            debug!("rustls crypto provider already installed, skipping aws-lc-rs install");
-        }
+        rustfs_tls_runtime::ensure_crypto_provider().map_err(std::io::Error::other)?;
 
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
         let timeouts = timeouts.validate()?;
@@ -1862,21 +1853,36 @@ mod tests {
         assert!(outcome.is_ok(), "TLS config creation should not panic");
     }
 
-    /// Installing the rustls crypto provider when one is already set must not panic or return
-    /// an error that surfaces to callers (the race-safe `get_default` check guards the install).
+    #[cfg(feature = "fips")]
     #[test]
-    fn provider_install_is_idempotent() {
-        // Install once (may already be set by another test in this binary — that's fine).
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
-        // A second install attempt on an already-set provider must not panic.
-        let outcome = std::panic::catch_unwind(|| {
-            if rustls::crypto::CryptoProvider::get_default().is_none() {
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-            }
-            // If a default is already present, the branch above is simply skipped.
-        });
-        assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
+    fn fips_transition_rejects_incompatible_host_provider() {
+        const CHILD: &str = "RUSTFS_TEST_TRANSITION_FIPS_PROVIDER";
+        if std::env::var_os(CHILD).is_some() {
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            provider.install_default().expect("install incompatible host provider");
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("test runtime");
+            let error = runtime
+                .block_on(super::TransitionClient::new("127.0.0.1:0", super::Options::default(), ""))
+                .expect_err("transition client must reject the incompatible provider");
+            assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("fips_transition_rejects_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated transition TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
     }
 
     #[test]

crates/targets/Cargo.toml

@@ -13,6 +13,7 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
 
 [features]
 default = []
+fips = ["rustfs-tls-runtime/fips"]
 # Exposes the builder-style `testkit::MockTarget` to downstream test suites. Never enable this
 # from a production `[dependencies]` entry; in-crate unit tests get the module via cfg(test).
 test-support = []

crates/targets/src/check.rs

@@ -162,7 +162,7 @@ pub async fn check_mysql_server_available(args: &crate::target::mysql::MySqlArgs
         .db_name(Some(dsn.database.clone()));
 
     if dsn.tls {
-        ensure_rustls_provider_installed();
+        ensure_rustls_provider_installed()?;
         let mut ssl_opts = SslOpts::default();
         if !args.tls_ca.is_empty() {
             ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);

crates/targets/src/target/amqp.rs

@@ -293,6 +293,10 @@ fn map_lapin_error(err: lapin::Error, context: &str) -> TargetError {
 
 pub async fn connect_amqp(args: &AMQPArgs) -> Result<AMQPConnection, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    if args.url.scheme() == "amqps" {
+        super::ensure_rustls_provider_installed()?;
+    }
     tokio::time::timeout(Duration::from_secs(5), async {
         let url = connection_url(args)?;
         // Reconnect explicitly so every new channel enables publisher confirms below.

crates/targets/src/target/mod.rs

@@ -27,7 +27,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::thread_local;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
-use tracing::{debug, warn};
+use tracing::warn;
 
 pub mod amqp;
 pub mod kafka;
@@ -947,16 +947,12 @@ pub(crate) fn encode_failed_entry(
 
 /// Ensures a rustls crypto provider is installed before any TLS operation.
 ///
-/// Multiple target modules (MySQL, Redis, Postgres, MQTT) need this because
-/// each may be the first to perform a TLS handshake. Idempotent: if a
-/// provider is already registered, returns immediately.
-pub(crate) fn ensure_rustls_provider_installed() {
-    if rustls::crypto::CryptoProvider::get_default().is_some() {
-        return;
-    }
-    if let Err(err) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
-        debug!("rustls provider already installed or unavailable: {err:?}");
-    }
+/// Target modules may be the first to construct a TLS client. In a FIPS build,
+/// an incompatible provider installed by the host is rejected.
+pub(crate) fn ensure_rustls_provider_installed() -> Result<(), TargetError> {
+    rustfs_tls_runtime::ensure_crypto_provider()
+        .map(|_| ())
+        .map_err(|err| TargetError::Configuration(err.to_string()))
 }
 
 #[cfg(test)]

crates/targets/src/target/mqtt.rs

@@ -241,7 +241,7 @@ fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::Roo
 }
 
 fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transport, TargetError> {
-    super::ensure_rustls_provider_installed();
+    super::ensure_rustls_provider_installed()?;
 
     let client_config = match tls
         .policy

crates/targets/src/target/mysql.rs

@@ -881,7 +881,7 @@ async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<(Pool, bool), Ta
         .db_name(Some(dsn.database.clone()));
 
     if dsn.tls {
-        super::ensure_rustls_provider_installed();
+        super::ensure_rustls_provider_installed()?;
         let mut ssl_opts = SslOpts::default();
         if !args.tls_ca.is_empty() {
             let _ =

crates/targets/src/target/nats/mod.rs

@@ -204,6 +204,8 @@ fn nats_sends_credentials_without_tls(args: &NATSArgs) -> bool {
 
 pub async fn connect_nats(args: &NATSArgs) -> Result<async_nats::Client, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    super::ensure_rustls_provider_installed()?;
 
     let mut options = async_nats::ConnectOptions::new().require_tls(args.tls_required);
 

crates/targets/src/target/postgres.rs

@@ -462,7 +462,7 @@ pub fn table_probe_sql(schema: &str, table: &str) -> String {
 /// When `tls_client_cert` and `tls_client_key` are both set the connection
 /// uses mTLS authentication; otherwise no client cert is sent.
 pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, TargetError> {
-    super::ensure_rustls_provider_installed();
+    super::ensure_rustls_provider_installed()?;
 
     let mut root_store = rustls::RootCertStore::empty();
 
@@ -939,6 +939,32 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "fips")]
+    #[test]
+    fn fips_postgres_rejects_incompatible_host_provider() {
+        const CHILD: &str = "RUSTFS_TEST_POSTGRES_FIPS_PROVIDER";
+        if std::env::var_os(CHILD).is_some() {
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            provider.install_default().expect("install incompatible host provider");
+            let error = build_tls_config(&base_args()).expect_err("Postgres must reject the incompatible provider");
+            assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("fips_postgres_rejects_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated Postgres TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+
     #[test]
     fn validate_disabled_skips_all_checks() {
         let args = PostgresArgs {

crates/targets/src/target/pulsar.rs

@@ -90,7 +90,7 @@ impl PulsarArgs {
             return Ok(());
         }
 
-        validate_pulsar_broker(&self.broker)?;
+        let parsed = validate_pulsar_broker(&self.broker)?;
 
         if self.topic.trim().is_empty() {
             return Err(TargetError::Configuration("Pulsar topic cannot be empty".to_string()));
@@ -116,8 +116,6 @@ impl PulsarArgs {
             return Err(TargetError::Configuration("Pulsar queue directory must be an absolute path".to_string()));
         }
 
-        let parsed = Url::parse(&self.broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid Pulsar broker URL: {e} (value: '{}')", self.broker)))?;
         let tls_enabled = parsed.scheme() == "pulsar+ssl";
         // A CA bundle is TLS trust material a plaintext `pulsar://` broker
         // silently ignores, so treat it as a genuine misconfiguration. The
@@ -173,6 +171,10 @@ fn pulsar_producer_name(target_id: &TargetID, target_type: TargetType, nonce: Uu
 
 pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    if Url::parse(&args.broker).is_ok_and(|url| url.scheme() == "pulsar+ssl") {
+        super::ensure_rustls_provider_installed()?;
+    }
 
     let mut builder = Pulsar::builder(args.broker.clone(), TokioExecutor);
 

crates/targets/src/target/redis.rs

@@ -766,7 +766,7 @@ pub(crate) fn build_redis_client(args: &RedisArgs) -> Result<Client, TargetError
 
     let secure_scheme = matches!(args.url.scheme(), "rediss" | "valkeys");
     if secure_scheme {
-        super::ensure_rustls_provider_installed();
+        super::ensure_rustls_provider_installed()?;
         let tls_certs = TlsCertificates {
             client_tls: read_client_tls(&args.tls)?,
             root_cert: read_root_cert(&args.tls)?,

crates/targets/src/target/webhook.rs

@@ -305,6 +305,8 @@ where
         args: &WebhookArgs,
         resolver: impl reqwest::dns::Resolve + 'static,
     ) -> Result<Client, TargetError> {
+        #[cfg(feature = "fips")]
+        super::ensure_rustls_provider_installed()?;
         let mut client_builder = Client::builder()
             .no_proxy()
             .dns_resolver(resolver)
@@ -848,6 +850,51 @@ mod tests {
         }
     }
 
+    #[test]
+    fn webhook_provider_policy_isolated() {
+        const CHILD: &str = "RUSTFS_TEST_WEBHOOK_PROVIDER_POLICY";
+        if std::env::var_os(CHILD).is_some() {
+            if cfg!(feature = "fips") {
+                let mut provider = rustfs_tls_runtime::default_crypto_provider();
+                provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+                provider.install_default().expect("install incompatible host provider");
+            } else {
+                assert!(rustls::crypto::CryptoProvider::get_default().is_none());
+            }
+            let args = WebhookArgs {
+                skip_tls_verify: true,
+                ..base_args()
+            };
+            let result = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(
+                &args,
+                StaticResolver("127.0.0.1".parse().expect("loopback IP")),
+            );
+            if cfg!(feature = "fips") {
+                let error = result.expect_err("FIPS webhook must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            } else {
+                result.expect("default webhook client should build");
+                assert!(
+                    rustls::crypto::CryptoProvider::get_default().is_none(),
+                    "building a default webhook must not install a process-wide provider"
+                );
+            }
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("webhook_provider_policy_isolated")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated webhook provider test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+
     async fn http_status_url(status: u16) -> Url {
         let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
         let address = listener.local_addr().expect("test server address");

crates/tls-runtime/src/certs.rs

@@ -610,10 +610,12 @@ pub fn validate_certified_key(
             }
         }
     }
-    let certified = CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &rustls::crypto::aws_lc_rs::default_provider())
-        .map_err(|source| TlsRuntimeError::InvalidCertifiedKey {
-            name: name.to_string(),
-            source,
+    let certified =
+        CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &crate::default_crypto_provider()).map_err(|source| {
+            TlsRuntimeError::InvalidCertifiedKey {
+                name: name.to_string(),
+                source,
+            }
         })?;
     certified
         .keys_match()

crates/tls-runtime/src/lib.rs

@@ -21,6 +21,7 @@ pub mod fingerprint;
 pub mod material;
 pub mod metrics;
 pub mod outbound;
+pub mod provider;
 pub mod server;
 pub mod source;
 pub mod state;
@@ -45,6 +46,7 @@ pub use outbound::{
     GlobalOutboundTlsStateSummary, GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation,
     load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state,
 };
+pub use provider::{CryptoProviderError, CryptoProviderStatus, default_crypto_provider, ensure_crypto_provider};
 pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop};
 pub use source::{TlsFileLayout, TlsSource, TlsSourceKind, TlsSourceSnapshot};
 pub use state::OutboundOnlySnapshotArgs;

crates/tls-runtime/src/provider.rs

@@ -0,0 +1,108 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustls::crypto::CryptoProvider;
+
+/// The outcome of ensuring the process-wide rustls provider.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CryptoProviderStatus {
+    Installed,
+    AlreadyInstalled,
+}
+
+/// An incompatible process-wide rustls provider prevents FIPS startup.
+#[derive(Debug, thiserror::Error)]
+#[error(
+    "FIPS TLS requested, but the process-wide rustls crypto provider is not FIPS compatible; configure the host to install a FIPS provider before starting RustFS"
+)]
+pub struct CryptoProviderError {
+    _private: (),
+}
+
+/// Construct the provider selected by this crate's feature configuration.
+pub fn default_crypto_provider() -> CryptoProvider {
+    #[cfg(feature = "fips")]
+    {
+        rustls::crypto::default_fips_provider()
+    }
+    #[cfg(not(feature = "fips"))]
+    {
+        rustls::crypto::aws_lc_rs::default_provider()
+    }
+}
+
+/// Install or reuse the process-wide rustls provider.
+///
+/// # Errors
+/// Returns an error in a FIPS build if the provider actually installed in the
+/// process is absent or does not satisfy rustls's FIPS checks.
+pub fn ensure_crypto_provider() -> Result<CryptoProviderStatus, CryptoProviderError> {
+    // Rustls's process-wide provider is write-once; check again after a race.
+    let status = if CryptoProvider::get_default().is_some() || default_crypto_provider().install_default().is_err() {
+        CryptoProviderStatus::AlreadyInstalled
+    } else {
+        CryptoProviderStatus::Installed
+    };
+    #[cfg(feature = "fips")]
+    if !CryptoProvider::get_default().is_some_and(|provider| provider.fips()) {
+        return Err(CryptoProviderError { _private: () });
+    }
+    Ok(status)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{CryptoProviderStatus, default_crypto_provider, ensure_crypto_provider};
+
+    #[test]
+    fn provider_policy_isolated() {
+        const SCENARIO: &str = "RUSTFS_TLS_TEST_PROVIDER_SCENARIO";
+        if let Ok(scenario) = std::env::var(SCENARIO) {
+            match scenario.as_str() {
+                "fresh" => {
+                    assert_eq!(ensure_crypto_provider().expect("initial install"), CryptoProviderStatus::Installed);
+                    assert_eq!(ensure_crypto_provider().expect("provider reuse"), CryptoProviderStatus::AlreadyInstalled);
+                }
+                "incompatible" => {
+                    let mut provider = default_crypto_provider();
+                    provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+                    provider.install_default().expect("install incompatible provider");
+                    let result = ensure_crypto_provider();
+                    if cfg!(feature = "fips") {
+                        assert!(result.is_err(), "FIPS mode must reject the installed provider");
+                    } else {
+                        assert_eq!(result.expect("reuse existing provider"), CryptoProviderStatus::AlreadyInstalled);
+                    }
+                }
+                _ => panic!("unknown provider scenario"),
+            }
+            return;
+        }
+
+        for scenario in ["fresh", "incompatible"] {
+            let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+                .arg("provider_policy_isolated")
+                .arg("--nocapture")
+                .env(SCENARIO, scenario)
+                .output()
+                .expect("start isolated provider test");
+            assert!(
+                output.status.success(),
+                "{scenario}: stdout={} stderr={}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+    }
+}

docs/operations/kms-cryptographic-compliance.md

@@ -1,7 +1,7 @@
 # Cryptographic compliance positioning
 
 **Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs` / `rustfs-tls-runtime` / `rustfs-crypto` `fips` features and algorithm deprecation.
-**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `rustfs/Cargo.toml` and `crates/tls-runtime/Cargo.toml` (`fips` feature); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
+**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); the `fips` features in `rustfs/Cargo.toml` and the affected library manifests; `crates/tls-runtime/src/provider.rs` (provider policy); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes).
 
 This document records where RustFS stands on cryptographic module validation and what may and may not be said about it, so the question is answered once, from the code. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md).
 
@@ -52,18 +52,22 @@ Boilerplate when the topic cannot be avoided:
 
 ## The `rustfs` and `rustfs-tls-runtime` `fips` feature
 
-`rustfs/Cargo.toml` (`fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]`) and `crates/tls-runtime/Cargo.toml` (`fips = ["rustls/fips"]`) define an opt-in Cargo feature (disabled by default) for builds that require the `aws-lc-fips-sys` cryptographic module on the `aws-lc-rs` / `rustls` path:
+`rustfs/Cargo.toml` and the TLS runtime, protocols, targets, S3 client, and ECStore library manifests define opt-in `fips` features (disabled by default) for the `aws-lc-fips-sys` / `rustls` path. The RustFS feature propagates the policy to these libraries; when enabling a protocol library independently, also enable the required protocol feature.
+
+Cargo's `--all-features` also enables `fips`, including for these individual libraries. Such builds inherit the FIPS backend's target and native-tool requirements; select supported feature combinations deliberately rather than assuming every all-features build is supported.
 
 ```bash
 cargo build --release --locked -p rustfs --bin rustfs --features fips
 ```
 
 - **What it enables:** Activates `rustls/fips` and `tokio-rustls/fips`, which pull in `aws-lc-rs/fips` (`aws-lc-fips-sys`) and `rustls-webpki/aws-lc-rs-fips`. Cargo unifies features for the same dependency version in a build graph; consumers of that `aws-lc-rs` instance use its FIPS backend, but this does not establish approval for every operation or cover independent dependency versions.
-- **Runtime provider installation:** `rustfs/src/startup_runtime_hooks.rs` attempts to install `rustls::crypto::default_fips_provider()` and checks the actual process-wide provider before observability initialization in both standalone and embedded startup. A FIPS build fails startup if an embedded host has already installed a provider that does not pass rustls `CryptoProvider::fips()`; the host must install a compatible provider before starting RustFS. The check does not inspect clients that use their own provider or a different TLS implementation.
+- **Runtime provider installation:** `rustfs-tls-runtime` installs or reuses the process-wide provider and checks the actual provider in FIPS builds. RustFS performs this check before observability initialization in standalone and embedded startup. FTPS, WebDAV, transition clients, and relevant notification TLS clients also invoke the shared policy before using the process-wide provider. An incompatible host provider causes a configuration error. The check does not inspect clients that use their own provider or a different TLS implementation.
 - **Build requirements:** The FIPS backend requires a C/C++ toolchain, CMake, and Go; targets without suitable pre-generated bindings also require bindgen and Clang/`libclang`. Check the locked dependency version and target platform for their exact requirements.
 - **What it does not change:** It does not alter the object data-path AEAD (`crates/kms`, `crates/rio`, `crates/rio-v2`) or `rustfs-crypto` config/IAM blob encryption, which continue to use pure-Rust `RustCrypto` crates.
 
-The RustFS server TLS configuration uses the process-wide rustls provider. FTPS, WebDAV, tier clients, and notification targets have their own provider initialization paths; the remote replication client also explicitly selects an AWS-LC TLS provider. Feature unification affects the same dependency version in a build graph, but independent builds, explicit providers, and third-party TLS stacks require separate verification. No complete outbound TLS or platform validation matrix is asserted here.
+The RustFS server TLS configuration uses the process-wide rustls provider. The Kafka notification client and remote replication SDK explicitly select AWS-LC, so their providers are independent of the process default; for the locked rustls version, the unified `rustls/fips` feature restricts that provider's suites and groups. ECStore's insecure replication-client path still uses the process default: an application using ECStore independently must directly depend on `rustfs-tls-runtime` with its `fips` feature and call `rustfs_tls_runtime::ensure_crypto_provider()` before constructing that client when it requires the FIPS policy. Skipping TLS certificate verification does not authenticate the remote target, regardless of provider selection. Feature unification applies only to the same dependency version in a build graph; independent builds and third-party TLS stacks require separate verification. No complete outbound TLS or platform validation matrix is asserted here.
+
+NATS can upgrade a connection to TLS based on the server's response, so the targets library checks the process provider before connecting in a FIPS build, including for a plaintext-looking NATS URL. Webhook targets construct a TLS-capable HTTP client even for an HTTP endpoint and also check the provider when built with `fips`. An incompatible host provider therefore prevents these initializations even if a particular connection would not use TLS.
 
 A provider check or linked FIPS backend does not establish CMVP validation for a deployment or approval of every operation. Verify the specific module version, certificate status, security policy, build, operating environment, and approved services against the [CMVP validated modules list](https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules). The feature is opt-in; before enabling it, test client algorithms, certificates, TLS versions, rolling upgrades, and rollback in the target environment.
 

rustfs/Cargo.toml

@@ -58,7 +58,7 @@ required-features = ["swift"]
 
 [features]
 default = ["ftps", "webdav", "gcs"]
-fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]
+fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips"]
 gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]

rustfs/src/startup_runtime_hooks.rs

@@ -14,10 +14,7 @@
 
 use crate::license::license_status;
 use crate::startup_runtime_sources;
-#[cfg(not(feature = "fips"))]
-use rustls::crypto::aws_lc_rs::default_provider;
-#[cfg(feature = "fips")]
-use rustls::crypto::default_fips_provider as default_provider;
+use rustfs_tls_runtime::{CryptoProviderStatus, ensure_crypto_provider};
 use std::future::Future;
 use std::io::Result;
 use tracing::{debug, info, warn};
@@ -108,25 +105,8 @@ where
     shutdown();
 }
 
-fn build_default_crypto_provider() -> rustls::crypto::CryptoProvider {
-    default_provider()
-}
-
-fn ensure_default_crypto_provider() -> Result<bool> {
-    // The process-wide provider is write-once, so an existing provider needs no replacement.
-    let already_installed =
-        rustls::crypto::CryptoProvider::get_default().is_some() || build_default_crypto_provider().install_default().is_err();
-    #[cfg(feature = "fips")]
-    if !rustls::crypto::CryptoProvider::get_default().is_some_and(|provider| provider.fips()) {
-        return Err(std::io::Error::other(
-            "FIPS TLS requested, but the process-wide rustls crypto provider is not FIPS compatible; configure the host to install a FIPS provider before starting RustFS",
-        ));
-    }
-    Ok(already_installed)
-}
-
 pub(crate) fn install_default_crypto_provider() -> Result<()> {
-    if ensure_default_crypto_provider()? {
+    if ensure_crypto_provider().map_err(std::io::Error::other)? == CryptoProviderStatus::AlreadyInstalled {
         debug!(
             target: "rustfs::main",
             event = EVENT_CRYPTO_PROVIDER_STATE,
@@ -177,7 +157,7 @@ pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<
 }
 
 fn install_embedded_default_crypto_provider() -> Result<()> {
-    if ensure_default_crypto_provider()? {
+    if ensure_crypto_provider().map_err(std::io::Error::other)? == CryptoProviderStatus::AlreadyInstalled {
         debug!(
             event = EVENT_CRYPTO_PROVIDER_STATE,
             component = LOG_COMPONENT_EMBEDDED,
@@ -192,9 +172,10 @@ fn install_embedded_default_crypto_provider() -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::{
-        build_default_crypto_provider, init_profiling_runtime_with, install_default_crypto_provider,
-        install_embedded_default_crypto_provider, shutdown_profiling_runtime_with,
+        init_profiling_runtime_with, install_default_crypto_provider, install_embedded_default_crypto_provider,
+        shutdown_profiling_runtime_with,
     };
+    use rustfs_tls_runtime::default_crypto_provider as build_default_crypto_provider;
     use std::sync::{
         Arc,
         atomic::{AtomicBool, Ordering},

scripts/ci_gate.py

@@ -212,6 +212,13 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
         ("connect-release-rejection", "connect-feature-boundaries", "release", ["connect-e2e-short-credentials"]),
         ("fips-provider-default-lint", "fips-provider", "dev", []),
         ("fips-provider-fips-lint", "fips-provider", "dev", ["fips"]),
+        ("fips-tls-runtime-default", "fips-provider", "test", []),
+        ("fips-tls-runtime-fips", "fips-provider", "test", ["fips"]),
+        ("fips-protocols", "fips-provider", "test", ["ftps", "webdav", "fips"]),
+        ("fips-targets", "fips-provider", "test", ["fips"]),
+        ("fips-s3-client", "fips-provider", "test", ["fips"]),
+        ("fips-targets-default", "fips-provider", "test", []),
+        ("fips-s3-client-default", "fips-provider", "test", []),
         ("fips-provider-default", "fips-provider", "test", []),
         ("fips-provider-fips", "fips-provider", "test", ["fips"]),
         ("rio-v2-lint", "test-and-lint-rio-v2", "dev", ["rio-v2"]),