fix(ecstore): complete and abort native multipart uploads on Windows

Native multipart completion publishes its payload from a TMP staging
parent that differs from the canonical head's parent. Linux pins that
parent with a dedicated payload commit guard, but Windows reused the
metadata guard, so the data-dir rename failed its parent check with
InvalidInput, which surfaced as FileNotFound/NoSuchKey.

- Add a Windows multipart payload commit guard that pins the staging
  tree and reuses the metadata transaction's destination parent handle.
- Run the owned-source rename precondition on Windows before opening the
  exclusive DELETE handle, under a non-exclusive identity anchor, and
  require the prepared handle to name the same file. The precondition's
  own opens previously hit ERROR_SHARING_VIOLATION during abort and
  stale-upload cleanup.
- Recognize ERROR_ACCESS_DENIED on a directory as a directory in the
  remove fallback, so remove/remove_std can delete directories on
  Windows; run each fallback in one blocking dispatch.
- Run the complete/abort and directory-removal tests in the Windows
  filesystem workflow, and add a Windows payload guard regression test.

.github/workflows/windows-filesystem.yml

@@ -107,6 +107,21 @@ jobs:
         shell: pwsh
         run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
 
+      # Native multipart completion publishes its payload from a different
+      # parent than its metadata, and abort renames under a precondition; both
+      # need their own Windows handle discipline.
+      - name: Test native multipart complete and abort publication
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
+          native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
+          --nocapture
+
+      - name: Test directory removal fallback
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
+
       - name: Test Windows path conversion
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

Cargo.lock

@@ -908,12 +908,28 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "aws-lc-fips-sys"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b744eed18cd75d814effdf2aba2880f912abe0fe1c327b361341d31aba9a7005"
+dependencies = [
+ "bindgen",
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "pkg-config",
+ "regex",
+]
+
 [[package]]
 name = "aws-lc-rs"
 version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53"
 dependencies = [
+ "aws-lc-fips-sys",
  "aws-lc-sys",
  "untrusted 0.7.1",
  "zeroize",
@@ -1524,6 +1540,8 @@ dependencies = [
  "bitflags 2.13.2",
  "cexpr",
  "clang-sys",
+ "log",
+ "prettyplease 0.3.0",
  "proc-macro2",
  "quote",
  "regex",

crates/ecstore/src/disk/fs.rs

@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
     native_io(move || create_dir_all_std(&path)).await
 }
 
-fn is_dir_error(e: &io::Error) -> bool {
+fn is_dir_error(path: &Path, e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
         // macOS: remove_file on a directory returns EPERM
         || (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
+        || is_windows_dir_access_denied(path, e)
+}
+
+/// Windows reports `remove_file` on a directory (or directory symlink) as
+/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
+/// failure without inspecting the entry itself.
+#[cfg(windows)]
+fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
+    use std::os::windows::fs::FileTypeExt;
+
+    e.kind() == io::ErrorKind::PermissionDenied
+        && std::fs::symlink_metadata(path).is_ok_and(|metadata| {
+            let file_type = metadata.file_type();
+            file_type.is_dir() || file_type.is_symlink_dir()
+        })
+}
+
+#[cfg(not(windows))]
+fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
+    false
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
-    // Try remove_file first; fall back to remove_dir if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
-        Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir(path)).await
-        }
-        Err(e) => Err(e),
-    }
+    let path = path.as_ref().to_path_buf();
+    native_io(move || remove_std(path)).await
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir_all if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
+    let path = path.as_ref().to_path_buf();
+    native_io(move || match std::fs::remove_file(&path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir_all(path)).await
-        }
+        Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
         Err(e) => Err(e),
-    }
+    })
+    .await
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir if it's a directory
-    match std::fs::remove_file(path.as_ref()) {
+    let path = path.as_ref();
+    match std::fs::remove_file(path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
+        Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
         Err(e) => Err(e),
     }
 }

crates/ecstore/src/disk/local/commit.rs

@@ -1594,12 +1594,13 @@ impl LocalDisk {
                 );
             }
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let payload_guard = if multipart_publication.is_some() {
                     Some(
                         os::multipart_payload_commit_guard(
                             &commit.directory_guard,
                             src_data_path.parent().ok_or(DiskError::InvalidPath)?,
+                            commit.publication_root,
                             commit.mutation_lease.clone(),
                         )
                         .await
@@ -1608,9 +1609,9 @@ impl LocalDisk {
                 } else {
                     None
                 };
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
-                #[cfg(not(target_os = "linux"))]
+                #[cfg(not(any(target_os = "linux", windows)))]
                 let directory_guard = &commit.directory_guard;
                 if let Err(err) = os::rename_all_with_commit_guard(
                     src_data_path,

crates/ecstore/src/disk/os.rs

@@ -5479,18 +5479,47 @@ pub(crate) fn multipart_payload_commit_guard_std(
     })
 }
 
+#[cfg(windows)]
+fn multipart_payload_commit_guard_std(
+    metadata_guard: &RenameCommitGuard,
+    source_parent: &Path,
+    publication_root: &PublicationRoot,
+) -> io::Result<RenameCommitGuard> {
+    // Pin the payload staging tree like a distinct-tree metadata source; the
+    // target stays the exact parent handle the metadata transaction holds.
+    let source_parent_guard = lock_windows_directory_tree(source_parent, Some(source_parent), publication_root)?;
+    Ok(RenameCommitGuard {
+        source_parent: source_parent.to_path_buf(),
+        destination_parent: metadata_guard.destination_parent.clone(),
+        source_parent_guard,
+        destination_parent_guard: metadata_guard.destination_parent_guard.clone(),
+    })
+}
+
 /// Native MPU metadata and payload have distinct source parents. Reuse the
 /// exact target identity already held by the metadata transaction, while
 /// independently pinning the private payload source under the same root.
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", windows))]
 pub(crate) async fn multipart_payload_commit_guard(
     metadata_guard: &RenameCommitGuard,
     source_parent: &Path,
+    publication_root: &PublicationRoot,
     lease: Arc<NamespaceMutationLease>,
 ) -> io::Result<RenameCommitGuard> {
     let metadata_guard = metadata_guard.clone();
     let source_parent = source_parent.to_path_buf();
-    run_blocking_namespace_operation(lease, move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)).await
+    #[cfg(target_os = "linux")]
+    let operation = {
+        // The Linux guard already carries its publication root.
+        let _ = publication_root;
+        move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)
+    };
+    #[cfg(windows)]
+    let operation = {
+        let publication_root = publication_root.clone();
+        move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent, &publication_root)
+    };
+    run_blocking_namespace_operation(lease, operation).await
 }
 
 pub(crate) fn prepare_rename_commit_guard(
@@ -6601,12 +6630,31 @@ pub(crate) fn rename_multipart_source_owned_std(
     {
         let _ = &guard;
         let source_guard = pin_multipart_source_directory(source_parent, base, root)?;
+        // The prepared Windows source holds DELETE access and shares only
+        // read, so the precondition's own opens of that entry (or, for a
+        // directory, of its guarded tree) would fail with a sharing violation.
+        // Check it first under a non-exclusive identity anchor, then require
+        // the exclusive handle to name the same file.
+        #[cfg(windows)]
+        let (_source_identity_anchor, expected_source_identity) = {
+            let anchor = open_windows_rename_source_identity(source, &source_guard)?;
+            precondition()?;
+            anchor
+        };
         let preparation = prepare_rename_with_retry(source, destination, base, root)?;
+        #[cfg(windows)]
+        if windows_file_identity(&preparation.source)? != expected_source_identity {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "rename source identity changed after its precondition",
+            ));
+        }
         #[cfg(all(any(test, feature = "test-util"), not(windows)))]
         {
             prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
             prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
         }
+        #[cfg(not(windows))]
         precondition()?;
         #[cfg(unix)]
         validate_prepared_rename_source(&pinned_source, source)?;
@@ -7193,7 +7241,7 @@ mod multipart_source_preservation_tests {
         let metadata_guard =
             prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
         let lease = acquire_namespace_mutation_lease(&target).await;
-        let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
+        let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
             .await
             .expect("distinct pinned payload source");
         assert!(
@@ -7228,6 +7276,61 @@ mod multipart_source_preservation_tests {
         );
     }
 
+    #[cfg(windows)]
+    #[tokio::test]
+    async fn windows_multipart_payload_guard_publishes_distinct_source_into_metadata_target() {
+        // Build paths from the configured root: the resolved handle path may
+        // be verbatim, where `/` in a joined key is not a separator.
+        let (root, publication, _source, _destination) = fixture();
+        let root = root.path();
+        let canonical = root.join("multipart/upload");
+        let temporary = root.join("tmp/attempt");
+        let bucket = root.join("bucket");
+        let target = bucket.join("object");
+        for directory in [&canonical, &temporary, &target] {
+            std::fs::create_dir_all(directory).expect("two-source guard directories");
+        }
+        let payload = temporary.join("payload");
+        std::fs::create_dir(&payload).expect("private payload DD");
+        std::fs::write(payload.join("part.1"), b"native selected shard").expect("private payload shard");
+        let head = canonical.join("xl.meta");
+        let destination = target.join("xl.meta");
+        let metadata_guard =
+            prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
+        let lease = acquire_namespace_mutation_lease(&target).await;
+        assert!(
+            rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &metadata_guard, lease.clone())
+                .await
+                .is_err(),
+            "the metadata guard does not pin the private payload parent"
+        );
+        assert!(
+            payload.join("part.1").exists(),
+            "a rejected payload rename must leave its source in place"
+        );
+        let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
+            .await
+            .expect("distinct pinned payload source");
+        let mut prepared = create_prepared_rename_source_with_commit_guard(&head, &destination, &metadata_guard)
+            .expect("canonical prepared metadata");
+        prepared
+            .write_all(b"prepared native metadata", true)
+            .expect("durable canonical metadata");
+        rename_all_with_commit_guard(&payload, target.join("payload"), &bucket, &publication, &payload_guard, lease.clone())
+            .await
+            .expect("payload rename uses its own pinned TMP parent");
+        assert!(head.exists(), "payload publication must not consume the canonical head");
+        rename_all_with_prepared_source(prepared, &head, &destination, &bucket, &publication, &metadata_guard, lease)
+            .await
+            .expect("metadata rename keeps its canonical parent");
+        assert!(!head.exists());
+        assert_eq!(std::fs::read(&destination).expect("published metadata"), b"prepared native metadata");
+        assert_eq!(
+            std::fs::read(target.join("payload/part.1")).expect("published selected shard"),
+            b"native selected shard"
+        );
+    }
+
     #[cfg(target_os = "linux")]
     #[tokio::test]
     async fn multipart_payload_guard_rejects_replaced_private_source_or_shared_destination() {
@@ -7249,7 +7352,7 @@ mod multipart_source_preservation_tests {
             let metadata_guard =
                 prepare_rename_commit_guard(&canonical, &target, &bucket, &publication).expect("canonical metadata guard");
             let lease = acquire_namespace_mutation_lease(&target).await;
-            let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, lease.clone())
+            let payload_guard = multipart_payload_commit_guard(&metadata_guard, &temporary, &publication, lease.clone())
                 .await
                 .expect("pin payload and shared target");
             let replaced = if replace_source { &temporary } else { &target };

crates/filemeta/src/metacache.rs

@@ -3775,6 +3775,74 @@ mod tests {
         assert!(corrupt_crc_entry.is_latest_delete_marker());
     }
 
+    #[test]
+    fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
+        let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
+            .expect("load realistic xl.meta");
+        let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
+        let mut free_meta_sys = HashMap::new();
+        rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
+        let free = FileMetaShallowVersion::try_from(FileMetaVersion {
+            version_type: VersionType::Delete,
+            delete_marker: Some(MetaDeleteMarker {
+                version_id: Some(Uuid::from_u128(42)),
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
+                meta_sys: free_meta_sys,
+            }),
+            write_version: 1,
+            ..Default::default()
+        })
+        .expect("encode cleanup record");
+        fm.versions.insert(0, free);
+        let entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scanner version decode");
+        assert_eq!(scanned.free_versions.len(), 1);
+        assert_eq!(scanned.versions.len(), 1);
+        assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
+        assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
+        let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
+        assert_eq!(listed.version_id, object.version_id);
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
+        assert_eq!(listed.size, scanned.versions[0].size);
+        assert!(!listed.deleted);
+
+        let marker_id = Uuid::from_u128(43);
+        fm.add_version(FileInfo {
+            volume: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(marker_id),
+            deleted: true,
+            mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
+            ..Default::default()
+        })
+        .expect("add current delete marker");
+        let mut marker_entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode marker metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scan marker stack");
+        assert_eq!(scanned.versions.len(), 2);
+        assert_eq!(scanned.versions[0].version_id, Some(marker_id));
+        assert!(scanned.versions[0].deleted);
+        assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
+        assert!(marker_entry.is_latest_delete_marker());
+        let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert!(listed.deleted);
+    }
+
     #[test]
     fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
         let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))

crates/tls-runtime/Cargo.toml

@@ -29,6 +29,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustls/fips"]
 hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-common/hotpath", "rustfs-config/hotpath"]
 hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"]
 hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"]

deny.toml

@@ -96,4 +96,8 @@ exceptions = [
     # transitively via `bzip2`. Not on a hot path.
     # owner: rustfs-maintainers  review: 2026-07
     { allow = ["bzip2-1.0.6"], crate = "libbz2-rs-sys" },
+    # `aws-lc-fips-sys` declares ISC, Apache-2.0, and the OpenSSL license for
+    # the FIPS-validated AWS-LC module when the optional `fips` feature is enabled.
+    # owner: rustfs-maintainers  review: 2026-10
+    { allow = ["OpenSSL"], crate = "aws-lc-fips-sys" },
 ]

docs/architecture/README.md

@@ -46,6 +46,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
 | [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller |
 | [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects |
 | [scanner-usage-publication.md](scanner-usage-publication.md) | changing scanner data-usage cache publication, quota-visible usage snapshots, scanner cycle recovery, or the persisted scanner usage artifacts |
+| [scanner-prefix-statistics-contract.md](scanner-prefix-statistics-contract.md) | designing, implementing, or reviewing scanner-derived prefix statistics for the object browser or another authorized consumer (proposed; not runtime implemented) |
 | [scanner-dirty-journal-source-contract.md](scanner-dirty-journal-source-contract.md) | designing storage-verified provenance, completion receipts, repair responsibility or replay for the node-owned scanner dirty-producer journal (proposed; not runtime implemented) |
 | [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
 | [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |

docs/architecture/scanner-prefix-statistics-contract.md

@@ -0,0 +1,84 @@
+# Scanner Prefix Statistics Contract
+
+**Use this when:** designing, implementing, or reviewing scanner-derived prefix statistics for the object browser or another authorized consumer.
+**Status:** proposed contract; scanner-derived current-object prefix statistics, their indexed persistence, and their query API are not implemented by this document.
+**Source of truth:** `crates/scanner/src/prefix_usage.rs`, `crates/data-usage/src/data_usage.rs`, `crates/scanner/src/scanner_io/io_disk.rs`, `crates/filemeta/src/metacache.rs`, `crates/ecstore/src/object_api/types.rs`, `crates/ecstore/src/store/list_objects.rs`, and `rustfs/src/storage/s3_api/bucket.rs`.
+
+## Purpose and current boundary
+
+A browser needs the total size of current visible objects below a displayed folder prefix and the latest modification time among those objects, including when a prefix contains very many objects. The intended result is an asynchronous scanner observation with explicit freshness and coverage, not an atomic S3 snapshot or an unconditional near-real-time guarantee. A consumer must be able to distinguish a completed observation from an incomplete, stale, or unavailable result. The requested operating scale is hundreds of millions of objects; a freshness target at that scale requires workload-specific measurement.
+
+The existing `bucket_prefix_usage` reads per-set usage caches and exposes subtree counters and cache update time. Its `PrefixUsageSummary` does not contain current-object modification time, and scanner size accounting can include historical versions. Cache path normalization and subtree compaction can lose the literal prefix identity or the child breakdown. Consequently that interface must not be presented as this contract's current-object statistics without a separate implementation and proof.
+
+The current S3 list path filters current delete markers and version-purge entries, and excludes free versions when selecting the visible metadata. Its response uses `ObjectInfo::get_actual_size_or_physical` and `mod_time`; for legacy data, a physical-size fallback can therefore appear where logical size is unknown. The scanner reads local metadata and has its own failure and completeness rules, so equivalent single-object decoding alone does not prove equivalence to a quorum-based listing.
+
+This contract does not change the existing scanner usage publication, quota, repair, or S3 API guarantees. Those guarantees remain owned by [scanner usage publication](scanner-usage-publication.md), [scanner convergence](scanner-per-bucket-convergence.md), and [placement and repair invariants](placement-repair-invariants.md).
+
+## Observable semantics
+
+A result describes the literal requested prefix and the current visible objects covered by that prefix. The modification time is the maximum modification time of those objects, not a property supplied by S3 for the prefix itself and not the time at which a scanner cache was written. Empty coverage has a zero byte count and no object modification time only when the entire requested scope was successfully observed and is known to be empty.
+
+Current-version selection and returned object size must be checked against the ordinary recursive `ListObjectsV2` projection, including noncurrent versions, a current delete marker, tier cleanup free versions, version-purge state, legacy compressed metadata, encryption, directory-marker objects, and absent or invalid timestamps. If the implementation cannot establish a field, it must mark that field unknown or the observation incomplete. A compatibility fallback used by S3 must be identified rather than represented as a proven logical byte count.
+
+Prefix keys are literal S3 names, not filesystem paths. Preserve separators, percent signs, Unicode, and distinct spellings without path cleaning, repeated decoding, or prefix normalization. The API must define its supported scope, including whether it accepts only the delimiter-derived folder prefixes returned by the browser or arbitrary byte prefixes. An unsupported, compacted-away, or absent index entry is unavailable unless independently certified as empty. A trailing-slash key such as `foo/` and a literal key such as `foo__XLDIR__` can map to the same physical encoding; `crates/scanner/src/scanner_folder/prefix_attribution.rs` explicitly treats this suffix as ambiguous. A physical name or suffix decoder alone is not trusted evidence of the original key: require a verified identity source that distinguishes the represented logical key, or report the affected scope as partial or unavailable without assigning the observation to either spelling.
+
+Counts and byte totals must use checked arithmetic. All byte totals must cross the API boundary as decimal strings, including small values, with their encoding specified by the API. Missing values, arithmetic overflow, corrupt metadata, and negative or indeterminate sizes must never become a plausible zero or saturated total.
+
+## Observation lifecycle
+
+The following states describe the externally visible observation, not new runtime types or implemented endpoints. Coverage and freshness remain separate fields even when the state is summarized for a consumer.
+
+| State | Admission and transition |
+|---|---|
+| Unavailable | No verified compatible observation covers the request, or the caller cannot be given one; never return a fabricated zero. |
+| Building | A bounded scan or rebuild is in progress; a separately verified last-known-good observation may remain available with its original coverage and age. |
+| Partial | A verified candidate covers only named ranges or sets, or contains unknown fields; missing, failed, and unsupported portions remain explicit. |
+| Complete | All required sets and requested ranges have compatible verified coverage, required fields, and quorum or equivalent authority for visible-version selection; this certifies an observation over a time interval, not an atomic snapshot. |
+| Stale | A retained verified observation is no longer within the declared freshness target or its scope was invalidated; retain its previous coverage and timestamps. |
+| Failed | A build, update, or verification failed; expose its reason without replacing a verified observation with an unverified candidate. |
+
+A successful build can move from building to complete only after the coverage and publication checks below succeed; otherwise it remains partial or unavailable. A failed build moves to failed while a previous observation, if any, remains separately labelled stale or retains the freshness supported by its evidence. Retry starts another bounded build; cancellation stops that build without changing the last verified observation. A bucket incarnation or layout mismatch invalidates the affected identity rather than relabelling its data for a new bucket or set.
+
+## Coverage and freshness
+
+A result carries its coverage and freshness independently. Internally, its evidence binds the bucket incarnation and requested literal prefix, contributing and expected set identities, each set's observation interval or oldest and newest represented observations, the completeness of its represented scope, and reasons for missing or unknown fields. The response exposes only the authorized, redacted coverage and freshness needed to interpret the result, as required below. A consumer may show a partial observed total only if it is labelled partial and its covered scope is explicit.
+
+A completed traversal proves only the coverage actually certified by that traversal; it does not create a point-in-time snapshot across sets or concurrent writes. A last-known-good shard or reused subtree retains its own oldest observation time. A publication timestamp, latest set update, or cache invalidation time must not be substituted for the time range of the data represented. Traversal completion on one local disk is not proof of an authoritative visible version. Before a scope is complete, require the same cross-disk quorum and version-selection evidence used by S3 listing, or separately verified equivalent evidence; stale replicas, competing delete markers, unavailable disks, and conflicting cross-pool migration observations must remain partial or unavailable until resolved. For each object, replica disagreement or missing authority cannot be settled by choosing whichever metadata the scanner saw first.
+
+Before any valid observation exists, the result is unavailable or building. A bounded traversal may publish an explicitly partial result when its covered ranges and unresolved gaps are known; after all required scope is certified it may publish a complete observation. A later mutation, expired freshness target, incomplete retry, topology change, unreadable set, or failed verification marks the affected result stale, partial, or unavailable as appropriate. A reader must never infer currentness solely from an earlier complete traversal.
+
+The result must expose the distinction between coverage and age even when a consumer requests a single summary. A complete but old result remains complete for its recorded observation and stale for current use; a recently updated partial result remains partial. The deployment must define freshness targets and report measured observation lag rather than claim an unverified deadline.
+
+Freshness certification requires evidence that relevant mutations and generation changes in the represented scope are covered through the claimed observation boundary. A dirty notification or local generation alone is not proof that every producer, peer, pending physical tail, or data-movement path was observed. A missing producer, unknown generation, replay gap, mixed-version peer, lost notification, or unverified journal interval must invalidate the freshness claim and leave the affected observation stale, partial, or unavailable until coverage is re-established. This document does not assert that a complete mutation journal or such certification already exists.
+
+## Persistence and recovery
+
+The index requires a separately versioned persistence contract. Reuse the scanner's admission, pacing, traversal, checkpoint, and coverage mechanisms only where their identities and guarantees match the indexed scope. Do not change the meaning of the existing `.usage-cache.bin` data or quota-visible totals to store the new result.
+
+Partition persistent index data so that reading or updating one scope does not require loading all prefixes in a bucket into memory. A manifest identifies its immutable bounded shards, literal key ranges, format version, bucket incarnation, topology and set identities, scanner generation, observation intervals, coverage, and integrity evidence. The supported prefix query and shard bounds must be explicit; pagination and continuation tokens must bind the requested scope and compatible generation. Validate untrusted cursors for authenticity, expiry, version, bounds, principal and authorized scope, bucket incarnation, prefix, and generation before use; reject altered, cross-scope, stale, or unsupported cursors without leaking indexed data.
+
+Publish a generation only after all referenced shard writes are durable and verified and the active writer's leadership, bucket incarnation, topology, and publication fences remain valid at the commit point. A concrete implementation must establish the atomic publication or conditional replacement primitive and its failure behavior; this document does not assert that a suitable primitive already exists. Readers validate the manifest and shard identity before use and retain a previous verified generation only with its original freshness and coverage labels.
+
+Interrupted work may resume only from a validated checkpoint bound to the same scope, generation, bucket incarnation, layout, and coverage evidence. Otherwise discard the candidate or rebuild it; do not combine incompatible generations or count both a replayed segment and its prior contribution. The release design must handle competing writers, crashes before and after publication, missing shards, failed reads, corruption, leader replacement, storage exhaustion, bucket deletion and recreation, resharding, rollback, and orphan cleanup.
+
+Unknown format versions and legacy records lacking the new proof are unavailable, not empty. During rolling upgrades, an older writer must not erase a newer index or certify new fields as complete; keep independent index ownership and version fencing, and provide a supported rollback path. A stopped or failed index must not block object writes, lifecycle, healing, or existing scanner usage publication.
+
+## Authorization and disclosure
+
+An index query must authenticate and authorize the caller for the actual bucket and literal prefix before reading or returning index data. Its decision must respect the same applicable `ListBucket` policy semantics as the corresponding recursive S3 listing, including prefix conditions, bucket policy, explicit denies, session restrictions, and current credentials. The operation must not use a broader background or administrator identity to disclose information unavailable to the caller.
+
+A parent-prefix aggregate may reveal objects in a restricted descendant. The implementation must establish that the caller can obtain the represented scope through the corresponding listing policy or return no aggregate for that scope. Pagination, cached responses, progress, error details, and last-known-good data are subject to the same authorization and must not expose other buckets, prefixes, or principals. Reauthorize each request; a cached allow decision must not survive a relevant policy or credential change without a proven invalidation mechanism. Ordinary users receive only the coverage and freshness information needed to interpret their authorized result; set identifiers, internal topology, physical paths, and detailed failure provenance require a separately authorized administrative surface. Redaction must also apply to cursors, progress, and error messages.
+
+The existing admin prefix-usage handler uses the shared admin gate before parsing the requested bucket. Its current authorization and response semantics must not be assumed to implement this contract; an integration must establish and test the actual resource and query-condition decision before using that route.
+
+## Resource and lifecycle boundaries
+
+Index construction and refresh are background work. Bound admission, queue depth, per-bucket and per-tenant concurrency, scan work per scheduling quantum, disk and network I/O, memory, shard size, retained generations, and disk usage. Cancellation and restart must stop or resume work without exposing unfinished data as complete. Refresh, compaction, cleanup, and a slow or unavailable set must not starve foreground object operations.
+
+A design must avoid accumulating the object population or an unbounded set of prefix rows in process memory, or issuing a full object-list scan for every displayed row. Query admission must enforce independent hard limits on requested page size, shards visited, bytes read, CPU time, memory, concurrent requests, and request duration; when a budget is exhausted, return a bounded partial or unavailable result with an appropriate continuation only when continuation is safe. It must account for key depth and write amplification, large single prefixes, many sibling prefixes, pagination, topology changes, and backpressure. Resource exhaustion must produce a visible degraded or unavailable state and retain object-store correctness; it must not cause silent data loss or an unbounded retry loop.
+
+## Required evidence
+
+Before the indexed result is enabled, tests must compare its current-object projection with S3 listings for ordinary, versioned, deleted, transitioned, legacy, corrupt, and unusual literal keys. Integration tests must cover scope-limited allow and deny policies, session changes, pagination and hostile cursors, internal-topology redaction, multiple sets, incomplete or corrupt shards, old formats, mixed versions, deletion and recreation, leader changes, cancellation, retries, and concurrent object mutations. Differential and fault tests must cover `foo/` versus literal `foo__XLDIR__`, stale replicas, competing delete markers, unavailable quorum, cross-pool migration conflicts, and gaps in mutation or generation evidence; no such case may silently certify a guessed key or version.
+
+Recovery tests must inject failure around shard writes, manifest publication, checkpoint replay, and cleanup and prove that readers never certify missing or mismatched data. Performance validation must measure scan completion and observation lag, memory and persistent-space growth, I/O load, and foreground workload impact under representative large populations and concurrent writes. Claims for the requested hundreds-of-millions scale and any near-real-time target require measured evidence at the stated scale and workload; a unit test, small successful scan, or unsupported extrapolation does not establish them. Larger scale claims require their own target and evidence and are not implied by this contract.

docs/operations/kms-cryptographic-compliance.md

@@ -1,29 +1,29 @@
 # Cryptographic compliance positioning
 
-**Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs-crypto` `fips` feature and algorithm deprecation.
-**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
+**Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs` / `rustfs-tls-runtime` / `rustfs-crypto` `fips` features and algorithm deprecation.
+**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `rustfs/Cargo.toml` and `crates/tls-runtime/Cargo.toml` (`fips` feature); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
 
 This document records where RustFS stands on cryptographic module validation and what may and may not be said about it, so the question is answered once, from the code. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md).
 
 ## Status: not FIPS 140-3 validated
 
-**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration. This is a deliberate position and not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or pursues.
+**RustFS is not FIPS 140-3 (or 140-2) validated as a product, and in the default build no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS itself, or the libraries it uses in the default shipped configuration. An opt-in `--features fips` build flag on `rustfs` switches the `rustls` / `aws-lc-rs` provider to `aws-lc-fips-sys` for TLS, X.509, and JWT operations, while object data-path and local KMS operations remain on unvalidated RustCrypto software implementations. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS as a whole has or pursues.
 
 ### What the process actually links
 
 | Layer | Where | Implementation | Primitives | Validated module |
 | --- | --- | --- | --- | --- |
-| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant |
+| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider (`default_provider` by default; `default_fips_provider` when built with `--features fips`) | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange (restricted to FIPS-approved suites and groups when `--features fips` is enabled) | No in the default build (`aws-lc-sys`); opt-in `--features fips` links `aws-lc-fips-sys` and asserts `provider.fips()` at startup |
 | Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
 | DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
 | Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
-| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
-| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS build | No |
+| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `rustfs-crypto` `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
+| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS `aws-lc-sys` build by default; unified with `aws-lc-fips-sys` when `rustfs` is built with `--features fips` | No in the default build; links `aws-lc-fips-sys` only when built with `-p rustfs --features fips` |
 
 Two consequences are commonly assumed the other way:
 
-- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it anywhere.
-- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see route 1 below.
+- **AWS-LC being present does not imply FIPS in the default build.** `aws-lc-rs` has a FIPS variant (`aws-lc-fips-sys`); default builds do not enable it. Building `rustfs` with `--features fips` enables `rustls/fips`, `tokio-rustls/fips`, and `rustfs-tls-runtime/fips`, which links `aws-lc-fips-sys` and installs `rustls::crypto::default_fips_provider()` (see [below](#the-rustfs-and-rustfs-tls-runtime-fips-feature)).
+- **The object data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations, even when `rustfs` is built with `--features fips`. Enabling the TLS FIPS provider does not change the object encryption data path; see route 1 below.
 
 ## Terminology red lines for external material
 
@@ -50,6 +50,15 @@ Boilerplate when the topic cannot be avoided:
 
 `scripts/check_fips_wording.sh` greps `README.md` and `CHANGELOG.md` for the banned phrases above, and separately rejects any wording under `crates/kms` that describes the Vault KV2 backend as wrapping key material through Vault's Transit engine (`KmsBackend::VaultKv2` stores RustFS-wrapped material in KV v2 and never calls Transit; use `VaultTransit` when cryptographic isolation is the requirement). This document is intentionally outside the scan: it needs the terminology to define the policy.
 
+## The `rustfs` and `rustfs-tls-runtime` `fips` feature
+
+`rustfs/Cargo.toml` (`fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]`) and `crates/tls-runtime/Cargo.toml` (`fips = ["rustls/fips"]`) define an opt-in Cargo feature (disabled by default) for builds that require the `aws-lc-fips-sys` cryptographic module on the `aws-lc-rs` / `rustls` path:
+
+- **What it enables:** Activates `rustls/fips` and `tokio-rustls/fips`, which pull in `aws-lc-rs/fips` (`aws-lc-fips-sys`) and `rustls-webpki/aws-lc-rs-fips`. Because Cargo unifies features across the dependency graph, other `aws-lc-rs` consumers in the binary (`jsonwebtoken`, `rcgen`, `google-cloud-auth`, `async-nats`) also link against `aws-lc-fips-sys`.
+- **Runtime provider installation:** In `rustfs/src/startup_runtime_hooks.rs`, `build_default_crypto_provider()` selects `rustls::crypto::default_fips_provider()` (instead of `rustls::crypto::aws_lc_rs::default_provider()`) and asserts `provider.fips()` on startup, restricting TLS negotiation to FIPS-approved cipher suites (e.g., `TLS_AES_256_GCM_SHA384`, `TLS_AES_128_GCM_SHA256`, `TLS_ECDHE_*_AES_*_GCM_*`) and key exchange groups (excluding `TLS_CHACHA20_POLY1305_SHA256` and `X25519`).
+- **Build requirements:** Compiling `aws-lc-fips-sys` requires CMake, Go, and Clang/`libclang` in the build environment (unlike the default `aws-lc-sys` build).
+- **What it does not change:** It does not alter the object data-path AEAD (`crates/kms`, `crates/rio`, `crates/rio-v2`) or `rustfs-crypto` config/IAM blob encryption, which continue to use pure-Rust `RustCrypto` crates.
+
 ## The `rustfs-crypto` `fips` feature: what it actually does
 
 `crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
@@ -71,7 +80,7 @@ Selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src
 
 | Route | Scope | What it buys | Verdict |
 | --- | --- | --- | --- |
-| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) | The TLS provider swap is the small part. Every AEAD call in the data path (`crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, `crates/crypto/src/encdec/`) would be re-implemented against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Heavy; re-opens an open platform-support problem. Justified only by a named customer or regulatory commitment |
+| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) across the data path | The TLS/X.509/JWT provider swap is wired via the opt-in `-p rustfs --features fips` build flag. Extending `aws-lc-fips-sys` to the data path would require re-implementing every AEAD/KDF call in `crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, Clang/`libclang`, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Data-path rewrite remains heavy and re-opens platform-support constraints; TLS/X.509/JWT is opt-in via `--features fips` |
 | 2. Let an externally validated KMS carry key operations | Mostly built: the `VaultTransit` backend never lets key-encryption key material leave Vault, so master key generation, wrapping, unwrapping, and rotation happen inside whatever module Vault's seal/HSM is validated against. Remaining work is configuration guidance and a supported-deployment statement | A defensible partial answer to "where do keys live and who validated that". The object data path stays RustCrypto and TLS stays non-FIPS AWS-LC: "key management operations are performed by an externally validated module; the object data path is not validated" | Nearest partial step, no code rewrite. Point customers here when the requirement is key custody rather than a certificate covering the storage layer |
 | 3. Make no validation claim (current default) | This document plus the grep guard | Nothing further | The current decision. FIPS 140-3 validation is not a roadmap target; adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI |
 

rustfs/Cargo.toml

@@ -58,6 +58,7 @@ required-features = ["swift"]
 
 [features]
 default = ["ftps", "webdav", "gcs"]
+fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]
 gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]

rustfs/src/startup_runtime_hooks.rs

@@ -14,7 +14,10 @@
 
 use crate::license::license_status;
 use crate::startup_runtime_sources;
+#[cfg(not(feature = "fips"))]
 use rustls::crypto::aws_lc_rs::default_provider;
+#[cfg(feature = "fips")]
+use rustls::crypto::default_fips_provider as default_provider;
 use std::future::Future;
 use std::io::Result;
 use tracing::{debug, info, warn};
@@ -105,8 +108,18 @@ where
     shutdown();
 }
 
+fn build_default_crypto_provider() -> rustls::crypto::CryptoProvider {
+    let provider = default_provider();
+    #[cfg(feature = "fips")]
+    assert!(
+        provider.fips(),
+        "FIPS crypto provider requested via `fips` feature, but AWS-LC FIPS mode is not active"
+    );
+    provider
+}
+
 pub(crate) fn install_default_crypto_provider() {
-    if default_provider().install_default().is_err() {
+    if build_default_crypto_provider().install_default().is_err() {
         debug!(
             target: "rustfs::main",
             event = EVENT_CRYPTO_PROVIDER_STATE,
@@ -155,7 +168,7 @@ pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<
 }
 
 fn install_embedded_default_crypto_provider() {
-    if let Err(err) = default_provider().install_default() {
+    if let Err(err) = build_default_crypto_provider().install_default() {
         debug!(
             component = LOG_COMPONENT_EMBEDDED,
             subsystem = LOG_SUBSYSTEM_EMBEDDED,
@@ -169,12 +182,18 @@ fn install_embedded_default_crypto_provider() {
 
 #[cfg(test)]
 mod tests {
-    use super::{init_profiling_runtime_with, shutdown_profiling_runtime_with};
+    use super::{build_default_crypto_provider, init_profiling_runtime_with, shutdown_profiling_runtime_with};
     use std::sync::{
         Arc,
         atomic::{AtomicBool, Ordering},
     };
 
+    #[test]
+    fn default_crypto_provider_matches_fips_feature() {
+        let provider = build_default_crypto_provider();
+        assert_eq!(provider.fips(), cfg!(feature = "fips"));
+    }
+
     #[tokio::test]
     async fn init_profiling_runtime_invokes_registered_hook() {
         let called = Arc::new(AtomicBool::new(false));

scripts/test/oidc_keycloak_live.sh

@@ -8,6 +8,7 @@ KEYCLOAK_IMAGE="${KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:6a7217a100bd3
 WORK_DIR="${RUNNER_TEMP:-/tmp}/rustfs-keycloak-live-${$}"
 KEYCLOAK_CONTAINER="rustfs-keycloak-live-${$}"
 RUSTFS_PID=""
+DOCKER=(docker)
 mkdir -p "${WORK_DIR}"
 
 cleanup() {
@@ -17,10 +18,10 @@ cleanup() {
     wait "${RUSTFS_PID}" 2>/dev/null || true
   fi
   if [[ "${status}" -ne 0 ]]; then
-    docker logs "${KEYCLOAK_CONTAINER}" >"${WORK_DIR}/keycloak.log" 2>&1 || true
+    "${DOCKER[@]}" logs "${KEYCLOAK_CONTAINER}" >"${WORK_DIR}/keycloak.log" 2>&1 || true
     echo "live-gate logs retained in ${WORK_DIR}" >&2
   fi
-  docker rm -f "${KEYCLOAK_CONTAINER}" >/dev/null 2>&1 || true
+  "${DOCKER[@]}" rm -f "${KEYCLOAK_CONTAINER}" >/dev/null 2>&1 || true
   if [[ "${status}" -eq 0 ]]; then
     rm -rf "${WORK_DIR}"
   fi
@@ -33,6 +34,10 @@ for command in curl docker python3 aws; do
     exit 1
   }
 done
+if ! docker info >/dev/null 2>&1; then
+  DOCKER=(sudo -n docker)
+  "${DOCKER[@]}" info >/dev/null
+fi
 [[ -x "${RUSTFS_BINARY}" ]] || {
   echo "RustFS binary is not executable: ${RUSTFS_BINARY}" >&2
   exit 1
@@ -59,7 +64,7 @@ ISSUER="${KEYCLOAK_ORIGIN}/realms/rustfs-ci"
 DISCOVERY_URL="${ISSUER}/.well-known/openid-configuration"
 RUSTFS_ORIGIN="http://127.0.0.1:${RUSTFS_PORT}"
 
-docker run --detach --rm \
+"${DOCKER[@]}" run --detach --rm \
   --name "${KEYCLOAK_CONTAINER}" \
   --memory 1g \
   --publish "127.0.0.1:${KEYCLOAK_PORT}:8080" \
@@ -74,8 +79,8 @@ for _ in $(seq 1 120); do
   if curl --noproxy '*' -fsS "${DISCOVERY_URL}" >"${WORK_DIR}/discovery.json" 2>/dev/null; then
     break
   fi
-  if [[ "$(docker inspect -f '{{.State.Running}}' "${KEYCLOAK_CONTAINER}" 2>/dev/null || true)" != "true" ]]; then
-    docker logs "${KEYCLOAK_CONTAINER}" >&2 || true
+  if [[ "$("${DOCKER[@]}" inspect -f '{{.State.Running}}' "${KEYCLOAK_CONTAINER}" 2>/dev/null || true)" != "true" ]]; then
+    "${DOCKER[@]}" logs "${KEYCLOAK_CONTAINER}" >&2 || true
     echo "Keycloak exited before discovery became ready" >&2
     exit 1
   fi

scripts/test_security_workflow.py

@@ -261,6 +261,47 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                 if missing:
                     self.assertIn(f"missing required STS field: {missing}", result.stderr)
 
+    def test_oidc_docker_access_and_cleanup_use_the_same_privileges(self):
+        tools = self.directory / "docker-tools"
+        tools.mkdir()
+        (tools / "docker").write_text(
+            '#!/bin/bash\n'
+            'printf "%s %s\\n" "${DOCKER_PRIVILEGED:-no}" "$*" >> "$DOCKER_CALLS"\n'
+            'case "$1" in\n'
+            '  info) [ "$DOCKER_MODE" = direct ] || '
+            '{ [ "$DOCKER_MODE" = sudo ] && [ "${DOCKER_PRIVILEGED:-no}" = yes ]; } ;;\n'
+            '  run) exit 43 ;;\n'
+            '  *) exit 0 ;;\n'
+            'esac\n'
+        )
+        (tools / "sudo").write_text(
+            '#!/bin/bash\n[ "$1" = -n ] || exit 92\nshift\n'
+            'export DOCKER_PRIVILEGED=yes\nexec "$@"\n'
+        )
+        for name in ("docker", "sudo"):
+            (tools / name).chmod(0o755)
+        (tools / "python3").symlink_to(sys.executable)
+        for name in ("aws", "curl"):
+            (tools / name).symlink_to("/usr/bin/true")
+        for mode in ("direct", "sudo", "unavailable"):
+            with self.subTest(mode=mode):
+                calls = self.directory / f"docker-{mode}.log"
+                result = subprocess.run(
+                    ["bash", str(ROOT / "scripts/test/oidc_keycloak_live.sh"), "/usr/bin/true"],
+                    env={**os.environ, "PATH": f"{tools}:/usr/bin:/bin", "RUNNER_TEMP": str(self.directory),
+                         "DOCKER_MODE": mode, "DOCKER_CALLS": str(calls)},
+                    capture_output=True, text=True, timeout=10,
+                )
+                log = calls.read_text()
+                if mode == "unavailable":
+                    self.assertNotEqual(result.returncode, 0)
+                    self.assertNotIn(" run ", log)
+                else:
+                    self.assertEqual(result.returncode, 43, result.stderr)
+                    privilege = "yes" if mode == "sudo" else "no"
+                    for command in ("run", "logs", "rm -f"):
+                        self.assertIn(f"{privilege} {command} ", log)
+
     def test_existing_evidence_directory_is_rejected(self) -> None:
         for suffix in ("", "-scratch"):
             self.setUp()