fix(capture): validate checkpoint cut against captured committed log

Co-authored-by: Hauser <housemecn@gmail.com>

.config/ecstore-required-tests.json

@@ -161,6 +161,11 @@
       "suite": "rustfs-ecstore",
       "name": "set_disk::core::metadata_quorum::tests::metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction"
     },
+    {
+      "invariant": "metadata-observation-mixed-current-identity-arrivals-preserve-selection",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::core::metadata_quorum::tests::metadata_observation_mixed_current_identity_arrivals_preserve_selection"
+    },
     {
       "invariant": "metadata-observation-pending-newer-version-and-same-time-different-directory-force-full-wait",
       "suite": "rustfs-ecstore",
@@ -186,6 +191,11 @@
       "suite": "rustfs-ecstore",
       "name": "set_disk::read::metadata_cache_tests::late_metadata_refresh_rejects_changed_generation_and_mapped_shards"
     },
+    {
+      "invariant": "late-full-get-rejects-changed-object-generation",
+      "suite": "rustfs-ecstore",
+      "name": "set_disk::prepared_get_object_metadata_tests::non_inline_two_phase_get_rejects_generation_change_before_late_refresh"
+    },
     {
       "invariant": "list-objects-shared-corpus-static-namespace",
       "suite": "rustfs-ecstore",

.config/nextest.toml

@@ -92,6 +92,12 @@ setup = 'ecstore-large-stack'
 filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
 
+# These handler/store fixtures and the relocated-pool GET compose deep storage
+# futures in libtest threads; on Linux they overflow the default 2 MiB stack.
+[[profile.default.scripts]]
+filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
+setup = 'ecstore-base-stack'
+
 [[profile.default.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
@@ -337,6 +343,10 @@ setup = 'ecstore-large-stack'
 filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'
 
+[[profile.ci.scripts]]
+filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
+setup = 'ecstore-base-stack'
+
 [[profile.ci.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'

.github/workflows/build.yml

@@ -440,6 +440,17 @@ jobs:
             fi
           done
 
+      - name: Verify macOS binary has no dynamic liblzma dependency
+        if: matrix.platform == 'macos'
+        shell: bash
+        run: |
+          binary="target/${{ matrix.target }}/release/rustfs"
+          if otool -L "$binary" | grep -E 'liblzma.*\.dylib'; then
+            echo "macOS release binary must not depend on a dynamic liblzma library" >&2
+            exit 1
+          fi
+          "$binary" --version
+
       - name: Generate the CPU symbol catalog
         if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
         shell: bash

.github/workflows/e2e-distributed.yml

@@ -168,11 +168,13 @@ jobs:
           FILTER: ${{ inputs.filter }}
         run: |
           set -euo pipefail
+          rm -f target/nextest/e2e-distributed/junit.xml
           if [ -n "${FILTER}" ]; then
             python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
           else
             python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
           fi
+          test -s target/nextest/e2e-distributed/junit.xml
 
       - name: Upload distributed e2e diagnostics
         if: always()

Cargo.lock

@@ -275,6 +275,15 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "anyerror"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71add24cc141a1e8326f249b74c41cfd217aeb2a67c9c6cf9134d175469afd49"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.104"
@@ -1706,6 +1715,30 @@ dependencies = [
  "syn 3.0.6",
 ]
 
+[[package]]
+name = "borsh"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
+dependencies = [
+ "borsh-derive",
+ "bytes",
+ "cfg_aliases",
+]
+
+[[package]]
+name = "borsh-derive"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
+dependencies = [
+ "once_cell",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.6",
+]
+
 [[package]]
 name = "brotli"
 version = "8.0.4"
@@ -1772,6 +1805,18 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "byte-unit"
+version = "5.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c719d56f7e96194cfc53460976d3ba51c85719747c9c62ed99981847b551152b"
+dependencies = [
+ "rust_decimal",
+ "schemars 1.2.2",
+ "serde",
+ "utf8-width",
+]
+
 [[package]]
 name = "bytemuck"
 version = "1.25.2"
@@ -3724,13 +3769,34 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "derive_more"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
+dependencies = [
+ "derive_more-impl 1.0.0",
+]
+
 [[package]]
 name = "derive_more"
 version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
 dependencies = [
- "derive_more-impl",
+ "derive_more-impl 2.1.1",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+ "unicode-xid",
 ]
 
 [[package]]
@@ -6234,7 +6300,7 @@ dependencies = [
  "bytes",
  "chrono",
  "dashmap",
- "derive_more",
+ "derive_more 2.1.1",
  "futures-util",
  "getrandom 0.4.3",
  "lazy_static",
@@ -6388,6 +6454,12 @@ dependencies = [
  "quote",
 ]
 
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
 [[package]]
 name = "matchers"
 version = "0.2.0"
@@ -7329,6 +7401,42 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "openraft"
+version = "0.9.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97014fb78acb77be3a40ac2da305f6dd3a6b243f3a908ace87d29b3972eaafd"
+dependencies = [
+ "anyerror",
+ "byte-unit",
+ "chrono",
+ "clap",
+ "derive_more 1.0.0",
+ "futures",
+ "maplit",
+ "openraft-macros",
+ "rand 0.8.8",
+ "serde",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+ "tracing-futures",
+ "validit",
+]
+
+[[package]]
+name = "openraft-macros"
+version = "0.9.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55417d79f46a787dd88c043d831d8b7a0713f7c69d1925f9de51cc42c437250f"
+dependencies = [
+ "chrono",
+ "proc-macro2",
+ "quote",
+ "semver",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -8958,6 +9066,15 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "redb"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb338a6c67830a61bed824b78c2bb034ab1ff401a20616bd7957524f4fdc2f22"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "redis"
 version = "1.7.1"
@@ -9464,6 +9581,23 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "rust_decimal"
+version = "1.43.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a"
+dependencies = [
+ "arrayvec",
+ "borsh",
+ "bytes",
+ "num-traits",
+ "rand 0.8.8",
+ "rand 0.9.5",
+ "serde",
+ "serde_json",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.28"
@@ -9666,6 +9800,21 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "rustfs-capture-authority"
+version = "1.0.1"
+dependencies = [
+ "openraft",
+ "redb",
+ "serde",
+ "serde_json",
+ "sha2 0.11.0",
+ "tempfile",
+ "thiserror 2.0.21",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "rustfs-checksums"
 version = "1.0.1"
@@ -10473,6 +10622,7 @@ dependencies = [
  "tokio",
  "tracing",
  "tracing-subscriber",
+ "uuid",
 ]
 
 [[package]]
@@ -10594,6 +10744,7 @@ dependencies = [
  "faster-hex",
  "futures",
  "hex-simd",
+ "hmac 0.13.0",
  "hotpath",
  "http 1.5.0",
  "http-body-util",
@@ -11060,6 +11211,7 @@ dependencies = [
  "async-compression",
  "futures",
  "hotpath",
+ "liblzma",
  "rustfs-rio",
  "rustfs-tokio-tar",
  "serde",
@@ -13044,6 +13196,16 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tracing-futures"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
+dependencies = [
+ "pin-project",
+ "tracing",
+]
+
 [[package]]
 name = "tracing-log"
 version = "0.2.0"
@@ -13194,7 +13356,7 @@ checksum = "79acf4b257aa56dd7f00466a81693d6c79c362c55092b3137909d6e76249ad80"
 dependencies = [
  "async-trait",
  "chrono",
- "derive_more",
+ "derive_more 2.1.1",
  "libc",
  "md-5 0.10.6",
  "thiserror 2.0.21",
@@ -13294,6 +13456,12 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
+[[package]]
+name = "utf8-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a"
+
 [[package]]
 name = "utf8_iter"
 version = "1.0.4"
@@ -13320,6 +13488,15 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "validit"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4417835826b78aa6265724624d424df2b1e6e71a33329227d8b04a88950780dc"
+dependencies = [
+ "anyerror",
+]
+
 [[package]]
 name = "valuable"
 version = "0.1.1"

Cargo.toml

@@ -18,6 +18,7 @@ members = [
     "crates/audit", # Audit target management system with multi-target fan-out
     "crates/checksums", # client checksums
     "crates/common", # Shared utilities and data structures
+    "crates/capture-authority", # Durable capture authority storage
     "crates/data-usage", # Shared data usage models and algorithms
     "crates/config", # Configuration management
     "crates/connect-symbol-catalog", # Release-only CPU profile symbol catalog generator
@@ -314,6 +315,8 @@ ratelimit = "2.0.0"
 rayon = "1.12.0"
 rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
+openraft = { version = "=0.9.25", default-features = false, features = ["serde"] }
+redb = "=4.3.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
 redis = { version = "1.7.1" }

crates/capture-authority/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "rustfs-capture-authority"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+description = "Durable local storage for capture authority"
+
+[dependencies]
+openraft = { workspace = true }
+redb = { workspace = true }
+sha2 = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+url = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true }
+tempfile = { workspace = true }
+
+[lints]
+workspace = true

crates/capture-authority/src/application_tests.rs

@@ -0,0 +1,531 @@
+use super::*;
+use crate::application::{APPLY_RESULTS, EVENTS, OPERATIONS, VERSIONS};
+use crate::tests::CrashBackend;
+use redb::ReadableTableMetadata;
+use std::{
+    io::{BufRead, BufReader, Write},
+    process::{Command, Stdio},
+    sync::mpsc,
+    time::Duration,
+};
+fn identity() -> NodeIdentity {
+    NodeIdentity {
+        cluster_id: [7; 16],
+        node_id: 0,
+    }
+}
+fn binding() -> CaptureBinding {
+    CaptureBinding {
+        revision: 9,
+        binding_id: [8; 16],
+        target_id: [9; 16],
+    }
+}
+fn command() -> DecideCreated {
+    DecideCreated {
+        operation_id: [1; 16],
+        object: ObjectIdentity {
+            bucket_incarnation: [2; 16],
+            key: vec![0, 255, 3],
+        },
+        expected_head: None,
+        prepared: PreparedIdentity {
+            version_id: [3; 16],
+            preparation_id: [4; 16],
+            content_digest: [5; 32],
+            content_length: 123,
+        },
+        binding_revision: 9,
+    }
+}
+fn id(index: u64) -> LogId {
+    LogId {
+        term: 8,
+        leader_node: 0,
+        index,
+    }
+}
+fn append(store: &mut Store, index: u64, c: &DecideCreated) {
+    store
+        .append(&[LogEntry {
+            id: id(index),
+            payload: c.encode().expect("encode command"),
+        }])
+        .expect("append command");
+}
+fn apply(store: &mut Store, index: u64, c: &DecideCreated) -> CreatedResult {
+    append(store, index, c);
+    store.save_committed(id(index)).expect("commit log");
+    store.apply_next_created(id(index)).expect("apply command")
+}
+fn success(c: &DecideCreated, index: u64) -> CreatedResult {
+    CreatedResult::Created {
+        version_id: c.prepared.version_id,
+        event_id: c.operation_id,
+        decision_log_id: id(index),
+    }
+}
+fn complete(store: &Store) {
+    let c = command();
+    assert_eq!(store.capture_binding().expect("binding"), Some(binding()));
+    assert_eq!(
+        store.created_head(&c.object).expect("head"),
+        Some(c.prepared.version_id),
+        "U11 acknowledged head lost"
+    );
+    assert_eq!(
+        store.created_version(&c.object, c.prepared.version_id).expect("version"),
+        Some(CreatedVersion {
+            prepared: c.prepared.clone(),
+            decision_log_id: id(0),
+            binding_revision: 9
+        })
+    );
+    assert_eq!(
+        store.created_event(c.operation_id).expect("event"),
+        Some(CreatedEvent {
+            object: c.object.clone(),
+            prepared: c.prepared.clone(),
+            decision_log_id: id(0),
+            binding: binding()
+        }),
+        "U11 complete event missing"
+    );
+    assert_eq!(
+        store.created_operation(c.operation_id).expect("operation"),
+        Some(CreatedOperation {
+            canonical_command: c.encode().expect("encode"),
+            result: success(&c, 0)
+        })
+    );
+    assert_eq!(
+        store.created_apply_result(0).expect("result"),
+        Some(CreatedApplyResult {
+            log_id: id(0),
+            result: success(&c, 0)
+        })
+    );
+    assert_eq!(store.last_applied().expect("last applied"), Some(id(0)));
+}
+fn fixture() -> (tempfile::TempDir, Store) {
+    let dir = tempfile::tempdir().expect("tempdir");
+    let mut store = Store::initialize(dir.path().join("db"), identity()).expect("initialize");
+    store.pin_capture_binding(binding()).expect("pin");
+    (dir, store)
+}
+fn reopen(path: &std::path::Path) -> Store {
+    let result = Store::open(path, identity());
+    assert!(result.is_ok(), "U11 complete reopen: {:?}", result.as_ref().err());
+    result.expect("asserted successful reopen")
+}
+#[test]
+fn created_codec_is_canonical_and_strict() {
+    let c = command();
+    let b = c.encode().expect("encode");
+    let mut golden = b"CRTD\x01".to_vec();
+    golden.extend([1; 16]);
+    golden.extend([2; 16]);
+    golden.extend(3u32.to_be_bytes());
+    golden.extend([0, 255, 3]);
+    golden.push(0);
+    golden.extend([3; 16]);
+    golden.extend([4; 16]);
+    golden.extend([5; 32]);
+    golden.extend(123u64.to_be_bytes());
+    golden.extend(9u64.to_be_bytes());
+    assert_eq!(b, golden);
+    assert_eq!(DecideCreated::decode(&b).expect("decode"), c);
+    let mut variants = Vec::new();
+    let mut v = c.clone();
+    v.operation_id = [6; 16];
+    variants.push(v);
+    let mut v = c.clone();
+    v.object.bucket_incarnation = [6; 16];
+    variants.push(v);
+    let mut v = c.clone();
+    v.object.key.push(4);
+    variants.push(v);
+    let mut v = c.clone();
+    v.expected_head = Some([6; 16]);
+    variants.push(v);
+    let mut v = c.clone();
+    v.prepared.version_id = [6; 16];
+    variants.push(v);
+    let mut v = c.clone();
+    v.prepared.preparation_id = [6; 16];
+    variants.push(v);
+    let mut v = c.clone();
+    v.prepared.content_digest = [6; 32];
+    variants.push(v);
+    let mut v = c.clone();
+    v.prepared.content_length += 1;
+    variants.push(v);
+    let mut v = c.clone();
+    v.binding_revision += 1;
+    variants.push(v);
+    for v in variants {
+        assert_ne!(v.encode().expect("variant"), b)
+    }
+    for n in 0..b.len() {
+        assert!(DecideCreated::decode(&b[..n]).is_err())
+    }
+    let mut bad = b.clone();
+    bad[4] = 2;
+    assert!(DecideCreated::decode(&bad).is_err());
+    let mut bad = b.clone();
+    bad[44] = 2;
+    assert!(DecideCreated::decode(&bad).is_err());
+    let mut bad = b.clone();
+    bad.push(0);
+    assert!(DecideCreated::decode(&bad).is_err());
+    let mut bad = b;
+    bad[37..41].copy_from_slice(&4097u32.to_be_bytes());
+    assert!(DecideCreated::decode(&bad).is_err());
+    let mut bad = c.clone();
+    bad.operation_id = [0; 16];
+    assert!(bad.encode().is_err());
+    bad = c;
+    bad.object.key = vec![0; MAX_OBJECT_KEY_LENGTH + 1];
+    assert!(bad.encode().is_err());
+}
+#[test]
+fn created_committed_order_and_identity_are_enforced() {
+    let (_dir, mut s) = fixture();
+    let c = command();
+    append(&mut s, 0, &c);
+    append(&mut s, 1, &c);
+    assert!(s.apply_next_created(id(0)).is_err());
+    assert_eq!(s.last_applied().expect("no apply"), None);
+    s.save_committed(id(1)).expect("committed");
+    assert!(matches!(s.apply_next_created(id(1)), Err(StoreError::InvalidApplySequence)));
+    let mut wrong = id(0);
+    wrong.term += 1;
+    assert!(s.apply_next_created(wrong).is_err());
+    wrong = id(0);
+    wrong.leader_node += 1;
+    assert!(s.apply_next_created(wrong).is_err());
+    assert_eq!(s.apply_next_created(id(0)).expect("first"), success(&c, 0));
+    complete(&s);
+    assert_eq!(s.apply_next_created(id(0)).expect("retry"), success(&c, 0));
+    complete(&s);
+    assert!(s.apply_next_created(wrong).is_err());
+    assert_eq!(s.apply_next_created(id(1)).expect("duplicate log"), success(&c, 0));
+    assert_eq!(s.last_applied().expect("last"), Some(id(1)));
+    assert_eq!(
+        s.created_apply_result(1).expect("result"),
+        Some(CreatedApplyResult {
+            log_id: id(1),
+            result: success(&c, 0)
+        })
+    );
+    let tx = s.database.begin_read().expect("read");
+    assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 1);
+}
+#[test]
+fn created_head_version_event_and_dedup_reopen_together() {
+    let (dir, mut s) = fixture();
+    let c = command();
+    assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
+    drop(s);
+    let mut s = reopen(&dir.path().join("db"));
+    complete(&s);
+    assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
+    let mut changed = c.clone();
+    changed.prepared.content_digest = [6; 32];
+    assert_eq!(
+        apply(&mut s, 2, &changed),
+        CreatedResult::OperationConflict,
+        "U11 full canonical operation conflict"
+    );
+    assert_eq!(
+        s.created_operation(c.operation_id)
+            .expect("original")
+            .expect("operation")
+            .result,
+        success(&c, 0)
+    );
+    let mut stale = c.clone();
+    stale.operation_id = [6; 16];
+    stale.prepared.version_id = [6; 16];
+    assert_eq!(
+        apply(&mut s, 3, &stale),
+        CreatedResult::HeadMismatch {
+            actual_head: Some(c.prepared.version_id)
+        }
+    );
+    let mut reuse = c.clone();
+    reuse.operation_id = [7; 16];
+    reuse.expected_head = Some(c.prepared.version_id);
+    assert_eq!(apply(&mut s, 4, &reuse), CreatedResult::VersionConflict);
+    let mut replacement = stale.clone();
+    replacement.operation_id = [10; 16];
+    replacement.expected_head = Some(c.prepared.version_id);
+    assert_eq!(apply(&mut s, 5, &replacement), success(&replacement, 5));
+    assert_eq!(s.created_head(&c.object).expect("head"), Some(replacement.prepared.version_id));
+    assert_eq!(
+        s.created_version(&c.object, c.prepared.version_id)
+            .expect("old version")
+            .expect("version")
+            .decision_log_id,
+        id(0)
+    );
+    assert!(s.created_event(c.operation_id).expect("old event").is_some());
+    assert!(s.created_event(replacement.operation_id).expect("new event").is_some());
+    assert!(s.created_event(stale.operation_id).expect("rejection event").is_none());
+    assert_eq!(
+        apply(&mut s, 6, &stale),
+        CreatedResult::HeadMismatch {
+            actual_head: Some(c.prepared.version_id)
+        }
+    );
+    drop(s);
+    let s = reopen(&dir.path().join("db"));
+    assert_eq!(s.last_applied().expect("last"), Some(id(6)));
+    let tx = s.database.begin_read().expect("read");
+    assert_eq!(tx.open_table(EVENTS).expect("events").len().expect("len"), 2);
+}
+#[test]
+fn created_binding_is_pinned_and_not_caller_selected() {
+    let (dir, mut s) = fixture();
+    let mut bad = command();
+    bad.binding_revision = 10;
+    assert_eq!(apply(&mut s, 0, &bad), CreatedResult::BindingMismatch);
+    assert!(s.created_head(&bad.object).expect("head").is_none());
+    assert!(s.created_event(bad.operation_id).expect("event").is_none());
+    assert_eq!(
+        s.created_operation(bad.operation_id)
+            .expect("operation")
+            .expect("saved")
+            .result,
+        CreatedResult::BindingMismatch
+    );
+    s.pin_capture_binding(binding()).expect("idempotent pin");
+    let mut changed = binding();
+    changed.target_id = [10; 16];
+    assert!(matches!(s.pin_capture_binding(changed), Err(StoreError::BindingConflict)));
+    drop(s);
+    let s = reopen(&dir.path().join("db"));
+    assert_eq!(s.capture_binding().expect("binding"), Some(binding()));
+}
+#[test]
+fn created_schema_compatibility_and_corruption_are_checked() {
+    let dir = tempfile::tempdir().expect("dir");
+    let path = dir.path().join("db");
+    let s = Store::initialize(&path, identity()).expect("schema1");
+    assert_eq!(s.capture_binding().expect("unmigrated"), None);
+    drop(s);
+    let mut s = reopen(&path);
+    assert_eq!(s.last_applied().expect("no apply"), None);
+    append(&mut s, 0, &command());
+    s.save_committed(id(0)).expect("schema2");
+    drop(s);
+    let mut s = reopen(&path);
+    assert_eq!(s.capture_binding().expect("schema2 unchanged"), None);
+    s.pin_capture_binding(binding()).expect("explicit upgrade");
+    drop(s);
+    let mut s = reopen(&path);
+    assert_eq!(s.capture_binding().expect("schema3"), Some(binding()));
+    s.apply_next_created(id(0)).expect("apply");
+    drop(s);
+    let db = Database::open(&path).expect("raw db");
+    let tx = db.begin_write().expect("write corruption");
+    tx.open_table(EVENTS)
+        .expect("events")
+        .remove(command().operation_id.as_slice())
+        .expect("remove reference");
+    tx.commit().expect("commit corruption");
+    drop(db);
+    let result = Store::open(&path, identity());
+    assert!(
+        matches!(result, Err(StoreError::CorruptRecord)),
+        "U11 missing event rejected before expect"
+    );
+    let (dir, mut s) = fixture();
+    assert_eq!(s.read_committed().expect("empty committed"), None);
+    append(&mut s, 0, &command());
+    s.save_committed(id(0)).expect("committed schema3");
+    assert_eq!(s.capture_binding().expect("no downgrade"), Some(binding()));
+    drop(s);
+    let path = dir.path().join("db");
+    let db = Database::open(&path).expect("raw db");
+    let tx = db.begin_write().expect("write");
+    let mut marker = vec![1];
+    marker.extend(encode_id(id(0)));
+    tx.open_table(META)
+        .expect("meta")
+        .insert("last_applied", marker.as_slice())
+        .expect("corrupt applied");
+    tx.commit().expect("commit");
+    drop(db);
+    assert!(
+        matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
+        "U11 last applied missing result rejected"
+    );
+    let (dir, s) = fixture();
+    drop(s);
+    let path = dir.path().join("db");
+    let db = Database::open(&path).expect("raw pinned db");
+    let tx = db.begin_write().expect("missing binding write");
+    tx.open_table(META)
+        .expect("meta")
+        .remove("capture_binding")
+        .expect("remove binding");
+    tx.commit().expect("binding corruption commit");
+    drop(db);
+    assert!(
+        matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
+        "U11 missing pinned binding rejected"
+    );
+
+    let (dir, mut s) = fixture();
+    let c = command();
+    assert_eq!(apply(&mut s, 0, &c), success(&c, 0));
+    assert_eq!(apply(&mut s, 1, &c), success(&c, 0));
+    drop(s);
+    let path = dir.path().join("db");
+    let db = Database::open(&path).expect("raw duplicate db");
+    let tx = db.begin_write().expect("retarget decision write");
+    let mut key = created::object_bytes(&c.object).expect("object key");
+    key.extend(c.prepared.version_id);
+    let mut version = Vec::new();
+    created::put_prepared(&mut version, &c.prepared);
+    version.extend(encode_id(id(1)));
+    version.extend(binding().revision.to_be_bytes());
+    tx.open_table(VERSIONS)
+        .expect("versions")
+        .insert(key.as_slice(), version.as_slice())
+        .expect("retarget version");
+    let mut event = created::object_bytes(&c.object).expect("event object");
+    created::put_prepared(&mut event, &c.prepared);
+    event.extend(encode_id(id(1)));
+    event.extend(created::binding_bytes(&binding()).expect("binding bytes"));
+    tx.open_table(EVENTS)
+        .expect("events")
+        .insert(c.operation_id.as_slice(), event.as_slice())
+        .expect("retarget event");
+    let canonical = c.encode().expect("canonical");
+    let mut operation = u64::try_from(canonical.len()).expect("length").to_be_bytes().to_vec();
+    operation.extend(canonical);
+    operation.extend(created::result_bytes(&success(&c, 1)));
+    tx.open_table(OPERATIONS)
+        .expect("operations")
+        .insert(c.operation_id.as_slice(), operation.as_slice())
+        .expect("retarget original response");
+    for index in 0..=1 {
+        let mut result = encode_id(id(index));
+        result.extend(created::result_bytes(&success(&c, 1)));
+        tx.open_table(APPLY_RESULTS)
+            .expect("results")
+            .insert(index, result.as_slice())
+            .expect("retarget duplicate response");
+    }
+    tx.commit().expect("coherent retarget commit");
+    drop(db);
+    assert!(
+        matches!(Store::open(&path, identity()), Err(StoreError::CorruptRecord)),
+        "U11 original decision cannot retarget duplicate log"
+    );
+}
+#[test]
+fn created_process_child() {
+    if let Some(path) = std::env::var_os("U11_CHILD_DB") {
+        let mut s = Store::initialize(path, identity()).expect("child init");
+        s.pin_capture_binding(binding()).expect("child pin");
+        assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
+        complete(&s);
+        println!("U11_ACK");
+        std::io::stdout().flush().expect("flush ACK");
+        loop {
+            std::thread::park()
+        }
+    }
+    let (_dir, mut s) = fixture();
+    assert_eq!(apply(&mut s, 0, &command()), success(&command(), 0));
+    complete(&s);
+}
+#[test]
+fn created_acknowledged_apply_survives_child_kill() {
+    let dir = tempfile::tempdir().expect("dir");
+    let path = dir.path().join("db");
+    let mut child = Command::new(std::env::current_exe().expect("test executable"))
+        .args(["--exact", "application_tests::created_process_child", "--nocapture"])
+        .env("U11_CHILD_DB", &path)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .expect("spawn");
+    let stdout = child.stdout.take().expect("stdout");
+    let (tx, rx) = mpsc::channel();
+    let reader = std::thread::spawn(move || {
+        for line in BufReader::new(stdout).lines() {
+            let line = line.expect("child line");
+            if line.ends_with("U11_ACK") {
+                let _ = tx.send(());
+                break;
+            }
+        }
+    });
+    let acknowledged = rx.recv_timeout(Duration::from_secs(20));
+    child.kill().expect("kill child");
+    child.wait().expect("wait child");
+    reader.join().expect("reader cleanup");
+    assert!(acknowledged.is_ok(), "U11 bounded child ACK missing");
+    complete(&reopen(&path));
+}
+#[test]
+fn created_power_cut_before_and_after_apply_flush() {
+    let (dir, mut s) = fixture();
+    append(&mut s, 0, &command());
+    s.save_committed(id(0)).expect("committed");
+    drop(s);
+    let path = dir.path().join("db");
+    let backend = CrashBackend::new(&path);
+    let mut s = Store::from_database(backend.database(), identity()).expect("backend");
+    assert_eq!(s.apply_next_created(id(0)).expect("apply ACK"), success(&command(), 0));
+    backend.cut();
+    drop(s);
+    complete(&reopen(&path));
+    for sync in 1..=2 {
+        for after in [false, true] {
+            let (dir, mut s) = fixture();
+            append(&mut s, 0, &command());
+            s.save_committed(id(0)).expect("committed");
+            drop(s);
+            let path = dir.path().join("db");
+            let backend = CrashBackend::new(&path);
+            let mut s = Store::from_database(backend.database(), identity()).expect("backend");
+            let (entered, resume) = backend.pause_sync(sync, after);
+            let writer = std::thread::spawn(move || {
+                let result = s.apply_next_created(id(0));
+                (s, result)
+            });
+            entered.wait();
+            backend.cut();
+            resume.wait();
+            let (s, result) = writer.join().expect("writer");
+            drop(s);
+            let reopened = Store::open(&path, identity());
+            assert!(
+                reopened.is_ok(),
+                "U11 apply cut must reopen whole old/new state sync={sync} after={after}: {:?}",
+                reopened.as_ref().err()
+            );
+            let s = reopened.expect("asserted reopen");
+            if s.last_applied().expect("last").is_some() {
+                complete(&s)
+            } else {
+                assert!(result.is_err());
+                assert_eq!(s.created_head(&command().object).expect("old head"), None);
+                assert_eq!(
+                    s.created_version(&command().object, command().prepared.version_id)
+                        .expect("old version"),
+                    None
+                );
+                assert_eq!(s.created_event(command().operation_id).expect("old event"), None);
+                assert_eq!(s.created_operation(command().operation_id).expect("old op"), None);
+                assert_eq!(s.created_apply_result(0).expect("old result"), None)
+            }
+            eprintln!("U11 cut boundary sync={sync} after={after}");
+        }
+    }
+}

crates/capture-authority/src/committed.rs

@@ -0,0 +1,63 @@
+use crate::{LOGS, LogId, META, Store, StoreError, decode_id, decode_log, encode_id, number};
+use redb::{ReadableDatabase, ReadableTable};
+
+pub(crate) fn validate_committed(
+    meta: &impl ReadableTable<&'static str, &'static [u8]>,
+    logs: &impl ReadableTable<u64, &'static [u8]>,
+) -> Result<Option<LogId>, StoreError> {
+    let schema = number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?;
+    let marker = meta.get("committed")?;
+    match (schema, marker) {
+        (1 | 3 | 4 | 5 | 6, None) => {
+            if matches!(schema, 3..=6) && meta.get("last_applied")?.is_none_or(|v| v.value() != [0]) {
+                return Err(StoreError::CorruptRecord);
+            }
+            Ok(None)
+        }
+        (2 | 3 | 4 | 5 | 6, Some(marker)) => {
+            let id = decode_id(marker.value())?;
+            let record = logs.get(id.index)?.ok_or(StoreError::CorruptRecord)?;
+            if decode_log(record.value())?.id != id {
+                return Err(StoreError::CorruptRecord);
+            }
+            Ok(Some(id))
+        }
+        (1 | 2, _) => Err(StoreError::CorruptRecord),
+        _ => Err(StoreError::UnsupportedSchema),
+    }
+}
+
+impl Store {
+    /// Read the exact committed boundary, validating it against the stored log.
+    pub fn read_committed(&self) -> Result<Option<LogId>, StoreError> {
+        let tx = self.database.begin_read()?;
+        validate_committed(&tx.open_table(META)?, &tx.open_table(LOGS)?)
+    }
+
+    /// Persist a monotonic committed boundary supplied by the authority owner.
+    pub fn save_committed(&mut self, id: LogId) -> Result<(), StoreError> {
+        let tx = self.write_transaction()?;
+        {
+            let mut meta = tx.open_table(META)?;
+            let logs = tx.open_table(LOGS)?;
+            let previous = validate_committed(&meta, &logs)?;
+            let record = logs.get(id.index)?.ok_or(StoreError::InvalidCommittedLog)?;
+            if decode_log(record.value())?.id != id {
+                return Err(StoreError::InvalidCommittedLog);
+            }
+            if let Some(previous) = previous {
+                if id.index < previous.index {
+                    return Err(StoreError::CommittedRegression);
+                }
+                if id == previous {
+                    return Ok(());
+                }
+            }
+            meta.insert("committed", encode_id(id).as_slice())?;
+            if previous.is_none() && !matches!(number(meta.get("schema")?.ok_or(StoreError::CorruptRecord)?.value())?, 3..=6) {
+                meta.insert("schema", 2u64.to_be_bytes().as_slice())?;
+            }
+        }
+        self.commit(tx)
+    }
+}

crates/capture-authority/src/created.rs

@@ -0,0 +1,247 @@
+use crate::{LogId, StoreError, decode_id, encode_id};
+
+pub const MAX_OBJECT_KEY_LENGTH: usize = 4096;
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ObjectIdentity {
+    pub bucket_incarnation: [u8; 16],
+    pub key: Vec<u8>,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PreparedIdentity {
+    pub version_id: [u8; 16],
+    pub preparation_id: [u8; 16],
+    pub content_digest: [u8; 32],
+    pub content_length: u64,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecideCreated {
+    pub operation_id: [u8; 16],
+    pub object: ObjectIdentity,
+    pub expected_head: Option<[u8; 16]>,
+    pub prepared: PreparedIdentity,
+    pub binding_revision: u64,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CaptureBinding {
+    pub revision: u64,
+    pub binding_id: [u8; 16],
+    pub target_id: [u8; 16],
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CreatedResult {
+    Created {
+        version_id: [u8; 16],
+        event_id: [u8; 16],
+        decision_log_id: LogId,
+    },
+    HeadMismatch {
+        actual_head: Option<[u8; 16]>,
+    },
+    OperationConflict,
+    BindingMismatch,
+    VersionConflict,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreatedVersion {
+    pub prepared: PreparedIdentity,
+    pub decision_log_id: LogId,
+    pub binding_revision: u64,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreatedEvent {
+    pub object: ObjectIdentity,
+    pub prepared: PreparedIdentity,
+    pub decision_log_id: LogId,
+    pub binding: CaptureBinding,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreatedOperation {
+    pub canonical_command: Vec<u8>,
+    pub result: CreatedResult,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreatedOperationProvenance {
+    pub operation: CreatedOperation,
+    pub first_decision_log_id: LogId,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreatedApplyResult {
+    pub log_id: LogId,
+    pub result: CreatedResult,
+}
+
+pub(crate) struct Decoder<'a> {
+    bytes: &'a [u8],
+}
+impl<'a> Decoder<'a> {
+    pub(crate) fn new(bytes: &'a [u8]) -> Self {
+        Self { bytes }
+    }
+    pub(crate) fn take<const N: usize>(&mut self) -> Result<[u8; N], StoreError> {
+        let a = self
+            .bytes
+            .get(..N)
+            .ok_or(StoreError::CorruptRecord)?
+            .try_into()
+            .map_err(|_| StoreError::CorruptRecord)?;
+        self.bytes = &self.bytes[N..];
+        Ok(a)
+    }
+    pub(crate) fn u64(&mut self) -> Result<u64, StoreError> {
+        Ok(u64::from_be_bytes(self.take()?))
+    }
+    pub(crate) fn uuid(&mut self) -> Result<[u8; 16], StoreError> {
+        let id = self.take()?;
+        if id == [0; 16] {
+            return Err(StoreError::CorruptRecord);
+        }
+        Ok(id)
+    }
+    pub(crate) fn option(&mut self) -> Result<Option<[u8; 16]>, StoreError> {
+        match self.take::<1>()?[0] {
+            0 => Ok(None),
+            1 => Ok(Some(self.uuid()?)),
+            _ => Err(StoreError::CorruptRecord),
+        }
+    }
+    pub(crate) fn object(&mut self) -> Result<ObjectIdentity, StoreError> {
+        let bucket_incarnation = self.uuid()?;
+        let len = usize::try_from(u32::from_be_bytes(self.take()?)).map_err(|_| StoreError::CorruptRecord)?;
+        if len > MAX_OBJECT_KEY_LENGTH {
+            return Err(StoreError::CorruptRecord);
+        }
+        let key = self.bytes.get(..len).ok_or(StoreError::CorruptRecord)?.to_vec();
+        self.bytes = &self.bytes[len..];
+        Ok(ObjectIdentity { bucket_incarnation, key })
+    }
+    pub(crate) fn prepared(&mut self) -> Result<PreparedIdentity, StoreError> {
+        Ok(PreparedIdentity {
+            version_id: self.uuid()?,
+            preparation_id: self.uuid()?,
+            content_digest: self.take()?,
+            content_length: self.u64()?,
+        })
+    }
+    pub(crate) fn id(&mut self) -> Result<LogId, StoreError> {
+        decode_id(&self.take::<24>()?)
+    }
+    pub(crate) fn binding(&mut self) -> Result<CaptureBinding, StoreError> {
+        Ok(CaptureBinding {
+            revision: self.u64()?,
+            binding_id: self.uuid()?,
+            target_id: self.uuid()?,
+        })
+    }
+    pub(crate) fn finish(self) -> Result<(), StoreError> {
+        if self.bytes.is_empty() {
+            Ok(())
+        } else {
+            Err(StoreError::CorruptRecord)
+        }
+    }
+    pub(crate) fn result(&mut self) -> Result<CreatedResult, StoreError> {
+        match self.take::<1>()?[0] {
+            0 => Ok(CreatedResult::Created {
+                version_id: self.uuid()?,
+                event_id: self.uuid()?,
+                decision_log_id: self.id()?,
+            }),
+            1 => Ok(CreatedResult::HeadMismatch {
+                actual_head: self.option()?,
+            }),
+            2 => Ok(CreatedResult::OperationConflict),
+            3 => Ok(CreatedResult::BindingMismatch),
+            4 => Ok(CreatedResult::VersionConflict),
+            _ => Err(StoreError::CorruptRecord),
+        }
+    }
+}
+pub(crate) fn put_option(b: &mut Vec<u8>, id: Option<[u8; 16]>) {
+    b.push(u8::from(id.is_some()));
+    if let Some(id) = id {
+        b.extend(id)
+    }
+}
+pub(crate) fn object_bytes(o: &ObjectIdentity) -> Result<Vec<u8>, StoreError> {
+    if o.bucket_incarnation == [0; 16] || o.key.len() > MAX_OBJECT_KEY_LENGTH {
+        return Err(StoreError::CorruptRecord);
+    }
+    let capacity = 20usize.checked_add(o.key.len()).ok_or(StoreError::CorruptRecord)?;
+    let mut b = Vec::with_capacity(capacity);
+    b.extend(o.bucket_incarnation);
+    b.extend(
+        u32::try_from(o.key.len())
+            .map_err(|_| StoreError::CorruptRecord)?
+            .to_be_bytes(),
+    );
+    b.extend(&o.key);
+    Ok(b)
+}
+pub(crate) fn put_prepared(b: &mut Vec<u8>, p: &PreparedIdentity) {
+    b.extend(p.version_id);
+    b.extend(p.preparation_id);
+    b.extend(p.content_digest);
+    b.extend(p.content_length.to_be_bytes())
+}
+pub(crate) fn binding_bytes(v: &CaptureBinding) -> Result<Vec<u8>, StoreError> {
+    let mut b = v.revision.to_be_bytes().to_vec();
+    b.extend(v.binding_id);
+    b.extend(v.target_id);
+    let mut d = Decoder::new(&b);
+    d.binding()?;
+    d.finish()?;
+    Ok(b)
+}
+pub(crate) fn result_bytes(r: &CreatedResult) -> Vec<u8> {
+    let mut b = Vec::new();
+    match r {
+        CreatedResult::Created {
+            version_id,
+            event_id,
+            decision_log_id,
+        } => {
+            b.push(0);
+            b.extend(version_id);
+            b.extend(event_id);
+            b.extend(encode_id(*decision_log_id))
+        }
+        CreatedResult::HeadMismatch { actual_head } => {
+            b.push(1);
+            put_option(&mut b, *actual_head)
+        }
+        CreatedResult::OperationConflict => b.push(2),
+        CreatedResult::BindingMismatch => b.push(3),
+        CreatedResult::VersionConflict => b.push(4),
+    }
+    b
+}
+impl DecideCreated {
+    pub fn encode(&self) -> Result<Vec<u8>, StoreError> {
+        let object = object_bytes(&self.object)?;
+        let capacity = object.len().checked_add(134).ok_or(StoreError::CorruptRecord)?;
+        let mut b = Vec::with_capacity(capacity);
+        b.extend(b"CRTD\x01");
+        b.extend(self.operation_id);
+        b.extend(object);
+        put_option(&mut b, self.expected_head);
+        put_prepared(&mut b, &self.prepared);
+        b.extend(self.binding_revision.to_be_bytes());
+        Self::decode(&b)?;
+        Ok(b)
+    }
+    pub fn decode(bytes: &[u8]) -> Result<Self, StoreError> {
+        let mut d = Decoder::new(bytes);
+        if d.take::<5>()? != *b"CRTD\x01" {
+            return Err(StoreError::CorruptRecord);
+        }
+        let c = Self {
+            operation_id: d.uuid()?,
+            object: d.object()?,
+            expected_head: d.option()?,
+            prepared: d.prepared()?,
+            binding_revision: d.u64()?,
+        };
+        d.finish()?;
+        Ok(c)
+    }
+}

crates/capture-authority/src/native_snapshot.rs

@@ -0,0 +1,154 @@
+//! Owned consistent native application snapshot view.
+use crate::{Store, StoreError, application, snapshot};
+use redb::{ReadTransaction, ReadableDatabase, ReadableTable, ReadableTableMetadata};
+use std::{
+    io::Write,
+    sync::{Arc, Mutex},
+};
+
+#[cfg(test)]
+std::thread_local! {
+    static AFTER_CAPTURE: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const { std::cell::RefCell::new(None) };
+}
+
+#[cfg(test)]
+pub(crate) struct CaptureHook;
+#[cfg(test)]
+impl Drop for CaptureHook {
+    fn drop(&mut self) {
+        AFTER_CAPTURE.with(|hook| {
+            hook.borrow_mut().take();
+        });
+    }
+}
+#[cfg(test)]
+pub(crate) fn after_capture(hook: impl FnOnce() + 'static) -> CaptureHook {
+    AFTER_CAPTURE.with(|slot| {
+        assert!(slot.borrow().is_none(), "one capture hook per test thread");
+        *slot.borrow_mut() = Some(Box::new(hook));
+    });
+    CaptureHook
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum NativeSnapshotError {
+    #[error(transparent)]
+    Store(#[from] StoreError),
+    #[error(transparent)]
+    Snapshot(#[from] snapshot::SnapshotError),
+    #[error("native snapshot Store lock poisoned")]
+    Poisoned,
+    #[error("native snapshot worker failed")]
+    Worker(#[source] tokio::task::JoinError),
+}
+
+pub struct NativeSnapshotView {
+    // Drop the transaction before the owner so the Database stays open throughout the view.
+    transaction: ReadTransaction,
+    header: snapshot::Header,
+    counts: [u64; 4],
+    schema: u64,
+    _owner: Arc<Mutex<Store>>,
+}
+impl NativeSnapshotView {
+    pub(crate) fn acquire(owner: Arc<Mutex<Store>>) -> Result<Self, NativeSnapshotError> {
+        let transaction = {
+            let store = owner.lock().map_err(|_| NativeSnapshotError::Poisoned)?;
+            if !store.writable {
+                return Err(StoreError::WriteDisabled.into());
+            }
+            store.database.begin_read().map_err(StoreError::from)?
+        };
+        #[cfg(test)]
+        if let Some(hook) = AFTER_CAPTURE.with(|slot| slot.borrow_mut().take()) {
+            hook();
+        }
+        let (applied, membership) = application::native_application_state(&transaction)?;
+        let (header, schema) = {
+            let meta = transaction.open_table(crate::META).map_err(StoreError::from)?;
+            let logs = transaction.open_table(crate::LOGS).map_err(StoreError::from)?;
+            let committed = crate::committed::validate_committed(&meta, &logs)?;
+            if let Some(applied) = applied {
+                let committed = committed.ok_or(StoreError::CorruptRecord)?;
+                if applied.index > committed.index || (applied.index == committed.index && applied != committed) {
+                    return Err(StoreError::CorruptRecord.into());
+                }
+            }
+            let identity = meta
+                .get("identity")
+                .map_err(StoreError::from)?
+                .ok_or(StoreError::CorruptRecord)?;
+            let schema = crate::number(
+                meta.get("schema")
+                    .map_err(StoreError::from)?
+                    .ok_or(StoreError::CorruptRecord)?
+                    .value(),
+            )?;
+            (
+                snapshot::Header {
+                    cluster_id: crate::decode_identity(identity.value())?.cluster_id,
+                    applied,
+                    membership,
+                    binding: application::read_binding(&meta)?,
+                },
+                schema,
+            )
+        };
+        let mut counts = [0; 4];
+        for (index, definition) in [
+            application::HEADS,
+            application::VERSIONS,
+            application::EVENTS,
+            application::OPERATIONS,
+        ]
+        .into_iter()
+        .enumerate()
+        {
+            counts[index] = transaction
+                .open_table(definition)
+                .map_err(StoreError::from)?
+                .len()
+                .map_err(StoreError::from)?;
+        }
+        Ok(Self {
+            transaction,
+            header,
+            counts,
+            schema,
+            _owner: owner,
+        })
+    }
+
+    pub fn stream_capability_artifact<W: Write>(
+        self,
+        writer: W,
+        limits: snapshot::Limits,
+    ) -> Result<snapshot::CapabilityArtifact<W>, NativeSnapshotError> {
+        if self.schema != 6 {
+            return Err(StoreError::UnsupportedSchema.into());
+        }
+        let output = snapshot::CapabilityOutput::new(writer, limits.total_bytes);
+        Ok(self.stream(output, limits)?.finish())
+    }
+
+    pub fn stream<W: Write>(self, writer: W, limits: snapshot::Limits) -> Result<W, NativeSnapshotError> {
+        let mut encoder = snapshot::Encoder::new(writer, &self.header, self.counts, limits)?;
+        for (definition, section) in [
+            (application::HEADS, snapshot::Section::Heads),
+            (application::VERSIONS, snapshot::Section::Versions),
+            (application::EVENTS, snapshot::Section::Events),
+            (application::OPERATIONS, snapshot::Section::Operations),
+        ] {
+            let table = self.transaction.open_table(definition).map_err(StoreError::from)?;
+            for row in table.iter().map_err(StoreError::from)? {
+                let (key, value) = row.map_err(StoreError::from)?;
+                encoder.row(snapshot::Row {
+                    section,
+                    key: key.value(),
+                    value: value.value(),
+                })?;
+            }
+        }
+        Ok(encoder.finish()?)
+    }
+}

crates/capture-authority/src/node.rs

@@ -0,0 +1,371 @@
+//! Validated capture peer metadata and its independent CNOD v1 codec.
+use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeStruct};
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CaptureNode {
+    rpc_uri: String,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CaptureNodeError {
+    #[error("invalid RPC URI")]
+    InvalidRpcUri,
+    #[error("invalid RPC URI")]
+    InvalidRpcUriParse(#[source] url::ParseError),
+    #[error("RPC URI too long")]
+    RpcUriTooLong,
+    #[error("invalid node magic")]
+    InvalidNodeMagic,
+    #[error("unsupported node version")]
+    UnsupportedNodeVersion,
+    #[error("invalid node length")]
+    InvalidNodeLength,
+    #[error("truncated node record")]
+    TruncatedNodeRecord,
+    #[error("invalid node UTF-8")]
+    InvalidNodeUtf8(#[source] std::str::Utf8Error),
+    #[error("trailing node bytes")]
+    TrailingNodeBytes,
+}
+impl CaptureNodeError {
+    pub fn code(&self) -> &'static str {
+        match self {
+            Self::InvalidRpcUri | Self::InvalidRpcUriParse(_) => "invalid_rpc_uri",
+            Self::RpcUriTooLong => "rpc_uri_too_long",
+            Self::InvalidNodeMagic => "invalid_node_magic",
+            Self::UnsupportedNodeVersion => "unsupported_node_version",
+            Self::InvalidNodeLength => "invalid_node_length",
+            Self::TruncatedNodeRecord => "truncated_node_record",
+            Self::InvalidNodeUtf8(_) => "invalid_node_utf8",
+            Self::TrailingNodeBytes => "trailing_node_bytes",
+        }
+    }
+}
+impl CaptureNode {
+    pub fn new(rpc_uri: String) -> Result<Self, CaptureNodeError> {
+        let node = Self { rpc_uri };
+        node.validate()?;
+        Ok(node)
+    }
+
+    pub fn rpc_uri(&self) -> &str {
+        &self.rpc_uri
+    }
+
+    pub fn validate(&self) -> Result<(), CaptureNodeError> {
+        let uri = &self.rpc_uri;
+        if uri.len() > 4096 {
+            return Err(CaptureNodeError::RpcUriTooLong);
+        }
+        if uri.is_empty() || uri.chars().any(|c| c.is_whitespace() || c.is_control() || c == '\\') {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        let rest = uri
+            .strip_prefix("http://")
+            .or_else(|| uri.strip_prefix("https://"))
+            .ok_or(CaptureNodeError::InvalidRpcUri)?;
+        if uri.contains('?') || uri.contains('#') {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p));
+        // An empty suffix after the first slash is the only allowed path.
+        if !path.is_empty() || authority.contains('@') {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        let (host, port) = if authority.starts_with('[') {
+            let end = authority.find(']').ok_or(CaptureNodeError::InvalidRpcUri)?;
+            let port = authority
+                .get(end + 1..)
+                .and_then(|suffix| suffix.strip_prefix(':'))
+                .ok_or(CaptureNodeError::InvalidRpcUri)?;
+            (&authority[..=end], port)
+        } else {
+            authority.split_once(':').ok_or(CaptureNodeError::InvalidRpcUri)?
+        };
+        if host.is_empty() || port.is_empty() || !port.bytes().all(|b| b.is_ascii_digit()) {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        let port = port.parse::<u16>().map_err(|_| CaptureNodeError::InvalidRpcUri)?;
+        if port == 0 {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        let parsed_url = url::Url::parse(uri).map_err(CaptureNodeError::InvalidRpcUriParse)?;
+        if parsed_url.host().is_none() {
+            return Err(CaptureNodeError::InvalidRpcUri);
+        }
+        Ok(())
+    }
+
+    pub fn encode(&self) -> Result<Vec<u8>, CaptureNodeError> {
+        self.validate()?;
+        let length = u32::try_from(self.rpc_uri.len()).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
+        let mut bytes = b"CNOD\x01".to_vec();
+        bytes.extend_from_slice(&length.to_be_bytes());
+        bytes.extend_from_slice(self.rpc_uri.as_bytes());
+        Ok(bytes)
+    }
+
+    pub fn decode(bytes: &[u8]) -> Result<Self, CaptureNodeError> {
+        if bytes.len() < 9 {
+            return Err(CaptureNodeError::TruncatedNodeRecord);
+        }
+        if &bytes[..4] != b"CNOD" {
+            return Err(CaptureNodeError::InvalidNodeMagic);
+        }
+        if bytes[4] != 1 {
+            return Err(CaptureNodeError::UnsupportedNodeVersion);
+        }
+        let length = u32::from_be_bytes(bytes[5..9].try_into().map_err(|_| CaptureNodeError::InvalidNodeLength)?);
+        if !(1..=4096).contains(&length) {
+            return Err(CaptureNodeError::InvalidNodeLength);
+        }
+        let length = usize::try_from(length).map_err(|_| CaptureNodeError::InvalidNodeLength)?;
+        let end = 9_usize.checked_add(length).ok_or(CaptureNodeError::InvalidNodeLength)?;
+        if bytes.len() < end {
+            return Err(CaptureNodeError::TruncatedNodeRecord);
+        }
+        if bytes.len() != end {
+            return Err(CaptureNodeError::TrailingNodeBytes);
+        }
+        let uri = std::str::from_utf8(&bytes[9..end]).map_err(CaptureNodeError::InvalidNodeUtf8)?;
+        Self::new(uri.to_owned())
+    }
+}
+impl Serialize for CaptureNode {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        self.validate().map_err(serde::ser::Error::custom)?;
+        let mut state = serializer.serialize_struct("CaptureNode", 1)?;
+        state.serialize_field("rpc_uri", &self.rpc_uri)?;
+        state.end()
+    }
+}
+impl<'de> Deserialize<'de> for CaptureNode {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        #[derive(Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct Fields {
+            rpc_uri: String,
+        }
+        let fields = Fields::deserialize(deserializer).map_err(|_| serde::de::Error::custom("invalid capture node fields"))?;
+        Self::new(fields.rpc_uri).map_err(serde::de::Error::custom)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::error::Error;
+
+    fn record(payload: &[u8]) -> Vec<u8> {
+        let mut bytes = b"CNOD\x01".to_vec();
+        bytes.extend_from_slice(&u32::try_from(payload.len()).expect("test payload length").to_be_bytes());
+        bytes.extend_from_slice(payload);
+        bytes
+    }
+    fn reject_uri(uri: &str) {
+        let result = CaptureNode::new(uri.to_owned());
+        assert!(result.is_err(), "URI should reject: {uri:?}");
+        assert_eq!(result.expect_err("rejected URI").code(), "invalid_rpc_uri");
+    }
+    fn reject_record(bytes: &[u8], code: &str) {
+        let result = CaptureNode::decode(bytes);
+        assert!(result.is_err(), "record should reject with {code}");
+        assert_eq!(result.expect_err("rejected record").code(), code);
+    }
+    #[test]
+    fn actual_openraft_node_bound() {
+        fn require_node<T: openraft::Node>() {}
+        require_node::<CaptureNode>();
+        assert_eq!(CaptureNode::default().rpc_uri(), "");
+    }
+    #[test]
+    fn default_and_empty_rejected_at_every_boundary() {
+        let node = CaptureNode::default();
+        assert_eq!(node.rpc_uri(), "");
+        reject_uri("");
+        assert_eq!(node.validate().expect_err("empty validation").code(), "invalid_rpc_uri");
+        assert!(serde_json::to_string(&node).is_err());
+        assert!(serde_json::from_str::<CaptureNode>(r#"{"rpc_uri":""}"#).is_err());
+        assert_eq!(node.encode().expect_err("empty encoding").code(), "invalid_rpc_uri");
+        reject_record(&record(b""), "invalid_node_length");
+    }
+    #[test]
+    fn accepted_uris_preserve_exact_bytes() {
+        for uri in [
+            "http://node:80",
+            "https://node:443",
+            "http://node:1",
+            "https://node:65535/",
+            "http://127.0.0.1:9000",
+            "https://[::1]:443/",
+            "http://[2001:db8::1]:1",
+            "http://NoDe:00080/",
+        ] {
+            let result = CaptureNode::new(uri.to_owned());
+            assert!(result.is_ok(), "valid URI construction: {uri}");
+            let node = result.expect("valid URI");
+            assert_eq!(node.rpc_uri(), uri, "preserve explicit default port and raw URI");
+            assert_eq!(node.encode().expect("encode URI"), record(uri.as_bytes()));
+            assert_eq!(CaptureNode::decode(&record(uri.as_bytes())).expect("decode URI"), node);
+        }
+    }
+    #[test]
+    fn golden_cnod_bytes() {
+        let node = CaptureNode {
+            rpc_uri: "http://node:80".to_owned(),
+        };
+        let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
+        assert_eq!(node.encode().ok().as_deref(), Some(golden.as_slice()), "golden CNOD bytes");
+    }
+    #[test]
+    fn valid_cnod_decode() {
+        let golden = b"CNOD\x01\x00\x00\x00\x0ehttp://node:80";
+        let decoded = CaptureNode::decode(golden);
+        assert!(decoded.is_ok(), "golden decode succeeds");
+        assert_eq!(decoded.expect("golden decoded node").rpc_uri(), "http://node:80");
+    }
+    #[test]
+    fn serde_shape_roundtrip_and_strict_fields() {
+        let node = CaptureNode::new("https://node:443/".to_owned()).expect("serde node");
+        let json = r#"{"rpc_uri":"https://node:443/"}"#;
+        assert_eq!(serde_json::to_string(&node).expect("serialize node"), json);
+        assert_eq!(serde_json::from_str::<CaptureNode>(json).expect("deserialize node"), node);
+        for json in [
+            r#"{}"#,
+            r#"{"rpc_uri":1}"#,
+            r#"{"rpc_uri":null}"#,
+            r#"{"rpc_uri":"http://node:1","extra":0}"#,
+            r#"{"rpc_uri":"http://node:1","rpc_uri":"http://node:2"}"#,
+        ] {
+            assert!(serde_json::from_str::<CaptureNode>(json).is_err(), "strict fields: {json}");
+        }
+    }
+    #[test]
+    fn credentials_and_raw_uri_shapes_reject() {
+        for uri in [
+            "http://user@node:1",
+            "http://user:password@node:1",
+            "http://@node:1",
+            "HTTP://node:1",
+            "Https://node:1",
+            "http://node",
+            "http://node:",
+            "http://node:0",
+            "http://node:-1",
+            "http://node:+1",
+            "http://node:65536",
+            "http://node:999999999999999999999",
+            "http://[::1:1",
+            "http://[::1]extra:1",
+            "http://[invalid]:1",
+            "http://::1:1",
+            "http://:1",
+            "/disk/path",
+            "file:///disk/path",
+            "http://node:1/./",
+            "http://node:1/../",
+            "http://node:1/%2e/",
+            "http://node:1/%2E%2e/",
+            "http://node:1/path",
+            "http://node:1\\",
+            " http://node:1",
+            "http://node:1\n",
+            "http://no\tde:1",
+            "http://node:1\u{00a0}",
+            "http://node:1\u{0085}",
+            "http://node:1\0",
+        ] {
+            reject_uri(uri);
+        }
+    }
+    #[test]
+    fn query_and_fragment_markers_reject() {
+        for uri in [
+            "http://node:1?",
+            "http://node:1?x=y",
+            "http://node:1/?",
+            "http://node:1#",
+            "http://node:1#fragment",
+            "http://node:1/#",
+        ] {
+            reject_uri(uri);
+        }
+    }
+    #[test]
+    fn utf8_byte_length_boundaries() {
+        for length in [4095, 4096] {
+            let prefix = "http://node:";
+            let uri = format!("{prefix}{}1", "0".repeat(length - prefix.len() - 1));
+            assert_eq!(uri.len(), length);
+            assert!(url::Url::parse(&uri).is_ok(), "boundary fixture accepted by actual URL parser");
+            let node = CaptureNode::new(uri.clone()).expect("boundary node");
+            assert_eq!(node.rpc_uri(), uri);
+            assert_eq!(
+                CaptureNode::decode(&node.encode().expect("boundary encoding")).expect("boundary decoding"),
+                node
+            );
+        }
+        let uri = format!("http://node:{}1", "0".repeat(4097 - "http://node:".len() - 1));
+        assert_eq!(CaptureNode::new(uri).expect_err("oversize URI").code(), "rpc_uri_too_long");
+        let unicode = format!("http://{}:1", "é".repeat(2044));
+        assert!(unicode.chars().count() < 4096);
+        assert!(unicode.len() > 4096);
+        assert_eq!(CaptureNode::new(unicode).expect_err("UTF-8 byte limit").code(), "rpc_uri_too_long");
+    }
+    #[test]
+    fn cnod_headers_and_lengths_reject_exact_codes() {
+        let valid = record(b"http://node:80");
+        for end in 0..9 {
+            reject_record(&valid[..end], "truncated_node_record");
+        }
+        let mut bad = valid.clone();
+        bad[0] = b'X';
+        reject_record(&bad, "invalid_node_magic");
+        bad = valid.clone();
+        bad[4] = 2;
+        reject_record(&bad, "unsupported_node_version");
+        for length in [0_u32, 4097, u32::MAX] {
+            bad = b"CNOD\x01".to_vec();
+            bad.extend_from_slice(&length.to_be_bytes());
+            reject_record(&bad, "invalid_node_length");
+        }
+        for end in 9..valid.len() {
+            reject_record(&valid[..end], "truncated_node_record");
+        }
+        reject_record(&record(&[0xff]), "invalid_node_utf8");
+        reject_record(&record(&[0xc3, 0x28]), "invalid_node_utf8");
+        reject_record(&record(b"invalid URI"), "invalid_rpc_uri");
+    }
+    #[test]
+    fn trailing_bytes_reject() {
+        for tail in [&b"x"[..], &b"xyz"[..]] {
+            let mut bytes = record(b"http://node:80");
+            bytes.extend_from_slice(tail);
+            reject_record(&bytes, "trailing_node_bytes");
+        }
+    }
+    #[test]
+    fn errors_and_sources_redact_input() {
+        let sentinel = "http://SECRET_USER:SECRET_PASSWORD@node:1";
+        let error = CaptureNode::new(sentinel.to_owned()).expect_err("credentials reject");
+        let mut current: &dyn Error = &error;
+        loop {
+            let rendered = format!("{current} {current:?}");
+            assert!(!rendered.contains("SECRET"));
+            match current.source() {
+                Some(source) => current = source,
+                None => break,
+            }
+        }
+        let json = format!(r#"{{"rpc_uri":"{sentinel}"}}"#);
+        let error = serde_json::from_str::<CaptureNode>(&json).expect_err("serde credentials reject");
+        assert!(!format!("{error} {error:?}").contains("SECRET"));
+        let json = format!(r#"{{"{sentinel}":0}}"#);
+        let error = serde_json::from_str::<CaptureNode>(&json).expect_err("unknown fields reject");
+        assert!(!format!("{error} {error:?}").contains("SECRET"));
+        let error = CaptureNode::new("http://[bad]:1".to_owned()).expect_err("parser rejection");
+        assert!(error.source().is_some(), "retain URL parser source");
+        let error = CaptureNode::decode(&record(&[0xff])).expect_err("UTF-8 rejection");
+        assert!(error.source().is_some(), "retain UTF-8 source");
+    }
+}

crates/capture-authority/src/truncate.rs

@@ -0,0 +1,43 @@
+use crate::{LOGS, META, Store, StoreError, decode_id, decode_log, encode_id};
+use redb::ReadableTable;
+
+impl Store {
+    /// Atomically remove every log entry at or after `from`.
+    pub fn truncate_suffix(&mut self, from: u64) -> Result<(), StoreError> {
+        let tx = self.write_transaction()?;
+        {
+            let mut meta = tx.open_table(META)?;
+            let logs = tx.open_table(LOGS)?;
+            if crate::committed::validate_committed(&meta, &logs)?.is_some_and(|id| from <= id.index) {
+                return Err(StoreError::CommittedLogProtected);
+            }
+            drop(logs);
+            let tail = meta.get("tail")?.map(|v| decode_id(v.value())).transpose()?;
+            if tail.is_none_or(|id| from > id.index) {
+                return Ok(());
+            }
+            let mut logs = tx.open_table(LOGS)?;
+            let new_tail = if from == 0 {
+                None
+            } else {
+                let index = from - 1;
+                let record = logs.get(index)?.ok_or(StoreError::InvalidLogSequence)?;
+                let id = decode_log(record.value())?.id;
+                if id.index != index {
+                    return Err(StoreError::CorruptRecord);
+                }
+                Some(id)
+            };
+            logs.retain(|index, _| index < from)?;
+            match new_tail {
+                Some(id) => {
+                    meta.insert("tail", encode_id(id).as_slice())?;
+                }
+                None => {
+                    meta.remove("tail")?;
+                }
+            }
+        }
+        self.commit(tx)
+    }
+}

crates/config/src/constants/app.rs

@@ -18,10 +18,8 @@ use const_str::concat;
 /// Default value: RustFS
 /// Environment variable: RUSTFS_APP_NAME
 pub const APP_NAME: &str = "RustFS";
-/// Application version
-/// Default value: 1.0.0
-/// Environment variable: RUSTFS_VERSION
-pub const VERSION: &str = "1.0.0";
+/// Cargo package version for this workspace member.
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Default configuration logger level
 /// Default value: error
@@ -39,12 +37,6 @@ pub const SAMPLE_RATIO: f64 = 1.0;
 /// Default value: 30
 pub const METER_INTERVAL: u64 = 30;
 
-/// Default configuration service version
-/// Default value: 1.0.0
-/// Environment variable: RUSTFS_OBS_SERVICE_VERSION
-/// Uses the same value as VERSION constant
-pub const SERVICE_VERSION: &str = "1.0.0";
-
 /// Default configuration environment
 /// Default value: production
 pub const ENVIRONMENT: &str = "production";
@@ -410,7 +402,6 @@ mod tests {
             APP_NAME,
             VERSION,
             DEFAULT_LOG_LEVEL,
-            SERVICE_VERSION,
             ENVIRONMENT,
             RUSTFS_TLS_KEY,
             RUSTFS_TLS_CERT,
@@ -446,8 +437,7 @@ mod tests {
     fn test_configuration_consistency() {
         // Test configuration consistency
 
-        // Version consistency
-        assert_eq!(VERSION, SERVICE_VERSION, "Application version should match service version");
+        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
 
         // Port conflict check
         let ports = [DEFAULT_PORT, DEFAULT_CONSOLE_PORT];

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -31,6 +31,7 @@ use super::harness::{
 use crate::common::{
     AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
 };
+use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use std::path::{Path, PathBuf};
@@ -400,6 +401,19 @@ async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> Tes
     Ok(())
 }
 
+async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed: &UpgradeSeed, node_idx: usize) -> TestResult {
+    dist.replace_node_with_current_binary(node_idx).await?;
+    // Node readiness proves reads can succeed; a rolling writer must also
+    // recover before replacing the next peer or checking upgrade contracts.
+    wait_for_upgrade_write_readiness(
+        &dist.clients()?,
+        &seed.history_bucket,
+        &format!("replaced-node-{node_idx}"),
+        CREDENTIAL_TIMEOUT,
+    )
+    .await
+}
+
 #[tokio::test]
 async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
     init_logging();
@@ -410,15 +424,15 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
 
     let seed = seed_history_and_iam(&dist).await?;
 
-    dist.replace_node_with_current_binary(0).await?;
+    replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
     assert_history_and_iam(&dist, &seed, "one-current-node").await?;
 
     for node_idx in [1, 2] {
-        dist.replace_node_with_current_binary(node_idx).await?;
+        replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
     }
     assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
 
-    dist.replace_node_with_current_binary(3).await?;
+    replace_node_and_wait_for_write_readiness(&mut dist, &seed, 3).await?;
     assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
     Ok(())
 }

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -578,7 +578,7 @@ async fn exercise_mixed_cluster(
     previous_node: usize,
 ) -> TestResult {
     let clients = cluster.create_all_clients()?;
-    wait_for_upgrade_write_readiness(&clients, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
+    wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     let current_client = &clients[current_node];
     let previous_client = &clients[previous_node];
 
@@ -646,7 +646,12 @@ fn upgrade_probe_client(client: &Client) -> Client {
     )
 }
 
-async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budget: Duration) -> TestResult {
+pub(crate) async fn wait_for_upgrade_write_readiness(
+    clients: &[Client],
+    bucket: &str,
+    phase: &str,
+    budget: Duration,
+) -> TestResult {
     // ListBuckets can succeed before peers recover a restarted disk. Cluster
     // health also accepts Returning disks whose write health is still FAULTY.
     // Probe every writer outside the asserted phase prefix; compatibility
@@ -666,7 +671,7 @@ async fn wait_for_upgrade_write_readiness(clients: &[Client], phase: &str, budge
                 deadline,
                 client
                     .put_object()
-                    .bucket(MIXED_BUCKET)
+                    .bucket(bucket)
                     .key(&key)
                     .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
                     .send(),
@@ -701,19 +706,20 @@ async fn prepare_previous_release_baseline(cluster: &mut RustFSTestClusterEnviro
     // waits for pool metadata (#7473). First prove the elected writer can
     // persist data, then restart each old process once with three peers still
     // readable. This preparation ends before any current binary is started.
-    wait_for_upgrade_write_readiness(&clients[..1], "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
+    wait_for_upgrade_write_readiness(&clients[..1], MIXED_BUCKET, "previous-seed", LISTING_CONVERGENCE_TIMEOUT).await?;
     for node in [1, 2, 3, 0] {
         cluster.stop_node(node)?;
         cluster.start_node_from_binary(node, previous_binary).await?;
         wait_for_upgrade_write_readiness(
             std::slice::from_ref(&clients[node]),
+            MIXED_BUCKET,
             &format!("previous-restart-{node}"),
             LISTING_CONVERGENCE_TIMEOUT,
         )
         .await?;
     }
 
-    wait_for_upgrade_write_readiness(&clients, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
+    wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, "previous-baseline", LISTING_CONVERGENCE_TIMEOUT).await?;
     for (reader, client) in clients.iter().enumerate() {
         assert_eq!(
             read_object(client, MIXED_BUCKET, PREVIOUS_RELEASE_SEED_KEY, None).await?.1,
@@ -740,9 +746,10 @@ mod upgrade_write_readiness_tests {
     #[tokio::test]
     async fn waits_for_each_writer_after_metadata_is_ready() -> TestResult {
         let target = FakeS3Target::start().await?;
-        target.create_bucket(MIXED_BUCKET);
+        let bucket = "distributed-upgrade-history";
+        target.create_bucket(bucket);
         let client = fake_source_client(&target);
-        client.head_bucket().bucket(MIXED_BUCKET).send().await?;
+        client.head_bucket().bucket(bucket).send().await?;
 
         let phase = "one-previous-node";
         let first_key = format!(".upgrade-readiness/{phase}/node-0");
@@ -762,7 +769,7 @@ mod upgrade_write_readiness_tests {
         // Metadata readiness does not prove that a data write can succeed.
         let premature = client
             .put_object()
-            .bucket(MIXED_BUCKET)
+            .bucket(bucket)
             .key(&first_key)
             .body(ByteStream::from_static(b"upgrade write readiness"))
             .send()
@@ -770,15 +777,15 @@ mod upgrade_write_readiness_tests {
             .expect_err("metadata readiness does not prove write readiness");
         assert_eq!(premature.raw_response().map(|response| response.status().as_u16()), Some(503));
 
-        wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], phase, Duration::from_secs(5)).await?;
+        wait_for_upgrade_write_readiness(&[client.clone(), client.clone()], bucket, phase, Duration::from_secs(5)).await?;
         assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &first_key), 3);
         assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &second_key), 2);
-        assert!(target.has_object(MIXED_BUCKET, &first_key));
-        assert!(target.has_object(MIXED_BUCKET, &second_key));
+        assert!(target.has_object(bucket, &first_key));
+        assert!(target.has_object(bucket, &second_key));
         assert!(
             client
                 .list_objects_v2()
-                .bucket(MIXED_BUCKET)
+                .bucket(bucket)
                 .prefix(format!("{phase}/"))
                 .send()
                 .await?
@@ -808,9 +815,10 @@ mod upgrade_write_readiness_tests {
             let phase = format!("permanent-{}", status.as_u16());
             let key = format!(".upgrade-readiness/{phase}/node-0");
             target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Status(status), 1);
-            let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), &phase, Duration::from_secs(5))
-                .await
-                .expect_err("a permanent error must not be retried into success");
+            let error =
+                wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, &phase, Duration::from_secs(5))
+                    .await
+                    .expect_err("a permanent error must not be retried into success");
             assert!(error.to_string().contains("node 0 write readiness failed"), "{error}");
             assert_eq!(target.count_requests(FakeTargetOperation::PutObject, &key), 1);
             assert!(!target.has_object(MIXED_BUCKET, &key));
@@ -833,7 +841,7 @@ mod upgrade_write_readiness_tests {
             FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE),
             10,
         );
-        let error = wait_for_upgrade_write_readiness(&[client], phase, Duration::from_secs(1))
+        let error = wait_for_upgrade_write_readiness(&[client], MIXED_BUCKET, phase, Duration::from_secs(1))
             .await
             .expect_err("persistent unavailability must exhaust the shared deadline");
         // Transport scheduling consumes the same budget; do not require a
@@ -852,7 +860,7 @@ mod upgrade_write_readiness_tests {
         let phase = "stalled";
         let key = format!(".upgrade-readiness/{phase}/node-0");
         target.inject_for_key(FakeTargetOperation::PutObject, &key, FaultAction::Stall(Duration::from_secs(30)), 1);
-        let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), phase, Duration::from_secs(1))
+        let error = wait_for_upgrade_write_readiness(std::slice::from_ref(&client), MIXED_BUCKET, phase, Duration::from_secs(1))
             .await
             .expect_err("a stalled request must not outlive the readiness deadline");
         assert!(error.to_string().contains("deadline exceeded during PutObject"), "{error}");

crates/ecstore/src/api/mod.rs

@@ -240,7 +240,10 @@ pub mod bucket {
     }
 
     pub mod quota {
-        pub use crate::bucket::quota::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
+        pub use crate::bucket::quota::{
+            BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
+            QuotaOperation,
+        };
 
         pub mod checker {
             pub use crate::bucket::quota::checker::QuotaChecker;
@@ -362,11 +365,39 @@ pub mod config {
             ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
             is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
             read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
-            read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
-            save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation,
-            server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
-            with_server_config_read_lock, with_server_config_write_lock,
+            read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_config_with_opts_quiet,
+            save_server_config, save_server_config_no_lock, save_server_config_snapshot,
+            save_server_config_snapshot_with_generation, server_config_path, try_migrate_server_config,
+            with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
+            with_server_config_write_lock,
         };
+
+        /// Observe persisted config bytes and their reader-associated metadata within a byte limit.
+        pub async fn read_config_observation(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            max_bytes: usize,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_observation(api, file, max_bytes).await
+        }
+
+        /// Read config bytes and metadata without treating an empty payload as absent.
+        pub async fn read_config_preserve_empty_with_metadata_opts(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            opts: &crate::object_api::ObjectOptions,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
+        }
+
+        pub async fn read_config_limited_preserve_empty_with_metadata_opts(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            opts: &crate::object_api::ObjectOptions,
+            max_bytes: usize,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await
+        }
     }
 
     pub mod storageclass {
@@ -468,7 +499,7 @@ pub mod event {
 pub mod global {
     pub use crate::runtime::global::{
         set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver, shutdown_background_services,
-        update_erasure_type,
+        update_erasure_type, wait_for_detached_mutations,
     };
 }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -87,6 +87,7 @@ use rustfs_filemeta::{
     FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
     is_restored_object_on_disk,
 };
+use rustfs_lock::{NamespaceLockGuard, error::LockError};
 use rustfs_scanner_metrics::metrics::{
     IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
 };
@@ -190,6 +191,8 @@ const TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL: StdDuration = StdDuration::from_
 // before another full sweep can discover work persisted by another node.
 const TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL: StdDuration = StdDuration::from_secs(10 * 60);
 const TIER_FREE_VERSION_RECOVERY_JITTER_PERCENT: u64 = 10;
+const TIER_FREE_VERSION_RECOVERY_LEADER_LOCK: &str = "tier-free-version-recovery/leader.lock";
+const TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT: StdDuration = StdDuration::from_secs(1);
 const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
 const EXPIRY_WORKER_QUEUE_CAPACITY: usize = 1000;
 const DEFAULT_MANUAL_TRANSITION_JOB_RECOVERY_LIMIT: usize = 100;
@@ -1441,13 +1444,24 @@ pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions
     queued
 }
 
-async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
+async fn enqueue_recovered_free_version_with_state(
+    state: &Arc<RwLock<ExpiryState>>,
+    oi: ObjectInfo,
+    cancel: &CancellationToken,
+) -> bool {
     let task = FreeVersionTask(oi);
     let hash = task.op_hash();
     let (wrkr, stats) = {
-        let state = state.read().await;
+        let state = select! {
+            biased;
+            _ = cancel.cancelled() => return false,
+            state = state.read() => state,
+        };
         (state.get_worker_ch(hash), Arc::clone(&state.stats))
     };
+    if cancel.is_cancelled() {
+        return false;
+    }
     let Some(wrkr) = wrkr else {
         stats.increment_missed_freevers_tasks();
         stats.record_scanner_expiry_state();
@@ -1497,9 +1511,13 @@ fn set_recovered_free_version_enqueue_observer(
     RecoveredFreeVersionEnqueueObserverGuard
 }
 
-pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
+pub(super) async fn enqueue_recovered_free_version_with_cancel(
+    api: &ECStore,
+    oi: ObjectInfo,
+    cancel: &CancellationToken,
+) -> bool {
     let expiry_state = api.ctx.expiry_state();
-    let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
+    let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi, cancel).await;
 
     #[cfg(test)]
     if let Some(observer) = RECOVERED_FREE_VERSION_ENQUEUE_OBSERVER
@@ -2921,6 +2939,20 @@ fn resolve_tier_free_version_recovery_enabled(value: Result<String, env::VarErro
 }
 
 fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<()>) -> Option<JoinHandle<()>> {
+    spawn_tier_free_version_recovery_once_with_options(
+        api,
+        started,
+        TIER_FREE_VERSION_RECOVERY_LEADER_LOCK.to_string(),
+        tier_free_version_recovery_leader_retry_delay,
+    )
+}
+
+fn spawn_tier_free_version_recovery_once_with_options(
+    api: Arc<ECStore>,
+    started: &OnceLock<()>,
+    leader_lock_name: String,
+    retry_delay: fn() -> StdDuration,
+) -> Option<JoinHandle<()>> {
     if !tier_free_version_recovery_enabled() {
         warn!(
             event = EVENT_LIFECYCLE_WORKER_STATE,
@@ -2937,20 +2969,43 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
 
     Some(tokio::spawn(async move {
         let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
-        let expiry_state = api.ctx.expiry_state();
-        run_tier_free_version_recovery_loop(
+        let acquire_api = Arc::clone(&api);
+        let acquire_lock_name = Arc::<str>::from(leader_lock_name);
+        run_tier_free_version_recovery_coordinator(
             cancel_token,
-            expiry_state,
-            jitter_tier_free_version_recovery_delay,
-            move |bucket_marker, object_marker, recovery_cancel| {
+            retry_delay,
+            move || {
+                let api = Arc::clone(&acquire_api);
+                let lock_name = Arc::clone(&acquire_lock_name);
+                async move {
+                    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &lock_name).await?;
+                    match lock.get_write_lock_quiet(TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT).await {
+                        Ok(guard) => Ok(Some(guard)),
+                        Err(LockError::Timeout { .. }) => Ok(None),
+                        Err(err) => Err(err.into()),
+                    }
+                }
+            },
+            move |leader_cancel| {
                 let api = Arc::clone(&api);
                 async move {
-                    recover_tier_free_versions_with_cancel(
-                        api,
-                        DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
-                        bucket_marker,
-                        object_marker,
-                        recovery_cancel,
+                    run_tier_free_version_recovery_loop(
+                        leader_cancel,
+                        api.ctx.expiry_state(),
+                        jitter_tier_free_version_recovery_delay,
+                        move |bucket_marker, object_marker, recovery_cancel| {
+                            let api = Arc::clone(&api);
+                            async move {
+                                recover_tier_free_versions_with_cancel(
+                                    api,
+                                    DEFAULT_FREE_VERSION_RECOVERY_LIMIT,
+                                    bucket_marker,
+                                    object_marker,
+                                    recovery_cancel,
+                                )
+                                .await
+                            }
+                        },
                     )
                     .await
                 }
@@ -2960,6 +3015,86 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
     }))
 }
 
+fn tier_free_version_recovery_leader_retry_delay() -> StdDuration {
+    StdDuration::from_secs(rand::rng().random_range(15..=30))
+}
+
+async fn run_tier_free_version_recovery_coordinator<A, AF, R, RF>(
+    cancel: CancellationToken,
+    retry_delay: fn() -> StdDuration,
+    mut acquire: A,
+    mut recover: R,
+) where
+    A: FnMut() -> AF,
+    AF: Future<Output = crate::error::Result<Option<NamespaceLockGuard>>>,
+    R: FnMut(CancellationToken) -> RF,
+    RF: Future<Output = ()>,
+{
+    loop {
+        let acquired = select! {
+            biased;
+            _ = cancel.cancelled() => return,
+            result = acquire() => result,
+        };
+        match acquired {
+            Ok(Some(mut guard)) => {
+                let leader_cancel = cancel.child_token();
+                // Hold ownership across every page, cooldown and idle wait. A
+                // follower taking over starts a fresh sweep of persisted markers.
+                let recovery = recover(leader_cancel.clone());
+                tokio::pin!(recovery);
+                debug!(
+                    event = EVENT_LIFECYCLE_WORKER_STATE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                    state = "free_version_recovery_leader_acquired",
+                    lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
+                    "Tier free-version recovery leader acquired"
+                );
+                select! {
+                    biased;
+                    _ = cancel.cancelled() => {
+                        leader_cancel.cancel();
+                        recovery.await;
+                    }
+                    _ = guard.lock_lost_notified() => {
+                        debug!(
+                            event = EVENT_LIFECYCLE_WORKER_STATE,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                            state = "free_version_recovery_leader_lost",
+                            lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
+                            "Tier free-version recovery leader lost"
+                        );
+                        // Dropping the recovery future would detach its active
+                        // walk. Cancel it and poll the same future through cleanup.
+                        leader_cancel.cancel();
+                        recovery.await;
+                    }
+                    _ = &mut recovery => {}
+                }
+                leader_cancel.cancel();
+                guard.release();
+            }
+            Ok(None) => {}
+            Err(err) => warn!(
+                event = EVENT_LIFECYCLE_WORKER_STATE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                state = "free_version_recovery_leader_acquire_failed",
+                lock_name = TIER_FREE_VERSION_RECOVERY_LEADER_LOCK,
+                error = %err,
+                "Tier free-version recovery leader acquisition failed"
+            ),
+        }
+        select! {
+            biased;
+            _ = cancel.cancelled() => return,
+            _ = tokio::time::sleep(retry_delay()) => {}
+        }
+    }
+}
+
 async fn run_tier_free_version_recovery_loop<F, Fut>(
     cancel_token: CancellationToken,
     expiry_state: Arc<RwLock<ExpiryState>>,
@@ -2969,7 +3104,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
     F: FnMut(Option<String>, Option<String>, CancellationToken) -> Fut,
     Fut: Future<Output = crate::error::Result<FreeVersionRecoveryStats>>,
 {
-    let recovery_notify = Arc::clone(&expiry_state.read().await.recovery_notify);
+    let recovery_notify = select! {
+        biased;
+        _ = cancel_token.cancelled() => return,
+        state = expiry_state.read() => Arc::clone(&state.recovery_notify),
+    };
     let mut schedule = TierFreeVersionRecoverySchedule::default();
 
     loop {
@@ -2992,7 +3131,11 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
                 schedule.record_success(&stats);
                 rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0);
                 let (pending_tasks, active_tasks) = {
-                    let state = expiry_state.read().await;
+                    let state = select! {
+                        biased;
+                        _ = cancel_token.cancelled() => return,
+                        state = expiry_state.read() => state,
+                    };
                     (state.pending_tasks(), state.stats.active_tasks())
                 };
                 debug!(
@@ -5652,8 +5795,10 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
     } else {
         None
     };
+    // The delete owns the publication guard from here on, so a cancelled
+    // expiry worker cannot let table-bucket publication overtake it.
     let mut dobj = match api
-        .delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
+        .delete_object_with_tier_delete_journal_and_guards(&oi.bucket, &encode_dir_object(&oi.name), opts, [publication_guard])
         .await
     {
         Ok(dobj) => dobj,
@@ -5904,10 +6049,10 @@ mod tests {
         merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned,
         persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs,
         resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
-        resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop,
-        select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
-        should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
-        wait_for_tier_free_version_recovery,
+        resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator,
+        run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
+        set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
+        transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
     };
     #[cfg(feature = "test-util")]
     use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
@@ -6001,6 +6146,231 @@ mod tests {
     use tokio_util::sync::CancellationToken;
     use uuid::Uuid;
 
+    fn recovery_test_lock() -> (Arc<rustfs_lock::NamespaceLock>, Vec<Arc<dyn rustfs_lock::LockClient>>) {
+        let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..3)
+            .map(|_| {
+                Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())))
+                    as Arc<dyn rustfs_lock::LockClient>
+            })
+            .collect();
+        let lock =
+            rustfs_lock::NamespaceLock::with_clients_and_quorum(format!("recovery-test-{}", Uuid::new_v4()), clients.clone(), 2);
+        (Arc::new(lock), clients)
+    }
+
+    async fn acquire_recovery_test_lock(
+        lock: Arc<rustfs_lock::NamespaceLock>,
+        owner: String,
+    ) -> crate::error::Result<Option<rustfs_lock::NamespaceLockGuard>> {
+        // Keep this lease longer than the virtual-time cooldown/idle advances;
+        // production uses the namespace wrapper's renewable 30-second lease.
+        lock.lock_guard(
+            rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, super::TIER_FREE_VERSION_RECOVERY_LEADER_LOCK),
+            &owner,
+            StdDuration::from_millis(5),
+            StdDuration::from_secs(3600),
+        )
+        .await
+        .map_err(Into::into)
+    }
+
+    fn recovery_test_retry_delay() -> StdDuration {
+        StdDuration::from_secs(15)
+    }
+
+    #[tokio::test(start_paused = true)]
+    #[serial]
+    async fn tier_free_version_recovery_eight_coordinators_hold_one_leader_and_take_over() {
+        let (lock, _clients) = recovery_test_lock();
+        let (pages_tx, mut pages_rx) = tokio::sync::mpsc::unbounded_channel();
+        let mut tasks = Vec::new();
+        let mut cancels = Vec::new();
+        for node in 0..8 {
+            let cancel = CancellationToken::new();
+            cancels.push(cancel.clone());
+            let lock = Arc::clone(&lock);
+            let pages_tx = pages_tx.clone();
+            tasks.push(tokio::spawn(async move {
+                run_tier_free_version_recovery_coordinator(
+                    cancel,
+                    recovery_test_retry_delay,
+                    move || acquire_recovery_test_lock(Arc::clone(&lock), format!("node-{node}")),
+                    move |leader_cancel| {
+                        let pages_tx = pages_tx.clone();
+                        async move {
+                            let mut pages = 0;
+                            run_tier_free_version_recovery_loop(
+                                leader_cancel,
+                                ExpiryState::new(),
+                                std::convert::identity,
+                                move |bucket, object, _| {
+                                    pages += 1;
+                                    pages_tx.send((node, bucket, object)).expect("page observer should be open");
+                                    std::future::ready(Ok(free_version_recovery_stats(0, 0, pages == 1)))
+                                },
+                            )
+                            .await;
+                        }
+                    },
+                )
+                .await;
+            }));
+        }
+        let (leader, bucket, object) = pages_rx.recv().await.expect("one coordinator should start recovery");
+        assert!(bucket.is_none() && object.is_none());
+        // The same leader must survive both a truncated page's cooldown and
+        // complete idle sweeps. All other coordinators keep probing the lock.
+        let mut observed_pages = 1;
+        for _ in 0..27 {
+            tokio::time::advance(StdDuration::from_secs(60)).await;
+            for _ in 0..100 {
+                tokio::task::yield_now().await;
+            }
+            while let Ok((node, bucket, object)) = pages_rx.try_recv() {
+                assert_eq!(node, leader, "followers must not repeat a namespace sweep");
+                if observed_pages == 1 {
+                    assert_eq!(bucket.as_deref(), Some("bucket"));
+                    assert_eq!(object.as_deref(), Some("object"));
+                } else {
+                    assert!(bucket.is_none() && object.is_none(), "complete sweeps should reset their cursors");
+                }
+                observed_pages += 1;
+            }
+        }
+        assert!(observed_pages >= 7, "the leader should finish a full ten-minute idle wait");
+        cancels[leader].cancel();
+        let (successor, bucket, object) = tokio::time::timeout(StdDuration::from_secs(60), pages_rx.recv())
+            .await
+            .expect("a follower should take over after leader shutdown")
+            .expect("page observer should remain open");
+        assert_ne!(successor, leader);
+        assert!(bucket.is_none() && object.is_none(), "a new leader must restart discovery safely");
+        for cancel in cancels {
+            cancel.cancel();
+        }
+        for task in tasks {
+            task.await.expect("all coordinators should shut down cleanly");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    #[serial]
+    async fn tier_free_version_recovery_lock_loss_waits_for_active_page_cleanup() {
+        let (lock, _clients) = recovery_test_lock();
+        let cancel = CancellationToken::new();
+        let loop_cancel = cancel.clone();
+        let (page_tx, mut page_rx) = tokio::sync::mpsc::unbounded_channel();
+        let cleanup_started = Arc::new(tokio::sync::Notify::new());
+        let finish_cleanup = Arc::new(tokio::sync::Notify::new());
+        let observed_cleanup = Arc::clone(&cleanup_started);
+        let release_cleanup = Arc::clone(&finish_cleanup);
+        let task = tokio::spawn(async move {
+            run_tier_free_version_recovery_coordinator(
+                loop_cancel,
+                recovery_test_retry_delay,
+                move || acquire_recovery_test_lock(Arc::clone(&lock), "old-leader".to_string()),
+                move |leader_cancel| {
+                    let page_tx = page_tx.clone();
+                    let cleanup_started = Arc::clone(&cleanup_started);
+                    let finish_cleanup = Arc::clone(&finish_cleanup);
+                    async move {
+                        run_tier_free_version_recovery_loop(
+                            leader_cancel,
+                            ExpiryState::new(),
+                            std::convert::identity,
+                            move |_, _, page_cancel| {
+                                let cleanup_started = Arc::clone(&cleanup_started);
+                                let finish_cleanup = Arc::clone(&finish_cleanup);
+                                page_tx.send(()).expect("page observer should remain open");
+                                async move {
+                                    page_cancel.cancelled().await;
+                                    cleanup_started.notify_one();
+                                    finish_cleanup.notified().await;
+                                    Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "cancelled page").into())
+                                }
+                            },
+                        )
+                        .await;
+                    }
+                },
+            )
+            .await;
+        });
+        page_rx.recv().await.expect("leader should start an active page");
+        // Let authoritative backend leases expire before the next heartbeat
+        // can renew them. A failed refresh must cancel the active page.
+        tokio::time::advance(StdDuration::from_secs(3600)).await;
+        tokio::time::timeout(StdDuration::from_secs(1), observed_cleanup.notified())
+            .await
+            .expect("expired backend leases must trigger recovery cancellation");
+        assert!(!task.is_finished(), "lock loss must wait for the active page's cleanup");
+        assert!(page_rx.try_recv().is_err(), "the old leader must not start another page");
+        cancel.cancel();
+        release_cleanup.notify_one();
+        task.await.expect("coordinator should finish cleanup before exiting");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn tier_free_version_recovery_coordinator_retries_without_scanning_after_lock_error() {
+        let cancel = CancellationToken::new();
+        let loop_cancel = cancel.clone();
+        let attempts = Arc::new(AtomicUsize::new(0));
+        let observed_attempts = Arc::clone(&attempts);
+        let task = tokio::spawn(async move {
+            run_tier_free_version_recovery_coordinator(
+                loop_cancel,
+                recovery_test_retry_delay,
+                move || {
+                    observed_attempts.fetch_add(1, Ordering::SeqCst);
+                    std::future::ready(Err(rustfs_lock::LockError::InsufficientNodes {
+                        required: 2,
+                        available: 1,
+                    }
+                    .into()))
+                },
+                |_| async { panic!("a node without ownership must never start discovery") },
+            )
+            .await;
+        });
+        tokio::task::yield_now().await;
+        assert_eq!(attempts.load(Ordering::SeqCst), 1);
+        tokio::time::advance(StdDuration::from_secs(14)).await;
+        assert_eq!(attempts.load(Ordering::SeqCst), 1, "lock errors must not cause a tight retry loop");
+        tokio::time::advance(StdDuration::from_secs(1)).await;
+        tokio::task::yield_now().await;
+        assert_eq!(attempts.load(Ordering::SeqCst), 2);
+        cancel.cancel();
+        task.await.expect("a follower should respond to shutdown during retry wait");
+    }
+
+    #[tokio::test]
+    async fn tier_free_version_recovery_enqueue_cancels_while_waiting_for_state_lock() {
+        let state = ExpiryState::new();
+        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
+        let mut held = state.write().await;
+        held.tasks_tx.push(tx);
+        let cancel = CancellationToken::new();
+        let enqueue_cancel = cancel.clone();
+        let enqueue_state = Arc::clone(&state);
+        let enqueue = tokio::spawn(async move {
+            enqueue_recovered_free_version_with_state(&enqueue_state, ObjectInfo::default(), &enqueue_cancel).await
+        });
+        tokio::task::yield_now().await;
+        assert!(!enqueue.is_finished(), "enqueue should be blocked on the state lock");
+        cancel.cancel();
+        let queued = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
+            .await
+            .expect("cancellation must not wait for the state write guard")
+            .expect("enqueue should return cleanly");
+        assert!(!queued);
+        assert_eq!(held.pending_tasks(), 0);
+        assert_eq!(held.stats.missed_free_vers_tasks(), 0);
+        assert!(rx.try_recv().is_err(), "a cancelled recovery must not enqueue a task");
+        drop(held);
+        assert!(!enqueue_recovered_free_version_with_state(&state, ObjectInfo::default(), &cancel).await);
+        assert!(rx.try_recv().is_err(), "cancellation must win even when the lock is available");
+    }
+
     fn free_version_recovery_stats(enqueued: usize, failed: usize, truncated: bool) -> FreeVersionRecoveryStats {
         FreeVersionRecoveryStats {
             scanned: enqueued.saturating_add(failed),
@@ -7680,7 +8050,7 @@ mod tests {
         assert_eq!(state.read().await.stats.pending_tasks(), 1);
 
         // The single-slot queue is full for both enqueue paths.
-        assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
+        assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await);
         assert_eq!(state.read().await.stats.pending_tasks(), 1);
         assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
 
@@ -7712,7 +8082,7 @@ mod tests {
             ..Default::default()
         };
 
-        let queued = enqueue_recovered_free_version_with_state(&state, oi).await;
+        let queued = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
         let state = state.read().await;
 
         assert!(!queued);
@@ -7841,8 +8211,8 @@ mod tests {
             ..Default::default()
         };
 
-        let first = enqueue_recovered_free_version_with_state(&state, oi.clone()).await;
-        let second = enqueue_recovered_free_version_with_state(&state, oi).await;
+        let first = enqueue_recovered_free_version_with_state(&state, oi.clone(), &CancellationToken::new()).await;
+        let second = enqueue_recovered_free_version_with_state(&state, oi, &CancellationToken::new()).await;
         let state = state.read().await;
 
         assert!(first);
@@ -7943,7 +8313,7 @@ mod tests {
         };
 
         assert!(
-            super::enqueue_recovered_free_version(&ecstore, oi).await,
+            super::enqueue_recovered_free_version_with_cancel(&ecstore, oi, &CancellationToken::new()).await,
             "the resized production worker queue should accept the task"
         );
         stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -13837,6 +14207,10 @@ mod tests {
         .await;
     }
 
+    fn fast_tier_free_version_recovery_leader_retry_delay() -> StdDuration {
+        StdDuration::from_millis(10)
+    }
+
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
@@ -13852,8 +14226,13 @@ mod tests {
         seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
 
         let started = OnceLock::new();
-        let recovery = super::spawn_tier_free_version_recovery_once(Arc::clone(&ecstore), &started)
-            .expect("production recovery entrypoint should start once");
+        let recovery = super::spawn_tier_free_version_recovery_once_with_options(
+            Arc::clone(&ecstore),
+            &started,
+            format!("tier-free-version-recovery/test/{}", Uuid::new_v4()),
+            fast_tier_free_version_recovery_leader_retry_delay,
+        )
+        .expect("production recovery entrypoint should start once");
         let task = tokio::time::timeout(StdDuration::from_secs(30), async { recovery_rx.lock().await.recv().await })
             .await
             .expect("production recovery entrypoint should enqueue the seeded item")
@@ -13896,13 +14275,14 @@ mod tests {
         };
         let mut recovery_rx = recovery_rx.lock().await;
         assert!(
-            super::enqueue_recovered_free_version(
+            super::enqueue_recovered_free_version_with_cancel(
                 &ecstore,
                 ObjectInfo {
                     bucket: "prefill".to_string(),
                     name: "prefill".to_string(),
                     ..Default::default()
                 },
+                &CancellationToken::new(),
             )
             .await,
             "the production recovery queue should accept its first task"

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -20,7 +20,7 @@ use std::time::Duration;
 use tokio::sync::mpsc;
 use tokio_util::sync::CancellationToken;
 
-use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version;
+use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version_with_cancel;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::Result;
 use crate::object_api::ObjectInfo;
@@ -201,7 +201,11 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
             return Err(tier_free_version_recovery_cancelled());
         }
         retry_cursor.visit(&oi);
-        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
+        let queued = enqueue_recovered_free_version_with_cancel(&api, oi, &cancel_token).await;
+        if cancel_token.is_cancelled() {
+            return Err(tier_free_version_recovery_cancelled());
+        }
+        if !record_recovered_free_version_enqueue(&mut stats, queued) {
             let (bucket_marker, object_marker) = retry_cursor.retry_markers();
             stats.truncated = true;
             stats.next_bucket_marker = bucket_marker;

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1153,7 +1153,7 @@ async fn acquire_transaction_lock_with_sys(
 /// agree on, so renaming it would leave a mixed-version cluster with two
 /// disjoint keys — and old and new nodes would stop excluding each other on
 /// the very writes that are serialized today.
-fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
+pub(crate) fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
     format!("bucket-targets/{bucket}/transaction.lock")
 }
 
@@ -1292,14 +1292,34 @@ pub(crate) async fn get_object_lock_config_state_in(
 pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
+    get_object_lock_config_and_incarnation_from_disk_with_guard_in(ctx, bucket, None).await
+}
+
+/// Bucket deletion already owns the namespace/lifecycle fences. Reuse its
+/// live guard rather than recursively acquiring the bucket namespace lock.
+/// The caller must also hold the metadata transaction read lock.
+pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
     let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
 
-    match bucket_meta_sys
-        .read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
-        .await?
-    {
+    let authority = match guard {
+        Some(guard) => {
+            bucket_meta_sys
+                .read_authoritative_metadata_from_disk_under_guard(bucket, guard)
+                .await?
+        }
+        None => {
+            bucket_meta_sys
+                .read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
+                .await?
+        }
+    };
+    match authority {
         BucketMetadataAuthority::Authoritative(metadata)
             if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
         {

crates/ecstore/src/bucket/quota/mod.rs

@@ -32,7 +32,10 @@ pub enum QuotaType {
     Hard,
 }
 
-pub(crate) const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
+pub const QUOTA_RESERVATION_PROTOCOL_V1: u32 = 1;
+/// Sharded durable reservations. Older nodes reject this protocol instead of
+/// silently falling back to the single-ledger protocol.
+pub const QUOTA_RESERVATION_PROTOCOL_V2: u32 = 2;
 
 /// Bucket quota configuration. quota_type defaults to Hard when omitted.
 #[derive(Debug, Default, Clone, PartialEq)]
@@ -89,7 +92,10 @@ impl<'de> Deserialize<'de> for BucketQuota {
         D: Deserializer<'de>,
     {
         let wire = BucketQuotaWire::deserialize(deserializer)?;
-        let quota = if wire.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1) {
+        let quota = if matches!(
+            wire.reservation_protocol,
+            Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
+        ) {
             Some(
                 wire.reservation_quota
                     .ok_or_else(|| D::Error::custom("reservation_quota is required for reservation protocol v1"))?,
@@ -129,17 +135,33 @@ impl BucketQuota {
         }
     }
 
+    /// Creates a quota using the sharded durable reservation protocol.
+    /// Deployment must complete the mixed-version rollout before enabling it;
+    /// older nodes fail closed when they see protocol v2.
+    pub fn new_sharded(quota: u64) -> Self {
+        let mut config = Self::new(Some(quota));
+        config.reservation_protocol = Some(QUOTA_RESERVATION_PROTOCOL_V2);
+        config
+    }
+
     pub fn get_quota_limit(&self) -> Option<u64> {
         self.quota
     }
 
     pub fn uses_durable_reservations(&self) -> bool {
-        self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V1)
+        matches!(
+            self.reservation_protocol,
+            Some(QUOTA_RESERVATION_PROTOCOL_V1) | Some(QUOTA_RESERVATION_PROTOCOL_V2)
+        )
+    }
+
+    pub fn uses_sharded_reservations(&self) -> bool {
+        self.reservation_protocol == Some(QUOTA_RESERVATION_PROTOCOL_V2)
     }
 
     pub fn has_unsupported_reservation_protocol(&self) -> bool {
         self.reservation_protocol
-            .is_some_and(|version| version != QUOTA_RESERVATION_PROTOCOL_V1)
+            .is_some_and(|version| !matches!(version, QUOTA_RESERVATION_PROTOCOL_V1 | QUOTA_RESERVATION_PROTOCOL_V2))
     }
 
     pub fn check_operation_allowed(&self, current_usage: u64, operation_size: u64) -> bool {
@@ -322,9 +344,21 @@ mod tests {
     }
 
     #[test]
-    fn unknown_reservation_protocol_does_not_activate_v1() {
+    fn sharded_quota_roundtrips_with_authoritative_reservation_quota() {
+        let quota = BucketQuota::new_sharded(4096);
+        let json = serde_json::to_value(&quota).expect("sharded quota should serialize");
+        assert_eq!(json["quota"], 0);
+        assert_eq!(json["reservation_protocol"], QUOTA_RESERVATION_PROTOCOL_V2);
+        assert_eq!(json["reservation_quota"], 4096);
+        let restored: BucketQuota = serde_json::from_value(json).expect("sharded quota should parse");
+        assert_eq!(restored.quota, Some(4096));
+        assert!(restored.uses_sharded_reservations());
+    }
+
+    #[test]
+    fn unknown_reservation_protocol_does_not_activate_any_known_protocol() {
         let quota: BucketQuota =
-            serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":2,"reservation_quota":2048}"#)
+            serde_json::from_str(r#"{"quota":0,"quota_type":"Hard","reservation_protocol":3,"reservation_quota":2048}"#)
                 .expect("future protocol should remain parseable");
 
         assert!(!quota.uses_durable_reservations());

crates/ecstore/src/config/com.rs

@@ -449,6 +449,22 @@ where
     Ok(data)
 }
 
+pub(crate) async fn read_config_observation<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    let opts = ObjectOptions {
+        suppress_read_repair: true,
+        no_lock: false,
+        skip_verify_bitrot: false,
+        ..Default::default()
+    };
+    crate::object_api::without_get_object_body_cache_hook(read_config_limited_preserve_empty_with_metadata_opts(
+        api, file, &opts, max_bytes,
+    ))
+    .await
+}
+
 pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
 where
     S: EcstoreObjectIO,
@@ -495,10 +511,22 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
 where
     S: EcstoreObjectIO,
 {
-    let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
+    let (data, _obj) = read_config_preserve_empty_with_metadata_opts(api, file, &ObjectOptions::default()).await?;
     Ok(data)
 }
 
+/// Preserve an empty config payload, its metadata, and caller-supplied read options.
+pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
+    api: Arc<S>,
+    file: &str,
+    opts: &ObjectOptions,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, opts, true, None).await
+}
+
 pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
 where
     S: ObjectIO<
@@ -527,15 +555,13 @@ pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc
 where
     S: EcstoreObjectIO,
 {
-    read_config_with_metadata_inner(
+    read_config_preserve_empty_with_metadata_opts(
         api,
         file,
         &ObjectOptions {
             no_lock: true,
             ..Default::default()
         },
-        true,
-        None,
     )
     .await
 }
@@ -2815,9 +2841,10 @@ mod tests {
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
         heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
         new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
-        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
-        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
-        server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
+        read_config_preserve_empty_with_metadata_opts, read_config_with_metadata, read_config_without_migrate,
+        read_server_config_snapshot, save_config_with_opts_inner, save_server_config, save_server_config_snapshot,
+        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
+        storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
     use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
@@ -5065,6 +5092,7 @@ mod tests {
         heal_replacement: Option<Vec<u8>>,
         heal_calls: AtomicUsize,
         write_calls: AtomicUsize,
+        last_get_no_lock: AtomicBool,
         last_put_no_lock: AtomicBool,
         last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
         revision: AtomicUsize,
@@ -5080,6 +5108,7 @@ mod tests {
                 heal_replacement,
                 heal_calls: AtomicUsize::new(0),
                 write_calls: AtomicUsize::new(0),
+                last_get_no_lock: AtomicBool::new(false),
                 last_put_no_lock: AtomicBool::new(false),
                 last_put_preconditions: Mutex::new(None),
                 revision: AtomicUsize::new(1),
@@ -5135,8 +5164,9 @@ mod tests {
             _object: &str,
             _range: Option<HTTPRangeSpec>,
             _h: HeaderMap,
-            _opts: &ObjectOptions,
+            opts: &ObjectOptions,
         ) -> Result<GetObjectReader> {
+            self.last_get_no_lock.store(opts.no_lock, Ordering::SeqCst);
             let data = match &*self.state.lock().expect("state lock poisoned") {
                 RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
                 RecoveryReadState::Blob(data) => data.clone(),
@@ -5411,6 +5441,166 @@ mod tests {
         assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
     }
 
+    struct ObservationStream {
+        inner: Cursor<Vec<u8>>,
+        consumed: Arc<AtomicUsize>,
+        fail: bool,
+    }
+
+    impl AsyncRead for ObservationStream {
+        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+            if self.fail {
+                return Poll::Ready(Err(std::io::Error::other("observation stream failed")));
+            }
+            let before = buf.filled().len();
+            let result = Pin::new(&mut self.inner).poll_read(cx, buf);
+            self.consumed.fetch_add(buf.filled().len() - before, Ordering::SeqCst);
+            result
+        }
+    }
+
+    struct ObservationStore {
+        body: Vec<u8>,
+        declared_size: i64,
+        error: Option<Error>,
+        stream_error: bool,
+        consumed: Arc<AtomicUsize>,
+        options: std::sync::Mutex<Option<(bool, bool, bool)>>,
+        writes: AtomicUsize,
+    }
+
+    impl Debug for ObservationStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("ObservationStore").finish()
+        }
+    }
+
+    impl ObservationStore {
+        fn new(body: Vec<u8>, declared_size: i64) -> Self {
+            Self {
+                body,
+                declared_size,
+                error: None,
+                stream_error: false,
+                consumed: Arc::new(AtomicUsize::new(0)),
+                options: std::sync::Mutex::new(None),
+                writes: AtomicUsize::new(0),
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for ObservationStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            bucket: &str,
+            object: &str,
+            range: Option<HTTPRangeSpec>,
+            _headers: HeaderMap,
+            opts: &ObjectOptions,
+        ) -> Result<GetObjectReader> {
+            assert_eq!(bucket, RUSTFS_META_BUCKET);
+            assert_eq!(object, "config/observation.bin");
+            assert!(range.is_none());
+            *self.options.lock().expect("observation options") =
+                Some((opts.suppress_read_repair, opts.no_lock, opts.skip_verify_bitrot));
+            if let Some(error) = &self.error {
+                return Err(error.clone());
+            }
+            Ok(GetObjectReader {
+                stream: Box::new(ObservationStream {
+                    inner: Cursor::new(self.body.clone()),
+                    consumed: self.consumed.clone(),
+                    fail: self.stream_error,
+                }),
+                object_info: ObjectInfo {
+                    size: self.declared_size,
+                    etag: Some("observation-etag".to_string()),
+                    ..Default::default()
+                },
+                buffered_body: None,
+                body_source: Default::default(),
+            })
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut PutObjReader,
+            _opts: &ObjectOptions,
+        ) -> Result<ObjectInfo> {
+            self.writes.fetch_add(1, Ordering::SeqCst);
+            Err(Error::other("observation must not write"))
+        }
+    }
+
+    #[tokio::test]
+    async fn observation_preserves_options_and_errors() {
+        for body in [Vec::new(), b"raw config".to_vec()] {
+            let store = Arc::new(ObservationStore::new(body.clone(), i64::try_from(body.len()).expect("fixture size")));
+            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+            assert!(result.is_ok(), "observation must preserve empty and nonempty objects: {result:?}");
+            let (observed, info) = result.expect("successful observation");
+            assert_eq!(observed, body);
+            assert_eq!(info.etag.as_deref(), Some("observation-etag"));
+            assert_eq!(
+                *store.options.lock().expect("options"),
+                Some((true, false, false)),
+                "I05_REPAIR_SUPPRESSED"
+            );
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+        for (source, expected) in [
+            (Error::FileNotFound, Error::ConfigNotFound),
+            (
+                Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "config/observation.bin".to_string()),
+                Error::ConfigNotFound,
+            ),
+            (Error::ErasureReadQuorum, Error::ErasureReadQuorum),
+        ] {
+            let mut fixture = ObservationStore::new(Vec::new(), 0);
+            fixture.error = Some(source);
+            let store = Arc::new(fixture);
+            let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+            assert_eq!(result.expect_err("read error must propagate"), expected);
+            assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+        }
+        let mut fixture = ObservationStore::new(Vec::new(), 0);
+        fixture.stream_error = true;
+        let store = Arc::new(fixture);
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 32).await;
+        let error = result.expect_err("stream error must propagate");
+        assert!(error.to_string().contains("observation stream failed"));
+        assert_eq!(store.writes.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn observation_enforces_byte_limit() {
+        let store = Arc::new(ObservationStore::new(vec![7; 64], 0));
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
+        assert_eq!(result.is_err(), true, "I05_BOUND_ENFORCED");
+        assert_eq!(
+            result.expect_err("bounded observation error"),
+            Error::CorruptedFormat,
+            "I05_BOUND_ENFORCED"
+        );
+        assert_eq!(store.consumed.load(Ordering::SeqCst), 9, "I05_BOUND_ENFORCED: drain exactly limit + 1");
+        let store = Arc::new(ObservationStore::new(vec![7; 8], 8));
+        let result = super::read_config_observation(store.clone(), "config/observation.bin", 8).await;
+        assert!(result.is_ok(), "exact byte limit must succeed: {result:?}");
+        assert_eq!(result.expect("exact limit").0, vec![7; 8]);
+        assert_eq!(store.consumed.load(Ordering::SeqCst), 8);
+    }
+
     #[tokio::test]
     async fn read_config_preserve_empty_distinguishes_empty_object() {
         let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
@@ -5431,6 +5621,60 @@ mod tests {
         assert!(data.is_empty());
     }
 
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_keeps_empty_and_options() {
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(Vec::new()), None));
+        for no_lock in [false, true] {
+            let opts = ObjectOptions {
+                no_lock,
+                ..Default::default()
+            };
+            let (data, info) = read_config_preserve_empty_with_metadata_opts(store.clone(), "config/empty.json", &opts)
+                .await
+                .expect("an existing empty config must retain its payload and metadata");
+            assert!(data.is_empty());
+            assert_eq!(info.size, 0);
+            assert_eq!(info.actual_size, 0);
+            assert_eq!(info.etag.as_deref(), Some("config-1"));
+            assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
+            assert_eq!(store.last_get_no_lock.load(Ordering::SeqCst), no_lock);
+        }
+        let err = read_config(store, "config/empty.json")
+            .await
+            .expect_err("the existing reader must still treat empty configs as missing");
+        assert!(matches!(err, Error::ConfigNotFound));
+    }
+
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_keeps_payload_and_metadata() {
+        let payload = b"original config bytes".to_vec();
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(payload.clone()), None));
+        let (data, info) =
+            read_config_preserve_empty_with_metadata_opts(store, "config/nonempty.json", &ObjectOptions::default())
+                .await
+                .expect("a nonempty config must retain its payload and metadata");
+        assert_eq!(data, payload);
+        let size = i64::try_from(payload.len()).expect("test payload size fits i64");
+        assert_eq!(info.size, size);
+        assert_eq!(info.actual_size, size);
+        assert_eq!(info.etag.as_deref(), Some("config-1"));
+        assert_eq!(info.data_dir, Some(uuid::Uuid::from_u128(1)));
+    }
+
+    #[tokio::test]
+    async fn read_config_preserve_empty_metadata_preserves_read_errors() {
+        let missing = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
+        let err = read_config_preserve_empty_with_metadata_opts(missing, "config/missing.json", &ObjectOptions::default())
+            .await
+            .expect_err("physical absence must remain ConfigNotFound");
+        assert!(matches!(err, Error::ConfigNotFound));
+        let quorum = Arc::new(RecoveryMockStore::new(RecoveryReadState::QuorumError, None));
+        let err = read_config_preserve_empty_with_metadata_opts(quorum, "config/quorum.json", &ObjectOptions::default())
+            .await
+            .expect_err("a quorum failure must retain its typed read error");
+        assert!(matches!(err, Error::ErasureReadQuorum));
+    }
+
     #[async_trait::async_trait]
     impl StorageAdminApi for RecoveryMockStore {
         type BackendInfo = rustfs_madmin::BackendInfo;

crates/ecstore/src/data_usage/mod.rs

@@ -738,7 +738,12 @@ where
         .filter(|etag| !etag.is_empty())
         .map(str::to_owned)
         .ok_or_else(|| Error::other("data usage snapshot has no ETag"))?;
-    let mut data_usage_info = parse_usage_snapshot(&reader.read_all().await?)?;
+    let mut data_usage_info = parse_usage_snapshot(
+        &reader
+            .read_all()
+            .await
+            .map_err(|err| map_data_usage_metadata_read_error(err, object))?,
+    )?;
     populate_backward_compatible_usage_maps(&mut data_usage_info);
     validate_complete_usage_snapshot(&mut data_usage_info);
     Ok(Some((data_usage_info, revision)))
@@ -2997,6 +3002,61 @@ mod tests {
         }
     }
 
+    #[derive(Debug)]
+    struct UsageSnapshotReadFailureStore;
+
+    struct UsageSnapshotFailureReader;
+
+    impl tokio::io::AsyncRead for UsageSnapshotFailureReader {
+        fn poll_read(
+            self: std::pin::Pin<&mut Self>,
+            _cx: &mut std::task::Context<'_>,
+            _buf: &mut tokio::io::ReadBuf<'_>,
+        ) -> std::task::Poll<std::io::Result<()>> {
+            std::task::Poll::Ready(Err(std::io::Error::other(Error::VolumeNotFound)))
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for UsageSnapshotReadFailureStore {
+        type Error = Error;
+        type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec;
+        type HeaderMap = http::HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = crate::object_api::GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _range: Option<Self::RangeSpec>,
+            _headers: Self::HeaderMap,
+            _opts: &Self::ObjectOptions,
+        ) -> Result<Self::GetObjectReader, Self::Error> {
+            Ok(crate::object_api::GetObjectReader {
+                stream: Box::new(UsageSnapshotFailureReader),
+                object_info: ObjectInfo {
+                    etag: Some("usage-revision".to_string()),
+                    ..Default::default()
+                },
+                buffered_body: None,
+                body_source: Default::default(),
+            })
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut Self::PutObjectReader,
+            _opts: &Self::ObjectOptions,
+        ) -> Result<Self::ObjectInfo, Self::Error> {
+            unimplemented!("the read-failure fixture does not write")
+        }
+    }
+
     /// Minimal ObjectIO backing `load_data_usage_cache` tests: records the keys
     /// read and fails the first N reads with a transient (non-absence) error.
     #[derive(Debug, Default)]
@@ -3161,6 +3221,20 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn data_usage_removal_maps_missing_system_volume_during_snapshot_stream_read() {
+        let store = UsageSnapshotReadFailureStore;
+
+        let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
+            .await
+            .expect_err("a system volume disappearing during body read must remain a retryable storage error");
+
+        assert_eq!(
+            error,
+            Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
+        );
+    }
+
     #[tokio::test]
     async fn load_data_usage_cache_retries_a_transient_failure() {
         let name = "usage-cache";

crates/ecstore/src/diagnostics/object_lock.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use rustfs_io_metrics::record_object_lock_acquire_failure;
 use rustfs_lock::{LockError, NamespaceLockGuard, NamespaceLockWrapper};
 use std::time::{Duration, Instant};
 use tracing::Span;
@@ -21,6 +22,26 @@ const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
 
+pub(crate) fn record_lock_acquire_failure(op: &'static str, bucket: &str, object: &str, mode: &'static str, error: &LockError) {
+    let resource_class = if bucket == crate::disk::RUSTFS_META_MULTIPART_BUCKET {
+        "multipart_metadata"
+    } else if bucket == crate::disk::RUSTFS_META_BUCKET || bucket == ".minio.sys" {
+        "system_metadata"
+    } else if object.is_empty() || object == bucket {
+        "bucket_namespace"
+    } else {
+        "object_namespace"
+    };
+    let reason = match error {
+        LockError::Timeout { .. } => "timeout",
+        LockError::AlreadyLocked { .. } => "contention",
+        LockError::QuorumNotReached { .. } | LockError::InsufficientNodes { .. } => "quorum",
+        LockError::Network { .. } | LockError::Internal { .. } => "transport",
+        _ => "other",
+    };
+    record_object_lock_acquire_failure(op, mode, resource_class, reason);
+}
+
 /// Diagnostic lifetime only; it never owns or changes a namespace lock.
 /// Keep it alongside the real guard when ownership moves to a commit tail.
 #[derive(Default)]

crates/ecstore/src/disk/disk_store.rs

@@ -39,13 +39,15 @@ use std::sync::atomic::AtomicBool;
 use std::{
     collections::HashMap,
     path::PathBuf,
+    pin::Pin,
     sync::{
         Arc, LazyLock, RwLock as StdRwLock,
         atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
     },
+    task::{Context, Poll},
     time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
-use tokio::{sync::RwLock, time};
+use tokio::{io::AsyncWrite, sync::RwLock, time};
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use uuid::Uuid;
@@ -58,6 +60,121 @@ const LOG_SUBSYSTEM_DISK: &str = "disk";
 const EVENT_DISK_HEALTH_CHECK_FAILED: &str = "disk_health_check_failed";
 const EVENT_DISK_RECOVERY_PROBE_STATE: &str = "disk_recovery_probe_state";
 const EVENT_DISK_TIMEOUT_POLICY_FALLBACK: &str = "disk_timeout_policy_fallback";
+const EVENT_DISK_OPERATION_SLOW: &str = "disk_operation_slow";
+const SLOW_DISK_OPERATION_THRESHOLD: Duration = Duration::from_secs(5);
+
+fn warn_slow_disk_operation(endpoint: &Endpoint, op: &'static str, elapsed: Duration) {
+    warn!(
+        event = EVENT_DISK_OPERATION_SLOW,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_DISK,
+        state = "pending",
+        endpoint = %endpoint,
+        op,
+        elapsed_ms = elapsed.as_millis(),
+        threshold_ms = SLOW_DISK_OPERATION_THRESHOLD.as_millis(),
+        "Disk operation is still pending"
+    );
+}
+
+async fn observe_disk_operation<F: std::future::Future>(
+    endpoint: impl FnOnce() -> Endpoint,
+    op: &'static str,
+    operation: F,
+) -> F::Output {
+    if !tracing::enabled!(tracing::Level::WARN) {
+        return operation.await;
+    }
+    let started = time::Instant::now();
+    tokio::pin!(operation);
+    tokio::select! {
+        biased;
+        result = &mut operation => result,
+        _ = time::sleep(SLOW_DISK_OPERATION_THRESHOLD) => {
+            warn_slow_disk_operation(&endpoint(), op, started.elapsed());
+            operation.await
+        }
+    }
+}
+
+struct SlowDiskWriter {
+    inner: crate::disk::FileWriter,
+    endpoint: Endpoint,
+    pending: Option<(&'static str, time::Instant)>,
+    timer: Option<Pin<Box<time::Sleep>>>,
+    warned: bool,
+}
+
+impl SlowDiskWriter {
+    fn new(inner: crate::disk::FileWriter, endpoint: Endpoint) -> Self {
+        Self {
+            inner,
+            endpoint,
+            pending: None,
+            timer: None,
+            warned: false,
+        }
+    }
+
+    fn observe<T>(&mut self, cx: &mut Context<'_>, op: &'static str, result: Poll<T>) -> Poll<T> {
+        if result.is_ready() {
+            self.pending = None;
+            self.warned = false;
+        } else if tracing::enabled!(tracing::Level::WARN) {
+            if self.pending.is_none_or(|(pending_op, _)| pending_op != op) {
+                let started = time::Instant::now();
+                self.pending = Some((op, started));
+                self.warned = false;
+                let deadline = started + SLOW_DISK_OPERATION_THRESHOLD;
+                // Reuse one timer per stream; ready writes need no timer allocation.
+                if let Some(timer) = &mut self.timer {
+                    timer.as_mut().reset(deadline);
+                } else {
+                    self.timer = Some(Box::pin(time::sleep_until(deadline)));
+                }
+            }
+            if !self.warned
+                && let Some(timer) = &mut self.timer
+                && std::future::Future::poll(timer.as_mut(), cx).is_ready()
+                && let Some((_, started)) = self.pending
+            {
+                warn_slow_disk_operation(&self.endpoint, op, started.elapsed());
+                self.warned = true;
+            }
+        }
+        result
+    }
+}
+
+impl AsyncWrite for SlowDiskWriter {
+    fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
+        let result = Pin::new(&mut self.inner).poll_write(cx, buf);
+        self.observe(cx, "write", result)
+    }
+
+    fn poll_write_vectored(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[std::io::IoSlice<'_>],
+    ) -> Poll<std::io::Result<usize>> {
+        let result = Pin::new(&mut self.inner).poll_write_vectored(cx, bufs);
+        self.observe(cx, "write_vectored", result)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored()
+    }
+
+    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+        let result = Pin::new(&mut self.inner).poll_flush(cx);
+        self.observe(cx, "flush", result)
+    }
+
+    fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+        let result = Pin::new(&mut self.inner).poll_shutdown(cx);
+        self.observe(cx, "shutdown", result)
+    }
+}
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 enum TimeoutHealthAction {
@@ -2019,9 +2136,10 @@ impl LocalDiskWrapper {
         let _waiting_guard = self.health.waiting_guard();
         let _metric_waiting_guard = self.metrics.waiting_guard();
         let started = Instant::now();
+        let operation = observe_disk_operation(|| self.endpoint(), op, operation(io_permit));
 
         if timeout_duration == Duration::ZERO {
-            let result = operation(io_permit).await;
+            let result = operation.await;
             self.metrics.record_operation_latency(op, started.elapsed());
             self.record_result_error_metrics(&result);
             if result.is_ok() {
@@ -2031,7 +2149,7 @@ impl LocalDiskWrapper {
             return result;
         }
         // Execute the operation with timeout
-        let result = tokio::time::timeout(timeout_duration, operation(io_permit)).await;
+        let result = tokio::time::timeout(timeout_duration, operation).await;
 
         match result {
             Ok(operation_result) => {
@@ -2311,7 +2429,7 @@ impl DiskAPI for LocalDiskWrapper {
 
         // Record operation start
         self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
-        self.health.increment_waiting();
+        let waiting_guard = self.health.waiting_guard();
         let metric_waiting_guard = self.metrics.waiting_guard();
         let started = Instant::now();
 
@@ -2320,7 +2438,7 @@ impl DiskAPI for LocalDiskWrapper {
         self.metrics.record_operation_latency("delete_versions", started.elapsed());
         self.record_batch_delete_error_metrics(&result);
 
-        self.health.decrement_waiting();
+        drop(waiting_guard);
         drop(metric_waiting_guard);
         let has_err = result.iter().any(|e| e.is_some());
         if !has_err {
@@ -2545,18 +2663,22 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
-        self.track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
-            .await
+        let writer = self
+            .track_disk_health_with_op("append_file", || async { self.disk.append_file(volume, path).await }, Duration::ZERO)
+            .await?;
+        Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
     }
 
     async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<crate::disk::FileWriter> {
-        self.track_disk_health_mutation(
-            "create_file",
-            DiskMetricMutation::Write,
-            || async { self.disk.create_file(origvolume, volume, path, file_size).await },
-            Duration::ZERO,
-        )
-        .await
+        let writer = self
+            .track_disk_health_mutation(
+                "create_file",
+                DiskMetricMutation::Write,
+                || async { self.disk.create_file(origvolume, volume, path, file_size).await },
+                Duration::ZERO,
+            )
+            .await?;
+        Ok(Box::new(SlowDiskWriter::new(writer, self.endpoint())))
     }
 
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
@@ -2709,7 +2831,268 @@ mod tests {
         pin::Pin,
         task::{Context, Poll},
     };
-    use tokio::io::AsyncWrite;
+    use tokio::io::AsyncWriteExt;
+    use tracing_subscriber::fmt::MakeWriter;
+
+    #[derive(Clone, Default)]
+    struct SlowOperationLogs(Arc<std::sync::Mutex<Vec<u8>>>);
+
+    impl SlowOperationLogs {
+        fn contents(&self) -> String {
+            String::from_utf8(self.0.lock().expect("log buffer should not be poisoned").clone()).expect("logs should be UTF-8")
+        }
+    }
+
+    impl io::Write for SlowOperationLogs {
+        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+            self.0
+                .lock()
+                .expect("log buffer should not be poisoned")
+                .extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for SlowOperationLogs {
+        type Writer = Self;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn slow_disk_operation_warns_once_while_pending_and_preserves_result() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("path should be UTF-8")).expect("endpoint should parse");
+        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+        let wrapper = LocalDiskWrapper::new(disk, false);
+        let logs = SlowOperationLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let _interest = crate::test_tracing::pin_callsite_interest_for_test();
+        let (release, wait) = tokio::sync::oneshot::channel();
+        let operation = wrapper.track_disk_health_with_op(
+            "rename_data",
+            || async { wait.await.expect("operation should be released") },
+            Duration::ZERO,
+        );
+        tokio::pin!(operation);
+        assert!(futures::poll!(&mut operation).is_pending());
+        time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
+        assert!(futures::poll!(&mut operation).is_pending());
+        assert!(logs.contents().is_empty(), "normal disk latency should not warn");
+        time::advance(Duration::from_millis(1)).await;
+        assert!(futures::poll!(&mut operation).is_pending());
+        let warning = logs.contents();
+        assert!(warning.contains("WARN"));
+        assert!(warning.contains("event=\"disk_operation_slow\""));
+        assert!(warning.contains("state=\"pending\""));
+        assert!(warning.contains(&format!("endpoint={endpoint}")));
+        assert!(warning.contains("op=\"rename_data\""));
+        assert!(warning.contains("elapsed_ms=5000"));
+        time::advance(Duration::from_secs(120)).await;
+        assert!(futures::poll!(&mut operation).is_pending(), "warning must not cancel the disk operation");
+        assert_eq!(logs.contents(), warning, "a pending operation should warn only once");
+        release
+            .send(Err::<(), _>(DiskError::FileAccessDenied))
+            .expect("operation should still be alive");
+        assert_eq!(operation.await, Err(DiskError::FileAccessDenied));
+        assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
+        assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
+
+        let timed = wrapper.track_disk_health_with_op(
+            "write_all",
+            std::future::pending::<Result<()>>,
+            SLOW_DISK_OPERATION_THRESHOLD + Duration::from_secs(1),
+        );
+        tokio::pin!(timed);
+        assert!(futures::poll!(&mut timed).is_pending());
+        time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
+        assert!(futures::poll!(&mut timed).is_pending());
+        assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 2);
+        assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
+        time::advance(Duration::from_secs(1)).await;
+        assert_eq!(
+            timed.await,
+            Err(DiskError::Timeout),
+            "existing timeout must still apply after the warning"
+        );
+        assert_eq!(wrapper.metrics_snapshot().total_errors_timeout, 1);
+        assert_eq!(wrapper.metrics_snapshot().total_waiting, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn slow_disk_operation_fast_path_is_silent_and_does_not_resolve_endpoint() {
+        let logs = SlowOperationLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let _interest = crate::test_tracing::pin_callsite_interest_for_test();
+        for delay in [Duration::ZERO, SLOW_DISK_OPERATION_THRESHOLD] {
+            let result =
+                observe_disk_operation(|| panic!("completed operation should not clone its endpoint"), "write_all", async {
+                    if !delay.is_zero() {
+                        time::sleep(delay).await;
+                    }
+                    Ok::<_, DiskError>(17)
+                })
+                .await;
+            assert_eq!(result, Ok(17));
+        }
+        assert!(logs.contents().is_empty());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn slow_disk_diagnostics_are_bypassed_when_warnings_are_disabled() {
+        let _subscriber = tracing::subscriber::set_default(tracing::subscriber::NoSubscriber::default());
+        let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
+        let mut writer = SlowDiskWriter::new(Box::new(PendingWriter), endpoint);
+        {
+            let write = writer.write(b"bytes");
+            tokio::pin!(write);
+            assert!(futures::poll!(&mut write).is_pending());
+        }
+        assert!(writer.timer.is_none(), "disabled warnings should not allocate a stream timer");
+        let result = observe_disk_operation(|| panic!("disabled warnings should not resolve endpoints"), "write_all", async {
+            time::sleep(SLOW_DISK_OPERATION_THRESHOLD * 2).await;
+            17
+        })
+        .await;
+        assert_eq!(result, 17);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn slow_disk_writer_preserves_vectored_bytes_and_warns_once_while_blocked() {
+        let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
+        let logs = SlowOperationLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let _interest = crate::test_tracing::pin_callsite_interest_for_test();
+        let (stream, mut receiver) = tokio::io::duplex(1);
+        let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint.clone());
+        writer.write_all(b"a").await.expect("first byte should fit in the buffer");
+        assert!(writer.timer.is_none(), "ready writes should not allocate a timer");
+        assert!(logs.contents().is_empty());
+        let mut received = [0; 1];
+        {
+            let bufs = [io::IoSlice::new(b"b"), io::IoSlice::new(b"c")];
+            let write = writer.write_vectored(&bufs);
+            tokio::pin!(write);
+            assert!(futures::poll!(&mut write).is_pending());
+            time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
+            assert!(futures::poll!(&mut write).is_pending());
+            assert!(logs.contents().contains("op=\"write_vectored\""));
+            time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
+            assert!(futures::poll!(&mut write).is_pending());
+            assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), 1);
+            tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
+                .await
+                .expect("buffered byte should be readable");
+            assert_eq!(received, *b"a");
+            assert_eq!(write.await.expect("blocked write should finish when capacity returns"), 1);
+        }
+        tokio::io::AsyncReadExt::read_exact(&mut receiver, &mut received)
+            .await
+            .expect("written byte should be readable");
+        assert_eq!(received, *b"b");
+
+        drop(receiver);
+        let err = writer.write_all(b"d").await.expect_err("closed receiver should fail writes");
+        assert_eq!(err.kind(), io::ErrorKind::BrokenPipe);
+    }
+
+    #[tokio::test]
+    async fn slow_disk_writer_preserves_full_vectored_write_and_capability() {
+        let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
+        let (stream, mut receiver) = tokio::io::duplex(64);
+        let vectored = stream.is_write_vectored();
+        let mut writer = SlowDiskWriter::new(Box::new(stream), endpoint);
+        assert_eq!(writer.is_write_vectored(), vectored);
+        let bufs = [io::IoSlice::new(b"hash"), io::IoSlice::new(b"data")];
+        assert_eq!(writer.write_vectored(&bufs).await.expect("vectored write should succeed"), 8);
+        writer.shutdown().await.expect("shutdown should succeed");
+        let mut received = Vec::new();
+        tokio::io::AsyncReadExt::read_to_end(&mut receiver, &mut received)
+            .await
+            .expect("all bytes should be readable");
+        assert_eq!(received, b"hashdata");
+        assert!(writer.timer.is_none());
+    }
+
+    struct StalledWriter(Arc<AtomicBool>);
+
+    impl AsyncWrite for StalledWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
+            if self.0.load(Ordering::Relaxed) {
+                Poll::Ready(Ok(buf.len()))
+            } else {
+                Poll::Pending
+            }
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            if self.0.load(Ordering::Relaxed) {
+                Poll::Ready(Ok(()))
+            } else {
+                Poll::Pending
+            }
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            self.poll_flush(cx)
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn slow_disk_writer_warns_for_write_flush_and_shutdown_and_rearms_after_progress() {
+        let endpoint = Endpoint::try_from("/slow-disk").expect("endpoint should parse");
+        let logs = SlowOperationLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let _interest = crate::test_tracing::pin_callsite_interest_for_test();
+        let ready = Arc::new(AtomicBool::new(false));
+        let mut writer = SlowDiskWriter::new(Box::new(StalledWriter(Arc::clone(&ready))), endpoint);
+        for (index, op) in ["write", "flush", "shutdown"].into_iter().enumerate() {
+            ready.store(false, Ordering::Relaxed);
+            let operation = async {
+                match op {
+                    "write" => writer.write_all(b"bytes").await,
+                    "flush" => writer.flush().await,
+                    "shutdown" => writer.shutdown().await,
+                    _ => unreachable!("only write operations are tested"),
+                }
+            };
+            tokio::pin!(operation);
+            assert!(futures::poll!(&mut operation).is_pending());
+            time::advance(SLOW_DISK_OPERATION_THRESHOLD).await;
+            assert!(futures::poll!(&mut operation).is_pending());
+            assert!(logs.contents().contains(&format!("op=\"{op}\"")));
+            assert_eq!(logs.contents().matches(EVENT_DISK_OPERATION_SLOW).count(), index + 1);
+            ready.store(true, Ordering::Relaxed);
+            operation.await.expect("warning should not prevent operation completion");
+        }
+        assert!(writer.pending.is_none());
+    }
 
     struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
 
@@ -3229,6 +3612,40 @@ mod tests {
         assert_eq!(snapshot.total_deletes, 1);
     }
 
+    // A batch delete dropped while the disk call is in flight (for example by a
+    // cancelled caller) must not leave the drive looking busy forever.
+    #[tokio::test]
+    async fn delete_versions_releases_waiting_slot_when_dropped() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse");
+        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
+        wrapper.make_volume("bucket").await.expect("volume should be created");
+        let object_path = disk.get_object_path("bucket", "object").expect("object path");
+        // Holding the object's metadata lease parks the disk call before it mutates anything.
+        let metadata_lease = crate::disk::os::acquire_metadata_mutation_lease(&object_path, None).await;
+
+        let mut delete = Box::pin(wrapper.delete_versions(
+            "bucket",
+            vec![FileInfoVersions {
+                name: "object".to_string(),
+                versions: vec![FileInfo::default()],
+                ..Default::default()
+            }],
+            DeleteOptions::default(),
+        ));
+        assert!(
+            futures::poll!(delete.as_mut()).is_pending(),
+            "the disk call should wait for the metadata lease"
+        );
+        assert_eq!(wrapper.health.waiting_count(), 1);
+
+        drop(delete);
+        assert_eq!(wrapper.health.waiting_count(), 0, "a dropped delete must release its waiting slot");
+        drop(metadata_lease);
+    }
+
     #[tokio::test]
     async fn delete_versions_counts_faulty_drive_availability_rejection() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/io_support/rio.rs

@@ -442,12 +442,19 @@ impl WritePlan {
         }
 
         if let Some(encryption) = self.encryption {
+            let etag_nonce = match encryption.mode {
+                WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
+                    base_nonce
+                }
+                WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
+            };
+            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
             reader = match encryption.mode {
                 WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]
-                    EncryptReader::new_with_object_key(reader, encryption.key_bytes),
+                    EncryptReader::new_with_object_key(protected_reader, encryption.key_bytes),
                     #[cfg(not(feature = "rio-v2"))]
-                    EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
+                    EncryptReader::new(protected_reader, encryption.key_bytes, [0u8; 12]),
                     HashReader::SIZE_PRESERVE_LAYER,
                     actual_size,
                     None,
@@ -457,12 +464,12 @@ impl WritePlan {
                 WriteEncryptionMode::Singlepart { base_nonce } => {
                     #[cfg(not(feature = "rio-v2"))]
                     let encrypt_reader = if encryption_frame_v2_enabled() {
-                        EncryptReader::new_v2(reader, encryption.key_bytes, base_nonce)
+                        EncryptReader::new_v2(protected_reader, encryption.key_bytes, base_nonce)
                     } else {
-                        EncryptReader::new(reader, encryption.key_bytes, base_nonce)
+                        EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce)
                     };
                     #[cfg(feature = "rio-v2")]
-                    let encrypt_reader = EncryptReader::new(reader, encryption.key_bytes, base_nonce);
+                    let encrypt_reader = EncryptReader::new(protected_reader, encryption.key_bytes, base_nonce);
                     HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
                 }
                 WriteEncryptionMode::MultipartLegacy {
@@ -471,20 +478,25 @@ impl WritePlan {
                 } => {
                     #[cfg(not(feature = "rio-v2"))]
                     let encrypt_reader = if encryption_frame_v2_enabled() {
-                        EncryptReader::new_multipart_v2(reader, encryption.key_bytes, base_nonce, multipart_part_number)
+                        EncryptReader::new_multipart_v2(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
                     } else {
-                        EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number)
+                        EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number)
                     };
                     #[cfg(feature = "rio-v2")]
                     let encrypt_reader =
-                        EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number);
+                        EncryptReader::new_multipart(protected_reader, encryption.key_bytes, base_nonce, multipart_part_number);
                     HashReader::from_reader(encrypt_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)?
                 }
                 WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]
-                    EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
+                    EncryptReader::new_multipart_with_object_key(protected_reader, encryption.key_bytes, multipart_part_number),
                     #[cfg(not(feature = "rio-v2"))]
-                    EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
+                    EncryptReader::new_multipart(
+                        protected_reader,
+                        encryption.key_bytes,
+                        [0u8; 12],
+                        multipart_part_number as usize,
+                    ),
                     HashReader::SIZE_PRESERVE_LAYER,
                     actual_size,
                     None,
@@ -497,6 +509,7 @@ impl WritePlan {
         // `ignore_value` deliberately avoids a second hasher over compressed or
         // encrypted bytes. The inner reader still validates the plaintext request
         // checksum while this outer reader exposes the request checksum context.
+        reader.encrypted_etag = encrypted;
         reader.add_non_trailing_checksum(checksum, true)?;
         reader.set_trailer(trailer);
 
@@ -513,6 +526,36 @@ mod tests {
     use std::io::Cursor;
     use tokio::io::AsyncReadExt;
 
+    #[tokio::test]
+    async fn write_plan_encryption_persists_protected_etags_for_every_mode() {
+        let data = b"Salary 2026: 85000 EUR";
+        let size = i64::try_from(data.len()).expect("test data size");
+        let mut control =
+            HashReader::from_stream(Cursor::new(data), size, size, None, None, false).expect("create control reader");
+        control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
+        let md5 = control.try_resolve_etag().expect("control MD5");
+        for encryption in [
+            WriteEncryption::singlepart([1; 32], [1; 12]),
+            WriteEncryption::singlepart_object_key([1; 32]),
+            WriteEncryption::multipart([1; 32], [1; 12], 1),
+            WriteEncryption::multipart_object_key([1; 32], 1),
+        ] {
+            let plaintext = HashReader::from_stream(Cursor::new(data), size, size, Some(md5.clone()), None, false)
+                .expect("create checked reader");
+            let mut reader = WritePlan::new()
+                .with_encryption(encryption)
+                .apply(plaintext, size)
+                .expect("apply encryption");
+            let mut ciphertext = Vec::new();
+            reader.read_to_end(&mut ciphertext).await.expect("stream ciphertext");
+            assert_ne!(ciphertext, data);
+            let tag = reader.try_resolve_etag().expect("resolve persisted encrypted ETag");
+            assert_ne!(tag, md5, "stored ETag must not reveal the plaintext MD5");
+            assert_eq!(tag.len(), 32, "multipart ETags require a 16-byte hexadecimal part tag");
+            assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag));
+        }
+    }
+
     async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
         let plaintext = b"checksum-context-through-write-plan".repeat(256);
         let actual_size = plaintext.len() as i64;

crates/ecstore/src/runtime/global.rs

@@ -307,6 +307,13 @@ pub fn shutdown_background_services() {
     }
 }
 
+/// Wait up to `timeout` for object mutations that outlived their callers
+/// (see `InstanceContext::wait_for_detached_mutations`). Returns `false` when
+/// some are still running at the deadline.
+pub async fn wait_for_detached_mutations(timeout: std::time::Duration) -> bool {
+    current_ctx().wait_for_detached_mutations(timeout).await
+}
+
 /// Set the global lock client (first LocalClient created)
 ///
 /// # Arguments

crates/ecstore/src/runtime/instance.rs

@@ -59,6 +59,8 @@ use std::sync::{
 use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
 use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+use tracing::Instrument as _;
 use uuid::Uuid;
 
 const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
@@ -243,6 +245,10 @@ pub struct InstanceContext {
     suppress_tier_delete_journal_recovery: bool,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
+    /// Object mutations that run on their own task so that dropping the
+    /// caller (for example, an S3 client disconnect) cannot stop them between
+    /// their on-disk steps. Shutdown waits for them after the HTTP drain.
+    detached_mutations: TaskTracker,
 }
 
 impl InstanceContext {
@@ -293,6 +299,7 @@ impl InstanceContext {
             suppress_tier_delete_journal_recovery: false,
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
+            detached_mutations: TaskTracker::new(),
         }
     }
 
@@ -306,6 +313,31 @@ impl InstanceContext {
         self.lock_manager.clone()
     }
 
+    /// Run a multi-step mutation on its own task and wait for its output.
+    ///
+    /// Dropping the returned future leaves the task running to completion, so
+    /// the mutation, together with the locks and guards it owns, never stops
+    /// between two of its on-disk steps because its caller went away.
+    pub(crate) async fn run_detached_mutation<F>(&self, mutation: F) -> Result<F::Output, tokio::task::JoinError>
+    where
+        F: std::future::Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.detached_mutations.spawn(mutation.in_current_span()).await
+    }
+
+    /// Wait up to `timeout` for every detached mutation to finish. Returns
+    /// `false` when some are still running at the deadline.
+    pub async fn wait_for_detached_mutations(&self, timeout: std::time::Duration) -> bool {
+        self.detached_mutations.close();
+        tokio::time::timeout(timeout, self.detached_mutations.wait()).await.is_ok()
+    }
+
+    #[cfg(test)]
+    pub(crate) fn detached_mutation_count(&self) -> usize {
+        self.detached_mutations.len()
+    }
+
     pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
         Arc::clone(&self.data_movement_operation_gate)
     }
@@ -920,6 +952,33 @@ mod tests {
         }
     }
 
+    // A detached mutation keeps running after its caller is dropped, and
+    // shutdown waits for it instead of tearing it down mid-way.
+    #[tokio::test]
+    async fn detached_mutation_outlives_dropped_caller_and_is_awaited() {
+        let ctx = InstanceContext::new();
+        let finished = Arc::new(AtomicBool::new(false));
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
+        let mut caller = Box::pin(ctx.run_detached_mutation({
+            let finished = Arc::clone(&finished);
+            async move {
+                let _ = release_rx.await;
+                finished.store(true, Ordering::Release);
+            }
+        }));
+        assert!(futures::poll!(caller.as_mut()).is_pending());
+        drop(caller);
+        assert_eq!(ctx.detached_mutation_count(), 1, "dropping the caller must not cancel the mutation");
+
+        assert!(
+            !ctx.wait_for_detached_mutations(std::time::Duration::from_millis(50)).await,
+            "the wait must report a mutation that is still running"
+        );
+        release_tx.send(()).expect("the detached mutation should still be waiting");
+        assert!(ctx.wait_for_detached_mutations(std::time::Duration::from_secs(5)).await);
+        assert!(finished.load(Ordering::Acquire));
+    }
+
     // A fresh context (before any update) reflects the initial all-false state.
     #[tokio::test]
     async fn fresh_context_is_all_false() {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -2340,7 +2340,8 @@ enum OrphanDirScan {
     /// under taking parents, root included when anything under it is blocked);
     /// `dirs` is the pre-order list of every directory reached that is not
     /// blocked, and `committed_files` the erasure data and committed delete
-    /// markers found in the unblocked UUID data dirs among them.
+    /// markers found in the unblocked UUID data dirs among them, plus the
+    /// rollback backups those markers prove.
     Scanned {
         blocked: HashSet<String>,
         dirs: Vec<String>,
@@ -2382,10 +2383,15 @@ fn is_safe_orphan_dir_entry(entry: &str) -> bool {
         && !component.contains('\\')
 }
 
-fn is_committed_delete_marker(entry: &str) -> bool {
+fn committed_delete_marker_transaction(entry: &str) -> Option<Uuid> {
     entry
         .strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
-        .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil()))
+        .and_then(|transaction| Uuid::parse_str(transaction).ok())
+        .filter(|uuid| !uuid.is_nil())
+}
+
+fn is_committed_delete_marker(entry: &str) -> bool {
+    committed_delete_marker_transaction(entry).is_some()
 }
 
 /// Outcome of a *post-quorum* `rename_data` commit, classifying whether the
@@ -5149,8 +5155,9 @@ impl SetDisks {
     }
 
     /// Scan a single disk's copy of `prefix` and classify every directory
-    /// under it. Only empty directories and UUID data directories with valid
-    /// committed delete markers are purgeable; anything else blocks its whole
+    /// under it. Only empty directories, UUID data directories with valid
+    /// committed delete markers, and the rollback backup of a delete whose
+    /// marker sits beside it are purgeable; anything else blocks its whole
     /// ancestor chain while sibling subtrees stay purgeable, so committed
     /// residue is still reclaimed when it shares an ancestor with residue from
     /// an older build that never wrote markers (#6898).
@@ -5163,6 +5170,12 @@ impl SetDisks {
         let mut committed_files: Vec<String> = Vec::new();
         let mut blocked: HashSet<String> = HashSet::new();
         let mut existed = false;
+        // `<object>/<T>` directories holding only a delete's rollback backup,
+        // and the `(object, T)` pairs a committed `delete-data.<T>` marker in a
+        // purgeable data dir of that object proves. Both are resolved after the
+        // walk because siblings are visited in no particular order.
+        let mut rollback_backup_dirs: Vec<String> = Vec::new();
+        let mut committed_transactions: HashSet<(String, Uuid)> = HashSet::new();
 
         while let Some(dir) = stack.pop() {
             let entries = match disk.list_dir("", bucket, &dir, 0).await {
@@ -5196,10 +5209,24 @@ impl SetDisks {
             }
 
             if !has_data && !files.is_empty() {
-                let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
+                let (parent, data_dir_name) = dir.rsplit_once(SLASH_SEPARATOR).unwrap_or(("", dir.as_str()));
                 let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
-                let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
-                has_data = !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
+                let committed = files
+                    .iter()
+                    .filter_map(|entry| committed_delete_marker_transaction(entry))
+                    .collect::<Vec<_>>();
+                if is_uuid_data_dir
+                    && dir != root
+                    && child_dirs.is_empty()
+                    && matches!(files.as_slice(), [entry] if entry == STORAGE_FORMAT_FILE_BACKUP)
+                {
+                    rollback_backup_dirs.push(dir);
+                    continue;
+                }
+                has_data = !is_uuid_data_dir || committed.is_empty() || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
+                if !has_data {
+                    committed_transactions.extend(committed.into_iter().map(|transaction| (parent.to_string(), transaction)));
+                }
             }
 
             if has_data {
@@ -5214,6 +5241,22 @@ impl SetDisks {
             stack.extend(child_dirs);
         }
 
+        // A rollback backup is purgeable only when a data dir of the same
+        // object proves its transaction committed on this disk; otherwise it may
+        // still be needed to roll the delete back, so it blocks as before.
+        for dir in rollback_backup_dirs {
+            let proven = dir.rsplit_once(SLASH_SEPARATOR).is_some_and(|(parent, transaction)| {
+                Uuid::parse_str(transaction)
+                    .is_ok_and(|transaction| committed_transactions.contains(&(parent.to_string(), transaction)))
+            });
+            if proven {
+                committed_files.push(format!("{dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}"));
+                dirs.push(dir);
+            } else {
+                block_orphan_dir_chain(&mut blocked, &root, &dir);
+            }
+        }
+
         if existed {
             OrphanDirScan::Scanned {
                 blocked,
@@ -6177,6 +6220,7 @@ pub(crate) mod rename_fanout_barrier {
         ReadVersion,
         MetadataSlowtailFault,
         NonInlineHedgeTimer,
+        LateMetadataRefresh,
     }
 
     impl From<&'static str> for Phase {
@@ -6188,6 +6232,7 @@ pub(crate) mod rename_fanout_barrier {
                 super::rename_fanout_barrier_phase::READ_VERSION => Self::ReadVersion,
                 "metadata_slowtail_fault" => Self::MetadataSlowtailFault,
                 "non_inline_hedge_timer" => Self::NonInlineHedgeTimer,
+                "late_metadata_refresh" => Self::LateMetadataRefresh,
                 _ => panic!("unknown commit fault phase: {label}"),
             }
         }
@@ -6202,6 +6247,9 @@ pub(crate) mod rename_fanout_barrier {
     /// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk.
     pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer;
 
+    /// Object-scoped retry boundary; slot zero identifies the refresh, not a disk.
+    pub const PHASE_LATE_METADATA_REFRESH: Phase = Phase::LateMetadataRefresh;
+
     /// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses.
     struct Armed {
         /// Signalled (task -> test) when the target task reaches the checkpoint.

crates/ecstore/src/set_disk/core/metadata_quorum.rs

@@ -514,6 +514,102 @@ mod tests {
         }
     }
 
+    #[test]
+    fn metadata_observation_mixed_current_identity_arrivals_preserve_selection() {
+        let permutations = (0..4)
+            .flat_map(|a| (0..4).filter(move |b| *b != a).map(move |b| (a, b)))
+            .flat_map(|(a, b)| (0..4).filter(move |c| *c != a && *c != b).map(move |c| (a, b, c)))
+            .map(|(a, b, c)| [a, b, c, 6 - a - b - c])
+            .collect::<Vec<_>>();
+        assert_eq!(permutations.len(), 24);
+
+        for (case, newer_version) in [("newer-current-version", true), ("same-time-different-directory", false)] {
+            let metadata = std::array::from_fn::<_, 4, _>(|index| {
+                let mut file_info = payload(index);
+                file_info.version_id = Some(Uuid::from_u128(if index > 0 && newer_version { 11 } else { 10 }));
+                file_info.is_latest = true;
+                if index > 0 {
+                    file_info.data_dir = Some(Uuid::from_u128(2));
+                    if newer_version {
+                        file_info.mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
+                    }
+                }
+                assert!(file_info_is_valid_for_metadata(&file_info), "case={case}, disk={index}");
+                file_info
+            });
+            let expected_current = &metadata[1];
+            let expected_hash = SetDisks::file_info_quorum_hash(expected_current);
+            assert_ne!(SetDisks::file_info_quorum_hash(&metadata[0]), expected_hash);
+
+            for enabled in [false, true] {
+                for order in &permutations {
+                    let mut slots = std::array::from_fn::<_, 4, _>(|index| observation(index, 4));
+                    let mut typed = MetadataQuorumAccumulator::new(4, 2, enabled);
+                    let mut legacy = MetadataQuorumAccumulator::new(4, 2, enabled);
+                    for slot in &slots {
+                        typed.observe(slot);
+                        observe_legacy(&mut legacy, slot);
+                    }
+                    assert_same_reduction(&typed, &legacy);
+                    assert_eq!(typed.valid_responses, 0, "pending slots cannot supply votes");
+
+                    let mut old_seen = false;
+                    let mut current_votes = 0;
+                    for (prefix, &index) in order.iter().enumerate() {
+                        slots[index].result = Some(Ok(metadata[index].clone()));
+                        typed.observe(&slots[index]);
+                        observe_legacy(&mut legacy, &slots[index]);
+                        assert_same_reduction(&typed, &legacy);
+                        if index == 0 {
+                            old_seen = true;
+                        } else {
+                            current_votes += 1;
+                        }
+
+                        // A+B+B must wait for pending B; three B responses may
+                        // finish early only before the conflicting A arrives.
+                        let expected_decision = if enabled && !old_seen && current_votes == 3 {
+                            Some(MetadataEarlyStopDecision {
+                                reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
+                            })
+                        } else {
+                            None
+                        };
+                        assert_eq!(
+                            typed.early_stop_decision(),
+                            expected_decision,
+                            "case={case}, enabled={enabled}, order={order:?}, prefix={prefix}"
+                        );
+                        assert_eq!(typed.valid_responses, prefix + 1);
+                        assert_eq!(typed.conflicting_metadata, old_seen && current_votes > 0);
+                        assert_eq!(typed.version_early_stop_decision(), None, "current reads do not request a version ID");
+                        if expected_decision.is_some() {
+                            assert_eq!(
+                                SetDisks::file_info_quorum_hash(
+                                    typed.candidate.as_ref().expect("three B votes need a candidate")
+                                ),
+                                expected_hash
+                            );
+                        }
+                    }
+
+                    let completed_metadata = slots
+                        .iter()
+                        .map(|slot| slot.file_info().expect("every disk must have arrived").clone())
+                        .collect::<Vec<_>>();
+                    let (_, selected, selection_quorum) =
+                        SetDisks::select_valid_fileinfo(&vec![None; 4], &completed_metadata, &vec![None; 4], "", 2, 3)
+                            .expect("three matching B disks must select the current identity");
+                    assert_eq!(selection_quorum, 3);
+                    assert_eq!(selected.version_id, expected_current.version_id);
+                    assert_eq!(selected.mod_time, expected_current.mod_time);
+                    assert_eq!(selected.data_dir, expected_current.data_dir);
+                    assert_eq!(SetDisks::file_info_quorum_hash(&selected), expected_hash);
+                }
+            }
+        }
+    }
+
     #[test]
     fn metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait() {
         let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);

crates/ecstore/src/set_disk/mod.rs

@@ -62,7 +62,7 @@ use crate::diagnostics::get::{
     classify_storage_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
     record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
 };
-use crate::diagnostics::object_lock::ObjectLockAttempt;
+use crate::diagnostics::object_lock::{ObjectLockAttempt, record_lock_acquire_failure};
 use crate::disk::error_reduce::{
     BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
     reduce_write_quorum_errs,
@@ -885,8 +885,6 @@ pub(crate) use ops::hermetic_set_disks_isolated;
 pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
-#[cfg(test)]
-pub(crate) use ops::object::DeleteObjectCommitBarrier;
 #[cfg(feature = "test-util")]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 #[cfg(all(test, feature = "test-util"))]
@@ -894,6 +892,8 @@ pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransition
 pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
+#[cfg(test)]
+pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
 #[cfg(all(test, feature = "test-util"))]
@@ -1095,6 +1095,217 @@ mod prepared_get_object_metadata_tests {
     use http::HeaderMap;
     use tokio::io::AsyncReadExt;
 
+    const OBSERVATION_HOOK_BODY_LEN: usize = 1024 * 1024 + 1;
+
+    struct ObservationBodyHook {
+        object: &'static str,
+        calls: Arc<AtomicU64>,
+        body: bytes::Bytes,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::object_api::GetObjectBodyCacheHook for ObservationBodyHook {
+        async fn lookup(&self, bucket: &str, object: &str, info: &ObjectInfo) -> Option<bytes::Bytes> {
+            if bucket == crate::disk::RUSTFS_META_BUCKET && object == self.object {
+                assert_eq!(info.size, i64::try_from(self.body.len()).expect("hook fixture size"));
+                self.calls.fetch_add(1, Ordering::SeqCst);
+                Some(self.body.clone())
+            } else {
+                None
+            }
+        }
+    }
+
+    struct ObservationHookGuard;
+
+    impl Drop for ObservationHookGuard {
+        fn drop(&mut self) {
+            crate::object_api::unregister_get_object_body_cache_hook();
+        }
+    }
+
+    async fn ordinary_observation_get(set: &SetDisks, object: &str) -> Vec<u8> {
+        let mut reader = set
+            .get_object_reader(crate::disk::RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("ordinary GET positive control");
+        let mut bytes = Vec::new();
+        reader.stream.read_to_end(&mut bytes).await.expect("ordinary GET body");
+        bytes
+    }
+
+    fn observation_register_hook(object: &'static str, body: Vec<u8>) -> (ObservationHookGuard, Arc<AtomicU64>) {
+        let calls = Arc::new(AtomicU64::new(0));
+        crate::object_api::register_get_object_body_cache_hook(Arc::new(ObservationBodyHook {
+            object,
+            calls: calls.clone(),
+            body: bytes::Bytes::from(body),
+        }));
+        (ObservationHookGuard, calls)
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_real_backend_boundaries() {
+        let object = "config/observation-real.bin";
+        let payload = b"persisted config";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(payload.to_vec());
+        let written = set
+            .put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let result = crate::config::com::read_config_observation(set.clone(), object, payload.len()).await;
+        assert_eq!(result.is_ok(), true, "I05_OBSERVATION_SUCCESS: {result:?}");
+        let (bytes, info) = result.expect("real observation success");
+        assert_eq!(bytes, payload);
+        assert_eq!(info.size, written.size);
+        assert_eq!(info.etag, written.etag);
+        assert_eq!(info.data_dir, written.data_dir);
+        assert_eq!(info.name, object);
+        let over = crate::config::com::read_config_observation(set.clone(), object, payload.len() - 1).await;
+        assert_eq!(over.is_err(), true, "I05_BOUND_ENFORCED");
+        assert_eq!(over.expect_err("over limit must fail"), Error::CorruptedFormat, "I05_BOUND_ENFORCED");
+        let absent = crate::config::com::read_config_observation(set.clone(), "config/observation-absent.bin", 32).await;
+        assert_eq!(absent.expect_err("absent config"), Error::ConfigNotFound);
+        let empty_object = "config/observation-empty.bin";
+        let mut empty = PutObjReader::from_vec(Vec::new());
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, empty_object, &mut empty, &ObjectOptions::default())
+            .await
+            .expect("persist empty config");
+        let empty = crate::config::com::read_config_observation(set, empty_object, 0).await;
+        assert!(empty.is_ok(), "existing empty config must succeed: {empty:?}");
+        let (bytes, info) = empty.expect("empty observation");
+        assert!(bytes.is_empty());
+        assert_eq!(info.size, 0);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_bypasses_registered_body_hook() {
+        let object = "config/observation-hook.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
+        let result = crate::config::com::read_config_observation(set, object, OBSERVATION_HOOK_BODY_LEN).await;
+        assert_eq!(result.is_ok(), true, "I05_HOOK_BYPASS: {result:?}");
+        assert_eq!(
+            result.expect("observed persisted body").0,
+            vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
+            "I05_HOOK_BYPASS"
+        );
+        assert_eq!(calls.load(Ordering::SeqCst), 0, "I05_HOOK_BYPASS");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_hook_scope_restores_after_error() {
+        let object = "config/observation-hook-error.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        let result = crate::config::com::read_config_observation(set.clone(), object, OBSERVATION_HOOK_BODY_LEN - 1).await;
+        assert_eq!(result.expect_err("bounded observation error"), Error::CorruptedFormat);
+        assert_eq!(calls.load(Ordering::SeqCst), 0);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
+    struct PendingObservationStore {
+        set: Arc<SetDisks>,
+        ready: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
+    }
+
+    impl std::fmt::Debug for PendingObservationStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("PendingObservationStore").finish()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for PendingObservationStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            bucket: &str,
+            object: &str,
+            range: Option<HTTPRangeSpec>,
+            headers: HeaderMap,
+            opts: &ObjectOptions,
+        ) -> Result<GetObjectReader> {
+            let mut reader = self.set.get_object_reader(bucket, object, range, headers, opts).await?;
+            let mut body = Vec::new();
+            reader.stream.read_to_end(&mut body).await.expect("pending producer real GET");
+            assert_eq!(
+                body,
+                vec![b'p'; OBSERVATION_HOOK_BODY_LEN],
+                "I05_HOOK_BYPASS: producer must enter its own scope"
+            );
+            self.ready
+                .lock()
+                .expect("pending signal")
+                .take()
+                .expect("one GET")
+                .send(())
+                .expect("producer entry signal");
+            std::future::pending::<Result<GetObjectReader>>().await
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut PutObjReader,
+            _opts: &ObjectOptions,
+        ) -> Result<ObjectInfo> {
+            assert!(false, "observation must not PUT");
+            Err(Error::other("unexpected observation PUT"))
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn observation_hook_scope_restores_after_cancel() {
+        let object = "config/observation-hook-cancel.bin";
+        let (_dirs, set) = make_local_set_disks(4, 2).await;
+        let mut reader = PutObjReader::from_vec(vec![b'p'; OBSERVATION_HOOK_BODY_LEN]);
+        set.put_object(crate::disk::RUSTFS_META_BUCKET, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("persist real config fixture");
+        let (_hook, calls) = observation_register_hook(object, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
+        let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
+        let adapter = Arc::new(PendingObservationStore {
+            set: set.clone(),
+            ready: std::sync::Mutex::new(Some(ready_tx)),
+        });
+        let mut scope = Box::pin(crate::config::com::read_config_observation(adapter, object, OBSERVATION_HOOK_BODY_LEN));
+        tokio::select! {
+            result = ready_rx => { result.expect("producer reached pending GET"); }
+            result = &mut scope => { assert!(false, "producer must remain pending until cancelled: {result:?}"); }
+        }
+        assert_eq!(calls.load(Ordering::SeqCst), 0);
+        drop(scope);
+        assert_eq!(ordinary_observation_get(&set, object).await, vec![b'x'; OBSERVATION_HOOK_BODY_LEN]);
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
     const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
 
     fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
@@ -1441,6 +1652,172 @@ mod prepared_get_object_metadata_tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn non_inline_two_phase_get_rejects_generation_change_before_late_refresh() {
+        use crate::object_api::WriteCompletion;
+
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (dirs, set_disks) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
+        let bucket = "non-inline-late-generation";
+        let object = object_with_initial_data_shards(bucket, "late-generation-object");
+        let generation_a_body = vec![0x5a; 1024 * 1024];
+        let generation_b_body = vec![0xa5; generation_a_body.len()];
+        // This unlocked, real-disk fixture exercises the internal generation
+        // boundary, not concurrent HTTP writes through a namespace read lock.
+        let opts = ObjectOptions {
+            no_lock: true,
+            suppress_read_repair: true,
+            write_completion: WriteCompletion::TailDrained,
+            ..Default::default()
+        };
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create late generation bucket");
+        set_disks
+            .put_object(bucket, &object, &mut PutObjReader::from_vec(generation_a_body), &opts)
+            .await
+            .expect("commit every generation A shard before GET");
+        let disks = set_disks.get_disks_internal().await;
+        let original = SetDisks::read_metadata_observed(&disks, "", bucket, &object, "", true, false, false, false, 2)
+            .await
+            .expect("read every generation A disk slot");
+        assert!(original.is_complete());
+        let (original_metadata, original_errors, _) = original.into_legacy();
+        assert!(original_errors.iter().all(Option::is_none));
+        let (_, generation_a, _) = SetDisks::select_valid_fileinfo(&disks, &original_metadata, &original_errors, "", 2, 3)
+            .expect("generation A must have real metadata quorum");
+        assert!(!generation_a.inline_data(), "fixture must use non-inline shard files");
+        let order = bounded_metadata_fanout_order(bucket, &object, 4, 2);
+        for disk_index in order.iter().take(2) {
+            let data_dir = original_metadata[*disk_index]
+                .data_dir
+                .expect("generation A metadata must name its real shard directory");
+            let part_path = dirs[*disk_index]
+                .path()
+                .join(bucket)
+                .join(&object)
+                .join(data_dir.to_string())
+                .join("part.1");
+            let mut shard = std::fs::read(&part_path).expect("selected generation A shard must exist");
+            *shard.first_mut().expect("real shard must contain a checksum frame") ^= 0xff;
+            std::fs::write(part_path, shard).expect("corrupt selected shard without changing generation A metadata");
+        }
+
+        temp_env::async_with_vars(
+            [
+                ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
+                ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
+                ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
+            ],
+            async {
+                let _hedge_timer = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_NON_INLINE_HEDGE_TIMER);
+                let refresh = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_LATE_METADATA_REFRESH);
+                let calls = disk_call_counters::observe(&object);
+                let get = set_disks.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts);
+                tokio::pin!(get);
+                tokio::time::timeout(READ_VERSION_BARRIER_GUARD, async {
+                    tokio::select! {
+                        result = &mut get => panic!("GET returned before the late refresh boundary: {:?}", result.err()),
+                        () = refresh.wait_until_paused() => {}
+                    }
+                })
+                .await
+                .expect("failed generation A decode must reach the real late refresh boundary");
+                assert!(refresh.is_paused());
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_READ_VERSION),
+                    3,
+                    "A must be an early metadata candidate"
+                );
+                assert_eq!(
+                    calls.for_disk(disk_call_counters::KIND_READ_VERSION, order[3]),
+                    0,
+                    "late refresh must not have read the omitted disk before replacement"
+                );
+
+                set_disks
+                    .put_object(bucket, &object, &mut PutObjReader::from_vec(generation_b_body.clone()), &opts)
+                    .await
+                    .expect("commit every healthy generation B shard while the late GET is paused");
+                let replacement = SetDisks::read_metadata_observed(&disks, "", bucket, &object, "", true, false, false, false, 2)
+                    .await
+                    .expect("read every committed generation B disk slot");
+                assert!(replacement.is_complete());
+                let (replacement_metadata, replacement_errors, _) = replacement.into_legacy();
+                assert!(replacement_errors.iter().all(Option::is_none));
+                let (online, generation_b, _) =
+                    SetDisks::select_valid_fileinfo(&disks, &replacement_metadata, &replacement_errors, "", 2, 3)
+                        .expect("replacement must have healthy real metadata quorum");
+                assert_eq!(online.iter().flatten().count(), 4);
+                assert_ne!(generation_a.data_dir, generation_b.data_dir);
+                assert_ne!(
+                    SetDisks::file_info_quorum_hash(&generation_a),
+                    SetDisks::file_info_quorum_hash(&generation_b)
+                );
+                assert_eq!(generation_a.size, generation_b.size);
+                assert_eq!(generation_a.erasure.distribution, generation_b.erasure.distribution);
+                let before_late = (0..4)
+                    .map(|index| calls.for_disk(disk_call_counters::KIND_READ_VERSION, index))
+                    .collect::<Vec<_>>();
+                refresh.release();
+                let result = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut get)
+                    .await
+                    .expect("late GET must finish after its refresh boundary is released");
+                for (index, before) in before_late.into_iter().enumerate() {
+                    assert_eq!(
+                        calls.for_disk(disk_call_counters::KIND_READ_VERSION, index) - before,
+                        1,
+                        "late refresh must read each real disk exactly once"
+                    );
+                }
+                match result {
+                    Err(Error::InsufficientReadQuorum(failed_bucket, failed_object)) => {
+                        assert_eq!(failed_bucket, bucket);
+                        assert_eq!(failed_object, object);
+                    }
+                    Err(error) => panic!("changed late generation must return InsufficientReadQuorum, got {error:?}"),
+                    Ok(mut reader) => {
+                        assert_eq!(reader.object_info.size, i64::try_from(generation_b_body.len()).expect("fixture size"));
+                        let mut body = Vec::new();
+                        reader
+                            .stream
+                            .read_to_end(&mut body)
+                            .await
+                            .expect("unexpected replacement body must be readable");
+                        assert_eq!(body.len(), generation_b_body.len());
+                        assert_eq!(body, generation_b_body);
+                        println!(
+                            "late generation oracle exposed complete B: initial_reads=3, late_reads=4, bytes={}",
+                            body.len()
+                        );
+                        panic!("late GET must reject a changed object generation before exposing a body");
+                    }
+                }
+                drop(refresh);
+                let mut reader = set_disks
+                    .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
+                    .await
+                    .expect("fresh GET must open the healthy replacement generation");
+                assert_eq!(reader.object_info.size, i64::try_from(generation_b_body.len()).expect("fixture size"));
+                let mut body = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut body)
+                    .await
+                    .expect("fresh generation B must stream completely");
+                assert_eq!(body.len(), generation_b_body.len());
+                assert_eq!(body, generation_b_body);
+                println!(
+                    "late generation oracle rejected crossing: initial_reads=3, late_per_disk=[1,1,1,1], fresh_bytes={}",
+                    body.len()
+                );
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(body_cache_hook)]
     async fn four_data_two_parity_two_phase_read_recovers_one_failed_data_shard() {
@@ -4549,6 +4926,9 @@ impl SetDisks {
         let timeout = get_lock_acquire_timeout();
         let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "read", timeout);
         let result = ns_lock.get_read_lock(timeout).await;
+        if let Err(error) = &result {
+            record_lock_acquire_failure(op, bucket, object, "read", error);
+        }
         attempt.observe(&result);
         let guard = result.map_err(|e| self.map_namespace_lock_error(bucket, object, "read", e))?;
         let owner = diag_enabled.then(|| ns_lock.owner().to_string());
@@ -4607,6 +4987,9 @@ impl SetDisks {
         let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
         let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", acquire_timeout);
         let result = ns_lock.get_write_lock(acquire_timeout).await;
+        if let Err(error) = &result {
+            record_lock_acquire_failure(op, bucket, object, "write", error);
+        }
         attempt.observe(&result);
         let guard = resolve_put_object_commit_lock_acquire_result(self, op, bucket, object, result)?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
@@ -4659,6 +5042,9 @@ impl SetDisks {
             std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
         })
         .await;
+        if let Err(error) = &result {
+            record_lock_acquire_failure(op, bucket, object, "write", error);
+        }
         attempt.observe(&result);
         let guard = resolve_put_object_commit_lock_acquire_result(self, op, bucket, object, result)?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
@@ -9655,6 +10041,141 @@ mod tests {
         assert!(nested.join(STORAGE_FORMAT_FILE).exists());
     }
 
+    /// The residue a delete leaves once it committed on a disk but before its
+    /// cleanup pass: a marked data dir and the rollback backup of the same
+    /// transaction. Returns `(data dir, rollback backup)`.
+    async fn write_committed_delete_with_rollback_backup(
+        object_dir: &std::path::Path,
+        backup_transaction: Uuid,
+        marker_transaction: Option<Uuid>,
+    ) -> (std::path::PathBuf, std::path::PathBuf) {
+        let data_dir = object_dir.join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&data_dir).await.expect("data dir should be created");
+        fs::write(data_dir.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        if let Some(transaction) = marker_transaction {
+            fs::write(
+                data_dir.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+                [],
+            )
+            .await
+            .expect("committed delete marker should be written");
+        }
+        let backup = object_dir
+            .join(backup_transaction.to_string())
+            .join(STORAGE_FORMAT_FILE_BACKUP);
+        fs::create_dir_all(backup.parent().expect("backup parent"))
+            .await
+            .expect("rollback dir should be created");
+        fs::write(&backup, b"rollback metadata")
+            .await
+            .expect("rollback backup should be written");
+        (data_dir, backup)
+    }
+
+    // #6898: the rollback backup of a delete whose committed marker sits in a
+    // data dir of the same object is residue too, so the empty listing can
+    // reclaim the whole object directory.
+    #[tokio::test]
+    async fn purge_orphan_dir_object_removes_rollback_backup_of_committed_delete() {
+        let (dir, disk) = make_single_local_disk().await;
+        let transaction = Uuid::new_v4();
+        let object_dir = dir.path().join("bucket/pfx/obj");
+        write_committed_delete_with_rollback_backup(&object_dir, transaction, Some(transaction)).await;
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("purge should succeed");
+
+        assert!(purged);
+        assert!(!dir.path().join("bucket/pfx").exists(), "the whole committed residue tree must go");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_rollback_backup_of_other_transaction() {
+        let (dir, disk) = make_single_local_disk().await;
+        let object_dir = dir.path().join("bucket/pfx/obj");
+        let (data_dir, backup) =
+            write_committed_delete_with_rollback_backup(&object_dir, Uuid::new_v4(), Some(Uuid::new_v4())).await;
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        set.purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!data_dir.exists(), "the committed data dir is still reclaimed");
+        assert!(backup.exists(), "a backup whose own transaction is unproven must remain");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_rollback_backup_without_marker() {
+        let (dir, disk) = make_single_local_disk().await;
+        let object_dir = dir.path().join("bucket/pfx/obj");
+        let backup = object_dir.join(Uuid::new_v4().to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
+        fs::create_dir_all(backup.parent().expect("backup parent"))
+            .await
+            .expect("rollback dir should be created");
+        fs::write(&backup, b"rollback metadata")
+            .await
+            .expect("rollback backup should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "a lone rollback backup may still be needed to roll a delete back");
+        assert!(backup.exists());
+    }
+
+    // Live metadata on one disk blocks the object there and keeps it intact;
+    // on the other disks only what each disk's own committed marker proves is
+    // reclaimed, the same rule that already applies to a marked data dir.
+    #[tokio::test]
+    async fn purge_orphan_dir_object_keeps_object_with_metadata_on_another_disk() {
+        let mut dirs = Vec::new();
+        let mut disks = Vec::new();
+        for _ in 0..4 {
+            let (dir, disk) = make_single_local_disk().await;
+            dirs.push(dir);
+            disks.push(Some(disk));
+        }
+        let transaction = Uuid::new_v4();
+        let mut residue = Vec::new();
+        for dir in &dirs[..3] {
+            residue.push(
+                write_committed_delete_with_rollback_backup(&dir.path().join("bucket/pfx/obj"), transaction, Some(transaction))
+                    .await,
+            );
+        }
+        let live_object = dirs[3].path().join("bucket/pfx/obj");
+        let live_part = live_object.join(Uuid::new_v4().to_string()).join("part.1");
+        fs::create_dir_all(live_part.parent().expect("data dir"))
+            .await
+            .expect("live data dir should be created");
+        fs::write(&live_part, b"live").await.expect("live part should be written");
+        fs::write(live_object.join(STORAGE_FORMAT_FILE), b"meta")
+            .await
+            .expect("live metadata should be written");
+
+        let set = make_set_disks_with(disks).await;
+        set.purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(
+            live_object.join(STORAGE_FORMAT_FILE).exists() && live_part.exists(),
+            "the object with metadata on another disk must stay intact there"
+        );
+        for (data_dir, backup) in &residue {
+            assert!(!data_dir.exists() && !backup.exists(), "this disk's committed residue is reclaimed");
+        }
+    }
+
     // An unreadable directory anywhere under the prefix aborts the purge on
     // every disk before anything is deleted.
     #[cfg(unix)]

crates/ecstore/src/set_disk/ops/bucket.rs

@@ -34,6 +34,41 @@ pub(crate) enum BucketInfoQuorum {
 }
 
 impl SetDisks {
+    /// Require proof from every configured disk before retiring a bucket
+    /// generation whose visible volume was removed outside the storage API.
+    pub(crate) async fn ensure_bucket_volume_absent_on_every_disk(&self, bucket: &str) -> Result<()> {
+        let disks = self.disk_inventory().await;
+        if disks.is_empty() {
+            return Err(Error::ErasureReadQuorum);
+        }
+
+        let futures = disks.into_iter().map(|disk| async move {
+            match disk {
+                Some(disk) => disk.stat_volume(bucket).await.map(|_| ()),
+                None => Err(DiskError::DiskNotFound),
+            }
+        });
+        let results = join_all(futures).await;
+        let mut volume_present = false;
+        let mut first_unknown = None;
+        for result in results {
+            match result {
+                Ok(()) => volume_present = true,
+                Err(DiskError::VolumeNotFound) => {}
+                Err(error) if first_unknown.is_none() => first_unknown = Some(error),
+                Err(_) => {}
+            }
+        }
+
+        if let Some(error) = first_unknown {
+            return Err(error.into());
+        }
+        if volume_present {
+            return Err(Error::BucketExists(bucket.to_owned()));
+        }
+        Ok(())
+    }
+
     pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
         let disks = self.disk_inventory().await;
         let disk_count = disks.len();

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -3004,6 +3004,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         // etag
         let etag = resolve_complete_etag(opts, &uploaded_parts);
 
+        // An upload can contain parts from an older writer or a peer. Until
+        // every part's format is attested, keep its ETag out of keyless lists.
+        rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
+
         fi.metadata.insert("etag".to_owned(), etag);
 
         let persist_encryption_original_size = should_persist_encryption_original_size(&fi.metadata);

crates/ecstore/src/set_disk/ops/object.rs

@@ -2788,6 +2788,13 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                 )
                 .await;
                 if prepare_late_materialized_retry(&initial_result, &mut output, object_size) {
+                    #[cfg(test)]
+                    super::super::rename_fanout_barrier::checkpoint(
+                        object,
+                        0,
+                        super::super::rename_fanout_barrier::PHASE_LATE_METADATA_REFRESH,
+                    )
+                    .await;
                     let (full_fi, full_parts_metadata, full_online_disks) = Self::refresh_late_metadata_fanout(
                         &late_metadata_fanout_disks,
                         bucket,
@@ -3840,6 +3847,16 @@ impl SetDisks {
                 .map(crate::io_support::rio::compression_index_storage_bytes);
 
             let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
+            if data.stream.encrypted_etag {
+                if opts.preserve_etag.as_ref().is_none_or(|preserved| preserved == &etag) {
+                    insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+                } else {
+                    rustfs_utils::http::metadata_compat::remove_str(
+                        &mut user_defined,
+                        rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
+                    );
+                }
+            }
             if let Some(ref tag) = opts.preserve_etag {
                 etag = tag.clone();
             }
@@ -7078,6 +7095,82 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
     }
 }
 
+/// Pauses a delete after its per-disk commit and before the cleanup or rollback
+/// pass, the window in which an abandoned delete used to leave metadata-less
+/// residue behind.
+#[cfg(test)]
+struct DeleteCleanupBarrierState {
+    bucket: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct DeleteCleanupBarrier {
+    state: Arc<DeleteCleanupBarrierState>,
+}
+
+#[cfg(test)]
+static DELETE_CLEANUP_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<DeleteCleanupBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl DeleteCleanupBarrier {
+    pub(crate) fn install(bucket: &str) -> Self {
+        let state = Arc::new(DeleteCleanupBarrierState {
+            bucket: bucket.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = DELETE_CLEANUP_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("delete cleanup barrier mutex should not poison");
+        assert!(slot.is_none(), "delete cleanup barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("delete should reach the cleanup barrier");
+    }
+
+    pub(crate) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for DeleteCleanupBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = DELETE_CLEANUP_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("delete cleanup barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_delete_before_cleanup(bucket: &str) {
+    let barrier = DELETE_CLEANUP_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("delete cleanup barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 fn persisted_transition_version_with_gate(
     remote_version: &str,
     remote_version_state_writer_enabled: bool,
@@ -7658,6 +7751,8 @@ impl SetDisks {
 
         let quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object);
         let should_rollback = quorum_result.is_err();
+        #[cfg(test)]
+        pause_delete_before_cleanup(bucket).await;
         let mut rollback_futures = Vec::new();
         for (index, err) in errs.iter().enumerate() {
             // backlog#1158: when rolling back, fan the idempotent undo out to every
@@ -8667,6 +8762,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
+        #[cfg(test)]
+        pause_delete_before_cleanup(bucket).await;
 
         let mut rollback_futures = Vec::new();
         let committed_receipt_indices =
@@ -10523,6 +10620,65 @@ mod object_encryption_resolver_wiring_tests {
     use std::sync::Mutex;
     use std::sync::atomic::{AtomicUsize, Ordering};
 
+    #[tokio::test]
+    async fn encrypted_put_attests_only_etags_produced_by_its_encryption_reader() {
+        use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
+        use crate::io_support::rio::{WriteEncryption, WritePlan};
+        use crate::storage_api_contracts::object::ObjectIO as _;
+        use tokio::io::AsyncReadExt;
+
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(4).await;
+        let bucket = "encrypted-etag-proof";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create bucket volume");
+        }
+        let plaintext = b"Salary 2026: 85000 EUR";
+        let size = plaintext.len() as i64;
+        let mut control = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("control reader");
+        control.read_to_end(&mut Vec::new()).await.expect("read control plaintext");
+        let plaintext_md5 = control.try_resolve_etag().expect("control MD5");
+        for (object, preserved) in [("new", None), ("preserved", Some(plaintext_md5.clone()))] {
+            let reader =
+                HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext reader");
+            let reader = WritePlan::new()
+                .with_encryption(WriteEncryption::singlepart([1; 32], [2; 12]))
+                .apply(reader, size)
+                .expect("encrypted reader");
+            let mut opts = ObjectOptions {
+                preserve_etag: preserved.clone(),
+                user_defined: HashMap::from([
+                    ("x-amz-server-side-encryption-customer-algorithm".to_owned(), "AES256".to_owned()),
+                    (rustfs_utils::http::SSEC_ORIGINAL_SIZE_HEADER.to_owned(), size.to_string()),
+                ]),
+                ..Default::default()
+            };
+            // A copied producer marker cannot attest to a preserved plaintext tag.
+            insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+            set_disks
+                .put_object(bucket, object, &mut PutObjReader::new(reader), &opts)
+                .await
+                .expect("store encrypted bytes");
+            let stored = set_disks
+                .get_object_info(bucket, object, &ObjectOptions::default())
+                .await
+                .expect("read persisted metadata");
+            if let Some(preserved) = preserved {
+                assert_eq!(stored.etag.as_deref(), Some(preserved.as_str()));
+                assert!(!rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
+            } else {
+                assert_ne!(stored.etag.as_ref(), Some(&plaintext_md5));
+                assert_eq!(
+                    rustfs_utils::http::get_consistent_str(
+                        &stored.user_defined,
+                        rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG
+                    ),
+                    Some("v1")
+                );
+                assert!(rustfs_utils::http::etag_can_be_listed(&stored.user_defined));
+            }
+        }
+    }
+
     #[derive(Clone, Default)]
     struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
 

crates/ecstore/src/store/bucket.rs

@@ -204,6 +204,52 @@ where
     .await
 }
 
+fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
+    StorageError::NamespaceLockQuorumUnavailable {
+        mode: "write",
+        bucket: bucket.to_string(),
+        object: object.to_string(),
+        required: 1,
+        achieved: 0,
+    }
+}
+
+async fn await_orphan_bucket_recovery<T, F>(
+    bucket: &str,
+    publication_guard: &rustfs_lock::NamespaceLockGuard,
+    lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
+    metadata_guard: &rustfs_lock::NamespaceLockGuard,
+    namespace_guard: &rustfs_lock::NamespaceLockGuard,
+    future: F,
+) -> Result<T>
+where
+    F: Future<Output = Result<T>>,
+{
+    let metadata_lock_object = crate::bucket::metadata_sys::bucket_metadata_transaction_lock_key(bucket);
+    for ((lock_bucket, lock_object), guard) in [
+        (
+            (bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH),
+            publication_guard,
+        ),
+        ((bucket, BUCKET_LIFECYCLE_LOCK_OBJECT), lifecycle_guard),
+        ((RUSTFS_META_BUCKET, metadata_lock_object.as_str()), metadata_guard),
+        ((bucket, bucket), namespace_guard),
+    ] {
+        if guard.is_lock_lost() {
+            return Err(bucket_recovery_lock_unavailable(lock_bucket, lock_object));
+        }
+    }
+
+    tokio::select! {
+        biased;
+        _ = publication_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)),
+        _ = lifecycle_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)),
+        _ = metadata_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(RUSTFS_META_BUCKET, &metadata_lock_object)),
+        _ = namespace_guard.lock_lost_notified() => Err(bucket_recovery_lock_unavailable(bucket, bucket)),
+        result = future => result,
+    }
+}
+
 async fn run_bucket_usage_cleanup<F>(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()>
 where
     F: Future<Output = Result<()>>,
@@ -272,6 +318,20 @@ async fn bucket_delete_local_blocker(
 }
 
 impl ECStore {
+    /// Run a multi-step storage mutation on the instance task tracker so its
+    /// caller may stop waiting without releasing guards before the mutation
+    /// finishes.
+    pub async fn run_detached_mutation<F>(&self, mutation: F) -> Result<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.ctx
+            .run_detached_mutation(mutation)
+            .await
+            .map_err(|error| StorageError::other_with_context("detached ECStore mutation failed", error))
+    }
+
     fn bucket_sets(&self) -> impl Iterator<Item = (usize, usize, Arc<crate::set_disk::SetDisks>)> + '_ {
         self.pools.iter().flat_map(|pool| {
             pool.disk_set
@@ -280,6 +340,113 @@ impl ECStore {
         })
     }
 
+    async fn ensure_bucket_volume_absent_on_all_sets(&self, bucket: &str) -> Result<()> {
+        let sets: Vec<_> = self.bucket_sets().map(|(_, _, set)| set).collect();
+        if sets.is_empty() {
+            return Err(StorageError::ErasureReadQuorum);
+        }
+        let results = join_all(sets.iter().map(|set| set.ensure_bucket_volume_absent_on_every_disk(bucket))).await;
+        for result in results {
+            result?;
+        }
+        Ok(())
+    }
+
+    /// Return whether an old bucket generation has a durable retirement record.
+    /// A live matching incarnation is never considered retired, even if a stale
+    /// record exists from a previous generation.
+    pub async fn is_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
+        if expected.is_nil() {
+            return Ok(false);
+        }
+        let retirement_store = metadata_sys::object_store_if_initialized_in(&self.ctx).await.ok_or_else(|| {
+            StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-retirements".to_string())
+        })?;
+        if crate::bucket::metadata::load_bucket_incarnation(retirement_store.clone(), bucket).await? == Some(expected) {
+            return Ok(false);
+        }
+        crate::bucket::retirement::is_retired(retirement_store, bucket, expected).await
+    }
+
+    /// Reconcile bucket metadata after an operator removed all physical volume
+    /// directories. The expected generation, all-disk absence proof and durable
+    /// retirement record make retries safe after partial metadata cleanup.
+    pub fn recover_orphaned_bucket<'a>(&'a self, bucket: &'a str, expected: Uuid) -> futures::future::BoxFuture<'a, Result<()>> {
+        Box::pin(async move { Box::pin(self.recover_orphaned_bucket_inner(bucket, expected)).await })
+    }
+
+    async fn recover_orphaned_bucket_inner(&self, bucket: &str, expected: Uuid) -> Result<()> {
+        if let Err(error) = check_valid_bucket_name_strict(bucket) {
+            return Err(StorageError::BucketNameInvalid(error.to_string()));
+        }
+        if is_meta_bucketname(bucket) || expected.is_nil() {
+            return Err(StorageError::InvalidArgument(
+                "RecoverOrphanedBucket".to_string(),
+                "bucket".to_string(),
+                "a user bucket and non-nil incarnation are required".to_string(),
+            ));
+        }
+
+        // Lock order matches bucket deletion: publication -> lifecycle ->
+        // metadata transaction -> exact namespace.
+        let publication_guard = self.acquire_bucket_publication_write_lock(bucket).await?;
+        let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
+        let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?;
+        let ns_lock = self.new_ns_lock(bucket, bucket).await?;
+        let ns_guard = ns_lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .map_err(|error| match error {
+                rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    bucket: bucket.to_string(),
+                    object: bucket.to_string(),
+                    required,
+                    achieved,
+                },
+                other => StorageError::Lock(other),
+            })?;
+
+        let recover = Box::pin(async {
+            self.ensure_bucket_volume_absent_on_all_sets(bucket).await?;
+            let retirement_store = metadata_sys::object_store_if_initialized_in(&self.ctx).await.ok_or_else(|| {
+                StorageError::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-retirements".to_string())
+            })?;
+            let current_incarnation = crate::bucket::metadata::load_bucket_incarnation(retirement_store.clone(), bucket).await?;
+            if current_incarnation.is_some_and(|current| current != expected) {
+                return Err(StorageError::PreconditionFailed);
+            }
+
+            let retired = crate::bucket::retirement::is_retired(retirement_store.clone(), bucket, expected).await?;
+            if current_incarnation.is_none() && !retired {
+                return Err(StorageError::PreconditionFailed);
+            }
+            if !retired {
+                let mut record_opts = ObjectOptions {
+                    max_parity: true,
+                    ..Default::default()
+                };
+                record_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
+                record_opts.add_namespace_lock_guard(&ns_guard);
+                let publish_retirement = Box::pin(crate::bucket::retirement::commit_retirement(
+                    retirement_store.clone(),
+                    bucket,
+                    expected,
+                    &record_opts,
+                ));
+                publish_retirement.await?;
+            }
+
+            crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(retirement_store.clone(), bucket, expected).await?;
+            crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, Some(&ns_guard)).await?;
+            crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, Some(&ns_guard)).await?;
+            self.cleanup_deleted_bucket_metadata_strict(bucket, true).await?;
+            crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
+            Ok(())
+        });
+        await_orphan_bucket_recovery(bucket, &publication_guard, &lifecycle_guard, &metadata_guard, &ns_guard, recover).await
+    }
+
     pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
         let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
         sys.read().await.get(bucket).await
@@ -604,6 +771,25 @@ impl ECStore {
         Ok(())
     }
 
+    async fn cleanup_deleted_bucket_metadata_strict(&self, bucket: &str, include_deleted_marker: bool) -> Result<()> {
+        let options = ObjectOptions {
+            delete_prefix_object: true,
+            ..Default::default()
+        };
+        for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
+            self.delete_prefix(RUSTFS_META_BUCKET, &prefix, &options).await?;
+        }
+
+        if include_deleted_marker {
+            let marker_prefix = bucket_deleted_marker_prefix(bucket);
+            self.delete_prefix(RUSTFS_META_BUCKET, &marker_prefix, &options).await?;
+        }
+
+        metadata_sys::remove_bucket_metadata_in(&self.ctx, bucket).await?;
+        runtime_sources::delete_bucket_monitor_entry(bucket);
+        Ok(())
+    }
+
     async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
         run_bucket_usage_cleanup(guard, bucket, async {
             crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
@@ -715,7 +901,46 @@ impl ECStore {
             }
         };
         let confirmed_missing = existing_bucket_info.is_none();
-        let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
+        let mut pending_creation_metadata = None;
+        if confirmed_missing
+            && !opts.no_lock
+            && !is_meta_bucketname(bucket)
+            && let Some(metadata_store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await
+            && let Some(old_incarnation) = crate::bucket::metadata::load_bucket_incarnation(metadata_store, bucket).await?
+        {
+            // A quorum-level absence is not proof that every disk lost the old
+            // generation. Require the explicit recovery path to verify all disks
+            // and retire this incarnation before a same-name create can mutate
+            // any volume.
+            self.ensure_bucket_volume_absent_on_all_sets(bucket)
+                .await
+                .map_err(|error| match error {
+                    StorageError::BucketExists(_) => StorageError::InvalidArgument(
+                        "CreateBucket".to_string(),
+                        "bucket".to_string(),
+                        "an old bucket volume remains on disk; administrator recovery refused".to_string(),
+                    ),
+                    other => other,
+                })?;
+            let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
+            let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
+            let pending_intent = persisted
+                && metadata.bucket_incarnation_id == old_incarnation
+                && metadata.needs_bucket_creation_commit()
+                && !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
+            if pending_intent {
+                pending_creation_metadata = Some(metadata);
+            } else {
+                return Err(StorageError::InvalidArgument(
+                    "CreateBucket".to_string(),
+                    "bucket".to_string(),
+                    format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
+                ));
+            }
+        }
+        let existing_metadata = if let Some(metadata) = pending_creation_metadata {
+            Some(metadata)
+        } else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
             let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
             if !persisted {
                 metadata = BucketMetadata::new(bucket);
@@ -1135,6 +1360,23 @@ impl ECStore {
         } else {
             None
         };
+        // Match metadata migration: lifecycle -> metadata transaction ->
+        // bucket namespace. Holding the transaction while waiting for the
+        // namespace prevents a migration writer and force delete from
+        // acquiring these two locks in opposite orders.
+        let object_lock_metadata_transaction = if opts.force && !is_meta_bucketname(bucket) {
+            Some(
+                await_bucket_namespace_operation(
+                    bucket_lifecycle_guard.as_ref(),
+                    bucket,
+                    "Object Lock metadata transaction acquisition",
+                    crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket),
+                )
+                .await?,
+            )
+        } else {
+            None
+        };
         let ns_guard = if !opts.no_lock {
             let ns_lock = self.new_ns_lock(bucket, bucket).await?;
             Some(
@@ -1190,6 +1432,29 @@ impl ECStore {
             }
         }
 
+        // Keep configuration changes fenced until data deletion finishes;
+        // a force delete cannot bypass any Object Lock protection.
+        let object_lock_metadata_guard = if bucket_exists && let Some(guard) = object_lock_metadata_transaction {
+            let (state, incarnation, _) =
+                crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_with_guard_in(
+                    &self.ctx,
+                    bucket,
+                    Some(ns_guard.as_ref().unwrap_or(&guard)),
+                )
+                .await?;
+            match state {
+                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent => Some((guard, incarnation)),
+                crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. } => {
+                    return Err(StorageError::PrefixAccessDenied(bucket.to_owned(), String::new()));
+                }
+                crate::bucket::metadata_sys::ObjectLockConfigState::Fabricated => {
+                    return Err(StorageError::FileCorrupt);
+                }
+            }
+        } else {
+            None
+        };
+
         if sr_delete && !bucket_exists {
             delete_opts.force_if_empty = true;
         }
@@ -1203,7 +1468,12 @@ impl ECStore {
                 ns_guard.as_ref(),
                 bucket,
                 "bucket delete marker creation",
-                self.mark_bucket_deleted(bucket),
+                await_bucket_namespace_operation(
+                    object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
+                    bucket,
+                    "Object Lock configuration fence during bucket retirement",
+                    self.mark_bucket_deleted(bucket),
+                ),
             )
             .await?;
         }
@@ -1212,9 +1482,13 @@ impl ECStore {
         // Legacy buckets without a stamp cannot produce retirement authority.
         let retirement = if bucket_exists && bucket_lifecycle_guard.is_some() && !is_meta_bucketname(bucket) {
             if let Some(store) = metadata_sys::object_store_if_initialized_in(&self.ctx).await {
-                crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
-                    .await?
-                    .map(|incarnation| (store, incarnation))
+                if let Some((_, incarnation)) = object_lock_metadata_guard.as_ref() {
+                    Some((store, *incarnation))
+                } else {
+                    crate::bucket::metadata::load_bucket_incarnation(store.clone(), bucket)
+                        .await?
+                        .map(|incarnation| (store, incarnation))
+                }
             } else {
                 None
             }
@@ -1226,13 +1500,19 @@ impl ECStore {
             bucket_lifecycle_guard.as_ref(),
             bucket,
             "physical bucket deletion",
-            run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
-                self.delete_bucket_on_sets(bucket, &delete_opts)
-                    .await
-                    .map_err(|err| to_object_err(err, vec![bucket]))
-            }),
+            await_bucket_namespace_operation(
+                object_lock_metadata_guard.as_ref().map(|(guard, _)| guard),
+                bucket,
+                "Object Lock configuration fence during bucket deletion",
+                run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async {
+                    self.delete_bucket_on_sets(bucket, &delete_opts)
+                        .await
+                        .map_err(|err| to_object_err(err, vec![bucket]))
+                }),
+            ),
         )
         .await;
+        drop(object_lock_metadata_guard);
         if let Err(err) = delete_result
             && (!sr_delete || !is_err_strict_volume_not_found(&err))
         {
@@ -1610,6 +1890,65 @@ mod tests {
         setup_bucket_quorum_test_env(&[4, 4], None).await
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial]
+    async fn orphaned_bucket_recovery_requires_every_volume_absent_and_is_retryable() {
+        let (temp_dir, store) = setup_multi_pool_bucket_test_env().await;
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("orphan-recovery-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket before simulated operator removal");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(&bucket)
+            .await
+            .expect("persisted bucket generation");
+
+        let first_disk_volume = temp_dir.path().join("pool0-disk0").join(&bucket);
+        tokio::fs::remove_dir_all(&first_disk_volume)
+            .await
+            .expect("simulate operator removal on one disk");
+        let blocked = store
+            .recover_orphaned_bucket(&bucket, incarnation)
+            .await
+            .expect_err("recovery must refuse while any configured disk still has the volume");
+        assert!(matches!(blocked, StorageError::BucketExists(_)), "unexpected error: {blocked}");
+        assert_eq!(
+            crate::bucket::metadata::load_bucket_incarnation(store.clone(), &bucket)
+                .await
+                .expect("generation sidecar remains readable"),
+            Some(incarnation),
+            "a refused recovery must preserve the old generation identity"
+        );
+
+        for disk_index in 1..4 {
+            tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool0-disk{disk_index}")).join(&bucket))
+                .await
+                .expect("remove remaining volume from first set");
+        }
+        for disk_index in 0..4 {
+            tokio::fs::remove_dir_all(temp_dir.path().join(format!("pool1-disk{disk_index}")).join(&bucket))
+                .await
+                .expect("remove volume from second set");
+        }
+
+        store
+            .recover_orphaned_bucket(&bucket, incarnation)
+            .await
+            .expect("recovery should retire the exact generation and clean stale metadata");
+        assert!(
+            store
+                .is_bucket_incarnation_retired(&bucket, incarnation)
+                .await
+                .expect("retirement proof remains readable")
+        );
+        store
+            .recover_orphaned_bucket(&bucket, incarnation)
+            .await
+            .expect("retry after metadata cleanup should converge from durable retirement proof");
+    }
+
     async fn setup_bucket_quorum_test_env(
         drives_per_pool: &[usize],
         standard_parity: Option<usize>,
@@ -3418,6 +3757,116 @@ mod tests {
         assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn force_bucket_delete_preserves_object_lock_versions() {
+        let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], Some(2)).await;
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("force-delete-worm-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    versioning_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        let until = (OffsetDateTime::now_utc() + time::Duration::days(365))
+            .format(&time::format_description::well_known::Rfc3339)
+            .expect("format future retention date");
+        let written = store
+            .put_object(
+                &bucket,
+                "retained",
+                &mut PutObjReader::from_vec(b"retained bytes".to_vec()),
+                &ObjectOptions {
+                    versioned: true,
+                    user_defined: HashMap::from([
+                        ("x-amz-object-lock-mode".to_owned(), "COMPLIANCE".to_owned()),
+                        ("x-amz-object-lock-retain-until-date".to_owned(), until),
+                    ]),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write retained object");
+        for operation in [SRBucketDeleteOp::NoOp, SRBucketDeleteOp::MarkDelete, SRBucketDeleteOp::Purge] {
+            let err = store
+                .delete_bucket(
+                    &bucket,
+                    &DeleteBucketOptions {
+                        force: true,
+                        srdelete_op: operation,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect_err("force deletion must not bypass Object Lock");
+            assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected denial: {err}");
+            let info = store
+                .get_object_info(
+                    &bucket,
+                    "retained",
+                    &ObjectOptions {
+                        version_id: written.version_id.map(|version| version.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("retained version must survive the rejected bucket deletion");
+            assert_eq!(info.etag, written.etag);
+            assert_eq!(info.size, written.size);
+            assert_eq!(info.version_id, written.version_id);
+        }
+        metadata_sys::inject_object_lock_disk_read_error_in(&store.ctx, &bucket)
+            .await
+            .expect("inject authoritative metadata read failure");
+        let error = store
+            .delete_bucket(
+                &bucket,
+                &DeleteBucketOptions {
+                    force: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("an unreadable Object Lock configuration must fail closed");
+        assert!(error.to_string().contains("injected Object Lock metadata disk read failure"));
+        let info = store
+            .get_object_info(&bucket, "retained", &ObjectOptions::default())
+            .await
+            .expect("metadata read failure must leave the retained object intact");
+        assert_eq!(info.version_id, written.version_id);
+
+        let unlocked = format!("force-delete-unlocked-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&unlocked, &MakeBucketOptions::default())
+            .await
+            .expect("create unlocked control bucket");
+        store
+            .put_object(
+                &unlocked,
+                "ordinary",
+                &mut PutObjReader::from_vec(b"ordinary bytes".to_vec()),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("write ordinary control object");
+        store
+            .delete_bucket(
+                &unlocked,
+                &DeleteBucketOptions {
+                    force: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("force deletion must still work for an unlocked bucket");
+    }
+
     #[tokio::test]
     #[serial]
     async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {

crates/ecstore/src/store/init.rs

@@ -14521,15 +14521,19 @@ mod tests {
             let barrier = TierDeleteChunkTestBarrier::install(stage);
             let worker_store = store.clone();
             let worker_bucket = bucket.clone();
+            // Simulate a crash at the barrier. Aborting a caller of the public entry
+            // point no longer stops the delete, which runs on a detached task, so
+            // drive the delete below that task.
             let worker = tokio::spawn(async move {
                 worker_store
-                    .delete_object_with_tier_delete_journal(
+                    .handle_delete_object_with_journal(
                         &worker_bucket,
                         prefix,
                         ObjectOptions {
                             delete_prefix: true,
                             ..Default::default()
                         },
+                        Some(worker_store.clone()),
                     )
                     .await
             });
@@ -14538,6 +14542,11 @@ mod tests {
                 .unwrap_or_else(|_| panic!("chunk delete did not reach crash boundary {stage:?}"));
             worker.abort();
             let _ = worker.await;
+            assert_eq!(
+                ctx.detached_mutation_count(),
+                0,
+                "the crash must stop the delete below the detached task, so nothing keeps running after the abort"
+            );
             drop(barrier);
             let released_bucket_guard =
                 tokio::time::timeout(Duration::from_secs(5), store.acquire_bucket_lifecycle_write_lock(&bucket))
@@ -14683,15 +14692,19 @@ mod tests {
 
         let barrier = TierDeleteChunkTestBarrier::install(TierDeleteChunkTestStage::DispatchAuthorized);
         let worker_store = store.clone();
+        // Simulate a crash at the barrier. Aborting a caller of the public entry
+        // point no longer stops the delete, which runs on a detached task, so
+        // drive the delete below that task.
         let worker = tokio::spawn(async move {
             worker_store
-                .delete_object_with_tier_delete_journal(
+                .handle_delete_object_with_journal(
                     bucket,
                     prefix,
                     ObjectOptions {
                         delete_prefix: true,
                         ..Default::default()
                     },
+                    Some(worker_store.clone()),
                 )
                 .await
         });
@@ -14700,6 +14713,11 @@ mod tests {
             .expect("chunk delete should persist its authorization before corruption injection");
         worker.abort();
         let _ = worker.await;
+        assert_eq!(
+            ctx.detached_mutation_count(),
+            0,
+            "the crash must stop the delete below the detached task, so nothing keeps running after the abort"
+        );
         drop(barrier);
 
         let records = store
@@ -17536,6 +17554,332 @@ mod tests {
             .expect("new-generation object must survive the old batch delete");
     }
 
+    async fn put_detached_delete_fixture(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create detached delete bucket");
+        // Large enough for a data dir, so the delete commit leaves a marked
+        // data dir and a rollback backup until its cleanup pass runs.
+        let mut reader = PutObjReader::from_vec(vec![7u8; 2 * 1024 * 1024]);
+        store
+            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("put detached delete fixture");
+    }
+
+    async fn assert_delete_lock_moved_to_detached_task(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
+        let lock = store
+            .handle_new_ns_lock(bucket, object)
+            .await
+            .expect("namespace lock handle for the deleted key");
+        lock.get_write_lock_quiet(Duration::from_millis(200))
+            .await
+            .expect_err("the detached delete must keep its object lock after the caller is dropped");
+    }
+
+    fn assert_no_delete_residue(temp_dir: &std::path::Path, bucket: &str, object: &str, disks: usize) {
+        for disk in 0..disks {
+            let object_dir = temp_dir.join(format!("pool0/set0/disk{disk}")).join(bucket).join(object);
+            assert!(
+                !object_dir.exists(),
+                "disk {disk} must hold neither metadata, rollback backup nor data of the deleted object: {object_dir:?}"
+            );
+        }
+    }
+
+    // #6898: a DeleteObjects request dropped between the per-disk commit and the
+    // cleanup pass (client disconnect) must not leave metadata-less residue.
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn delete_objects_finishes_cleanup_after_caller_is_dropped() {
+        let temp = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "detached-batch-delete", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("detached-batch-delete-{}", uuid::Uuid::new_v4());
+        let object = "residue/batch.bin";
+        put_detached_delete_fixture(&store, &bucket, object).await;
+
+        let barrier = crate::set_disk::DeleteCleanupBarrier::install(&bucket);
+        let delete = store.delete_objects_with_tier_delete_journal_and_accounting(
+            &bucket,
+            vec![ObjectToDelete {
+                object_name: object.to_string(),
+                ..Default::default()
+            }],
+            ObjectOptions::default(),
+        );
+        tokio::select! {
+            () = barrier.wait_until_paused() => {}
+            _ = delete => panic!("batch delete must pause between its commit and its cleanup"),
+        }
+        assert_eq!(ctx.detached_mutation_count(), 1, "the dropped request must leave its delete running");
+        assert_delete_lock_moved_to_detached_task(&store, &bucket, object).await;
+
+        barrier.release();
+        assert!(
+            ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
+            "the detached batch delete should finish"
+        );
+        assert_no_delete_residue(temp.path(), &bucket, object, 4);
+        store
+            .get_object_info(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect_err("the object must be deleted");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn delete_object_finishes_cleanup_after_caller_is_dropped() {
+        let temp = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "detached-single-delete", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("detached-single-delete-{}", uuid::Uuid::new_v4());
+        let object = "residue/single.bin";
+        put_detached_delete_fixture(&store, &bucket, object).await;
+
+        let barrier = crate::set_disk::DeleteCleanupBarrier::install(&bucket);
+        let delete = store.delete_object_with_tier_delete_journal(&bucket, object, ObjectOptions::default());
+        tokio::select! {
+            () = barrier.wait_until_paused() => {}
+            _ = delete => panic!("single delete must pause between its commit and its cleanup"),
+        }
+        assert_eq!(ctx.detached_mutation_count(), 1, "the dropped request must leave its delete running");
+        assert_delete_lock_moved_to_detached_task(&store, &bucket, object).await;
+
+        barrier.release();
+        assert!(
+            ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
+            "the detached single delete should finish"
+        );
+        assert_no_delete_residue(temp.path(), &bucket, object, 4);
+        store
+            .get_object_info(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect_err("the object must be deleted");
+    }
+
+    // A recursive (force) delete runs under its caller's bucket lifecycle write
+    // lock and carries only that lock's fence. If the caller is dropped, the
+    // detached delete must still hold the lock: otherwise a PUT under the prefix
+    // succeeds and is then removed by the delete that is still running.
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn recursive_delete_keeps_bucket_lock_after_caller_is_dropped() {
+        let temp = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "detached-recursive-delete", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("detached-recursive-delete-{}", uuid::Uuid::new_v4());
+        let old_object = "folder/old.bin";
+        let new_object = "folder/new.bin";
+        put_detached_delete_fixture(&store, &bucket, old_object).await;
+
+        // The S3 force-delete path: the caller owns the lock, opts carry its fence.
+        let guard = store
+            .lock_bucket_for_recursive_delete(&bucket)
+            .await
+            .expect("recursive delete should lock the bucket");
+        let mut opts = ObjectOptions {
+            delete_prefix: true,
+            ..Default::default()
+        };
+        opts.add_bucket_lifecycle_lock_guard(&guard);
+        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
+        let delete = store.delete_object_with_tier_delete_journal_and_guards(&bucket, "folder/", opts, Some(guard));
+        tokio::select! {
+            () = barrier.wait_until_paused() => {}
+            _ = delete => panic!("recursive delete must pause inside its detached task"),
+        }
+        assert_eq!(ctx.detached_mutation_count(), 1, "the dropped caller must leave its delete running");
+
+        let put_store = Arc::clone(&store);
+        let put_bucket = bucket.clone();
+        let mut put = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(b"written while the delete runs".to_vec());
+            put_store
+                .put_object(&put_bucket, new_object, &mut reader, &ObjectOptions::default())
+                .await
+        });
+        assert!(
+            tokio::time::timeout(Duration::from_millis(500), &mut put).await.is_err(),
+            "a PUT under the prefix must wait while the detached recursive delete runs"
+        );
+
+        barrier.release();
+        assert!(
+            ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
+            "the detached recursive delete should finish"
+        );
+        put.await
+            .expect("PUT task should join")
+            .expect("the PUT should succeed once the recursive delete has finished");
+        store
+            .get_object_info(&bucket, old_object, &ObjectOptions::default())
+            .await
+            .expect_err("the recursive delete must remove the object it covered");
+        store
+            .get_object_info(&bucket, new_object, &ObjectOptions::default())
+            .await
+            .expect("the PUT that waited for the recursive delete must survive it");
+        for disk in 0..4 {
+            let metadata = temp
+                .path()
+                .join(format!("pool0/set0/disk{disk}"))
+                .join(&bucket)
+                .join(new_object)
+                .join(crate::disk::STORAGE_FORMAT_FILE);
+            assert!(
+                metadata.exists(),
+                "disk {disk} must keep the object written after the delete: {metadata:?}"
+            );
+        }
+    }
+
+    // Lifecycle expiry holds the table-bucket publication read lock and passes
+    // only its lock-lost signal to the delete. A cancelled expiry worker must not
+    // release that lock while its detached delete is still running.
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn lifecycle_expiry_keeps_publication_lock_after_worker_is_dropped() {
+        let temp = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "detached-lifecycle-expiry", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("detached-lifecycle-expiry-{}", uuid::Uuid::new_v4());
+        let object = "expired.bin";
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create lifecycle bucket");
+        let mut reader = PutObjReader::from_vec(b"expired data".to_vec());
+        let object_info = store
+            .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("put expired object");
+        let bucket_incarnation_id = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
+
+        let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket);
+        let expiry_store = Arc::clone(&store);
+        let expiry = tokio::spawn(async move {
+            crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+                expiry_store,
+                &object_info,
+                &crate::bucket::lifecycle::lifecycle::Event {
+                    action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction,
+                    ..Default::default()
+                },
+                &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+                bucket_incarnation_id,
+            )
+            .await
+        });
+        tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("lifecycle delete should pause inside its detached task");
+        // A cancelled worker drops its future while the delete keeps running.
+        expiry.abort();
+        assert!(
+            expiry
+                .await
+                .expect_err("aborted expiry worker should not join")
+                .is_cancelled(),
+            "expiry worker should be cancelled"
+        );
+        assert_eq!(ctx.detached_mutation_count(), 1, "the cancelled worker must leave its delete running");
+
+        let publication_lock = store
+            .handle_new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
+            .await
+            .expect("table-bucket publication lock handle");
+        publication_lock
+            .get_write_lock_quiet(Duration::from_millis(300))
+            .await
+            .expect_err("table-bucket publication must wait for the detached lifecycle delete");
+
+        barrier.release();
+        assert!(
+            ctx.wait_for_detached_mutations(Duration::from_secs(30)).await,
+            "the detached lifecycle delete should finish"
+        );
+        let publication_guard = publication_lock
+            .get_write_lock_quiet(crate::set_disk::get_lock_acquire_timeout())
+            .await
+            .expect("table-bucket publication should proceed once the lifecycle delete has finished");
+        drop(publication_guard);
+        store
+            .get_object_info(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect_err("the detached lifecycle delete should still remove the expired object");
+    }
+
+    // #6898: residue that an older build stranded (every disk committed the
+    // delete, the cleanup pass never ran) is reclaimed by deleting the key again.
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn repeated_delete_objects_finishes_committed_delete_residue() {
+        use crate::disk::DiskAPI as _;
+
+        for drives in [1, 4] {
+            let temp = tempfile::tempdir().expect("create temp store dir");
+            let (_ctx, store, _shutdown) =
+                without_storage_class_env(build_isolated_test_store(temp.path(), "residue-retry", &[drives])).await;
+            crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+            let bucket = format!("residue-retry-{drives}-{}", uuid::Uuid::new_v4());
+            let object = "residue/retry.bin";
+            put_detached_delete_fixture(&store, &bucket, object).await;
+
+            let transaction = uuid::Uuid::new_v4();
+            let disks = store.pools[0].disk_set[0].disks.read().await.clone();
+            for disk in disks.iter().flatten() {
+                let errors = disk
+                    .delete_versions(
+                        &bucket,
+                        vec![crate::disk::FileInfoVersions {
+                            name: object.to_string(),
+                            versions: vec![rustfs_filemeta::FileInfo {
+                                name: object.to_string(),
+                                ..Default::default()
+                            }],
+                            ..Default::default()
+                        }],
+                        crate::disk::DeleteOptions {
+                            old_data_dir: Some(transaction),
+                            ..Default::default()
+                        },
+                    )
+                    .await;
+                assert!(errors.iter().all(Option::is_none), "the delete should commit on every disk: {errors:?}");
+            }
+            for disk in 0..drives {
+                let object_dir = temp.path().join(format!("pool0/set0/disk{disk}")).join(&bucket).join(object);
+                assert!(!object_dir.join(crate::disk::STORAGE_FORMAT_FILE).exists());
+                assert!(
+                    object_dir
+                        .join(transaction.to_string())
+                        .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
+                        .exists(),
+                    "disk {disk} should hold the stranded rollback backup"
+                );
+            }
+
+            let (_deleted, errors) = store
+                .delete_objects(
+                    &bucket,
+                    vec![ObjectToDelete {
+                        object_name: object.to_string(),
+                        ..Default::default()
+                    }],
+                    ObjectOptions::default(),
+                )
+                .await;
+            assert!(errors.iter().all(Option::is_none), "a missing key deletes successfully: {errors:?}");
+            assert_no_delete_residue(temp.path(), &bucket, object, drives);
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() {
@@ -17974,7 +18318,7 @@ mod tests {
         let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
         let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
         intent.lock_enabled = true;
-        crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
+        crate::bucket::metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
             .await
             .expect("persist pre-visibility lock intent");
         assert!(

crates/ecstore/src/store/list_objects.rs

@@ -448,8 +448,8 @@ const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000;
 const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only";
 const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only";
 const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only";
-const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 2;
-const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v2";
+const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 3;
+const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v3";
 const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket=";
 const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation=";
 const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark=";
@@ -656,7 +656,10 @@ impl PersistentListMetadataObject {
             name: object.name.clone(),
             size: object.size,
             mod_time: object.mod_time,
-            etag: object.etag.clone(),
+            etag: object
+                .etag
+                .clone()
+                .filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined)),
             storage_class: object.storage_class.clone(),
         }
     }
@@ -2194,6 +2197,41 @@ fn last_scanned_entry_name(entries: Option<&MetaCacheEntriesSorted>) -> Option<S
     entries.and_then(|entries| entries.entries().last().map(|entry| entry.name.clone()))
 }
 
+fn filter_versions_common_prefixes_after_marker(objects: &mut Vec<ObjectInfo>, delimiter: Option<&str>, marker: Option<&str>) {
+    if delimiter.is_some()
+        && let Some(marker) = marker
+    {
+        // S3 excludes CommonPrefixes that are not lexicographically greater than KeyMarker.
+        // Filter before applying max-keys so a resumed prefix cannot consume the page budget.
+        objects.retain(|object| !(object.is_dir && object.mod_time.is_none()) || object.name.as_str() > marker);
+    }
+}
+
+fn last_raw_key_for_prefix<'a>(raw_keys: impl Iterator<Item = &'a str>, prefix: &str) -> Option<&'a str> {
+    raw_keys.filter(|key| key.starts_with(prefix)).last()
+}
+
+fn versioned_prefix_page_boundary_key<'a>(
+    objects: &[ObjectInfo],
+    raw_keys: impl Iterator<Item = &'a str>,
+    delimiter: Option<&str>,
+    max_keys: i32,
+    disk_has_more: bool,
+) -> Option<String> {
+    if delimiter.is_none() || !disk_has_more || max_keys <= 0 || objects.len() < max_keys as usize {
+        return None;
+    }
+
+    let last_visible = objects.get(max_keys as usize - 1)?;
+    if !last_visible.is_dir || last_visible.mod_time.is_some() {
+        return None;
+    }
+
+    // A folded prefix cannot advance the raw versioned walker. Continue at the last
+    // raw key represented by the final returned prefix, not at the next prefix's key.
+    last_raw_key_for_prefix(raw_keys, &last_visible.name).map(str::to_owned)
+}
+
 fn build_list_next_marker(objects: &[ObjectInfo], prefixes: &[String], cache_id: Option<&str>) -> Option<String> {
     if let Some(last) = objects.last() {
         Some(append_list_cache_id_to_marker(last.name.clone(), cache_id))
@@ -2280,14 +2318,20 @@ fn list_objects_paginate(
         }
     }
 
-    if !is_truncated && disk_has_more && !include_version_id {
+    if !is_truncated && disk_has_more {
         let visible_count = objects.len() + prefixes.len();
         let should_truncate = if delimiter.is_none() {
             visible_count > 0
         } else {
             visible_count >= max_keys as usize
         };
-        if should_truncate {
+        if delimiter.is_some()
+            && include_version_id
+            && let Some(last_scanned) = last_scanned_key
+        {
+            is_truncated = true;
+            next_marker = Some(append_list_cache_id_to_marker(last_scanned.to_owned(), cache_id));
+        } else if should_truncate {
             is_truncated = true;
             if include_version_id {
                 (next_marker, next_version_idmarker) = build_list_versions_next_marker(&objects, &prefixes, cache_id);
@@ -2320,6 +2364,49 @@ fn list_objects_paginate(
     (objects, prefixes, is_truncated, next_marker, next_version_idmarker)
 }
 
+struct VersionedListPagination<'a, I> {
+    delimiter: &'a Option<String>,
+    max_keys: i32,
+    disk_has_more: bool,
+    cache_id: Option<&'a str>,
+    marker: Option<&'a str>,
+    last_scanned_key: Option<&'a str>,
+    raw_keys: I,
+}
+
+fn list_objects_paginate_versions<'a, I>(
+    mut get_objects: Vec<ObjectInfo>,
+    pagination: VersionedListPagination<'a, I>,
+) -> (Vec<ObjectInfo>, Vec<String>, bool, Option<String>, Option<String>)
+where
+    I: Iterator<Item = &'a str>,
+{
+    filter_versions_common_prefixes_after_marker(&mut get_objects, pagination.delimiter.as_deref(), pagination.marker);
+    let prefix_at_page_boundary = versioned_prefix_page_boundary_key(
+        &get_objects,
+        pagination.raw_keys,
+        pagination.delimiter.as_deref(),
+        pagination.max_keys,
+        pagination.disk_has_more,
+    );
+
+    let (objects, prefixes, is_truncated, mut next_marker, mut next_version_idmarker) = list_objects_paginate(
+        get_objects,
+        pagination.delimiter,
+        pagination.max_keys,
+        pagination.disk_has_more,
+        pagination.cache_id,
+        true,
+        pagination.last_scanned_key,
+    );
+    if is_truncated && let Some(raw_key) = prefix_at_page_boundary {
+        next_marker = Some(append_list_cache_id_to_marker(raw_key, pagination.cache_id));
+        next_version_idmarker = None;
+    }
+
+    (objects, prefixes, is_truncated, next_marker, next_version_idmarker)
+}
+
 enum VerifiedIndexVisibleEntry {
     Object(Box<ObjectInfo>),
     Prefix(String),
@@ -4259,14 +4346,17 @@ impl ECStore {
             ObjectInfo::from_meta_cache_entries_sorted_versions(&entries, bucket, prefix, delimiter.clone(), version_marker).await
         };
 
-        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate(
+        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
             get_objects,
-            &delimiter,
-            max_keys,
-            disk_has_more,
-            next_cache_id.as_deref(),
-            true,
-            last_scanned_key.as_deref(),
+            VersionedListPagination {
+                delimiter: &delimiter,
+                max_keys,
+                disk_has_more,
+                cache_id: next_cache_id.as_deref(),
+                marker: opts.marker.as_deref(),
+                last_scanned_key: last_scanned_key.as_deref(),
+                raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
+            },
         );
 
         Ok(ListObjectVersionsInfo {
@@ -5681,23 +5771,22 @@ impl Sets {
         // Last RAW scanned key, captured before folding (ECA-03 / #944).
         let last_scanned_key = last_scanned_entry_name(list_result.entries.as_ref());
 
-        let get_objects = ObjectInfo::from_meta_cache_entries_sorted_versions(
-            &list_result.entries.unwrap_or_default(),
-            bucket,
-            prefix,
-            delimiter.clone(),
-            version_marker,
-        )
-        .await;
+        let entries = list_result.entries.unwrap_or_default();
+        let get_objects =
+            ObjectInfo::from_meta_cache_entries_sorted_versions(&entries, bucket, prefix, delimiter.clone(), version_marker)
+                .await;
 
-        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate(
+        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
             get_objects,
-            &delimiter,
-            max_keys,
-            disk_has_more,
-            next_cache_id.as_deref(),
-            true,
-            last_scanned_key.as_deref(),
+            VersionedListPagination {
+                delimiter: &delimiter,
+                max_keys,
+                disk_has_more,
+                cache_id: next_cache_id.as_deref(),
+                marker: opts.marker.as_deref(),
+                last_scanned_key: last_scanned_key.as_deref(),
+                raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
+            },
         );
 
         Ok(ListObjectVersionsInfo {
@@ -6498,23 +6587,22 @@ impl SetDisks {
         // Last RAW scanned key, captured before folding (ECA-03 / #944).
         let last_scanned_key = last_scanned_entry_name(list_result.entries.as_ref());
 
-        let get_objects = ObjectInfo::from_meta_cache_entries_sorted_versions(
-            &list_result.entries.unwrap_or_default(),
-            bucket,
-            prefix,
-            delimiter.clone(),
-            version_marker,
-        )
-        .await;
+        let entries = list_result.entries.unwrap_or_default();
+        let get_objects =
+            ObjectInfo::from_meta_cache_entries_sorted_versions(&entries, bucket, prefix, delimiter.clone(), version_marker)
+                .await;
 
-        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate(
+        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate_versions(
             get_objects,
-            &delimiter,
-            max_keys,
-            disk_has_more,
-            next_cache_id.as_deref(),
-            true,
-            last_scanned_key.as_deref(),
+            VersionedListPagination {
+                delimiter: &delimiter,
+                max_keys,
+                disk_has_more,
+                cache_id: next_cache_id.as_deref(),
+                marker: opts.marker.as_deref(),
+                last_scanned_key: last_scanned_key.as_deref(),
+                raw_keys: entries.entries().iter().map(|entry| entry.name.as_str()),
+            },
         );
 
         Ok(ListObjectVersionsInfo {
@@ -8697,6 +8785,223 @@ mod test {
         assert_eq!(next_marker.as_deref(), Some("obj-0002"));
     }
 
+    // ECA-03 / #944 (versioned variant): a versions listing whose raw keys fully
+    // collapse into fewer than max_keys common prefixes must still report
+    // truncation and carry a continuation marker, otherwise every key and every
+    // version past the scan window is silently dropped.
+    #[test]
+    fn list_objects_paginate_versioned_delimiter_refold_reports_truncation() {
+        let delimiter = Some("-".to_string());
+        let get_objects = vec![folded_prefix_object("data-")];
+
+        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate(
+            get_objects,
+            &delimiter,
+            1000,
+            true, // disk_has_more: walker filled its raw candidate limit
+            None,
+            true,
+            Some("data-1001"), // last RAW scanned key
+        );
+
+        assert!(objects.is_empty());
+        assert_eq!(prefixes, vec!["data-".to_string()]);
+        assert!(is_truncated, "versioned re-folded page with more on disk must be truncated");
+        // The marker must be the last RAW key, NOT the folded prefix "data-":
+        // versioned walks are excluded from the gather-level common-prefix
+        // collector, so a prefix marker could never advance `forward_past`.
+        assert_eq!(next_marker.as_deref(), Some("data-1001"));
+        assert!(next_version_idmarker.is_none(), "a folded prefix page carries no version marker");
+    }
+
+    // Same contract when the page is FILLED by folded common prefixes
+    // (`visible_count >= max_keys`): the versioned path must still prefer the
+    // raw scanned key over `prefixes.last()`, which would loop forever.
+    #[test]
+    fn list_objects_paginate_versioned_delimiter_full_prefix_page_uses_raw_marker() {
+        let delimiter = Some("-".to_string());
+        let get_objects = vec![folded_prefix_object("data-")];
+
+        let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) =
+            list_objects_paginate(get_objects, &delimiter, 1, true, None, true, Some("data-0002"));
+
+        assert!(objects.is_empty());
+        assert_eq!(prefixes, vec!["data-".to_string()]);
+        assert!(is_truncated, "a prefix-filled versioned page with more on disk must be truncated");
+        assert_eq!(next_marker.as_deref(), Some("data-0002"));
+        assert!(next_version_idmarker.is_none());
+    }
+
+    // End-to-end simulation of versioned pagination with `max_keys=1`, where the
+    // next page resumes via a bare key marker (`forward_past` keeps `name > marker`
+    // only, matching `inner_list_object_versions_with_projection` without a version
+    // marker). Before the raw-key marker the same window replayed forever, so
+    // `other-`/`zzz` were never listed.
+    #[test]
+    fn list_objects_paginate_versioned_delimiter_prefix_filled_pages_terminate() {
+        let all: Vec<String> = [
+            "data-0001",
+            "data-0002",
+            "data-0003",
+            "data-0004",
+            "data-0005",
+            "other-0001",
+            "zzz",
+        ]
+        .into_iter()
+        .map(str::to_string)
+        .collect();
+        let delimiter = Some("-".to_string());
+        let max_keys = 1i32;
+        let limit = max_keys_plus_one(max_keys, true) as usize;
+
+        let mut marker: Option<String> = None;
+        let mut seen_prefixes = Vec::new();
+        let mut seen_objects = Vec::new();
+        let mut page_prefixes = Vec::new();
+        let mut pages = 0;
+
+        loop {
+            pages += 1;
+            assert!(pages <= 16, "versioned pagination did not terminate (possible infinite loop)");
+
+            let start = match &marker {
+                Some(m) => all.partition_point(|k| k.as_str() <= m.as_str()),
+                None => 0,
+            };
+            let window: Vec<String> = all[start..].iter().take(limit).cloned().collect();
+            let disk_has_more = window.len() == limit;
+            let last_scanned = window.last().cloned();
+
+            let get_objects = fold_delimiter_page(&window, "", "-");
+            let (objects, prefixes, is_truncated, next_marker, _v) = super::list_objects_paginate_versions(
+                get_objects,
+                super::VersionedListPagination {
+                    delimiter: &delimiter,
+                    max_keys,
+                    disk_has_more,
+                    cache_id: None,
+                    marker: marker.as_deref(),
+                    last_scanned_key: last_scanned.as_deref(),
+                    raw_keys: window.iter().map(String::as_str),
+                },
+            );
+
+            assert!(
+                prefixes.windows(2).all(|pair| pair[0] != pair[1]),
+                "a page must not contain duplicate common prefixes"
+            );
+            assert!(
+                prefixes
+                    .iter()
+                    .all(|prefix| marker.as_deref().is_none_or(|marker| prefix.as_str() > marker)),
+                "a resumed page must not repeat a prefix at or before its key marker"
+            );
+            page_prefixes.push(prefixes.clone());
+            seen_objects.extend(objects.into_iter().map(|object| object.name));
+            seen_prefixes.extend(prefixes);
+
+            if !is_truncated {
+                assert!(next_marker.is_none());
+                break;
+            }
+
+            let next = next_marker.expect("truncated versioned page must carry a continuation marker");
+            if let Some(prev) = &marker {
+                assert!(
+                    next.as_str() > prev.as_str(),
+                    "next_marker must strictly advance to stay finite, got {next} after {prev}"
+                );
+            }
+            marker = Some(next);
+        }
+
+        assert_eq!(
+            page_prefixes,
+            vec![vec!["data-".to_string()], vec![], vec!["other-".to_string()], vec![]],
+            "versioned pagination must not repeat a folded prefix while scanning later raw keys"
+        );
+        assert_eq!(
+            seen_prefixes,
+            vec!["data-".to_string(), "other-".to_string()],
+            "versioned pagination must preserve the ordered visible result sequence"
+        );
+        assert_eq!(seen_objects, vec!["zzz".to_string()]);
+    }
+
+    #[test]
+    fn list_objects_paginate_versioned_prefix_boundary_uses_last_raw_key_in_returned_prefix() {
+        let all: Vec<String> = ["alpha-0001", "beta-0001", "charlie-0001"]
+            .into_iter()
+            .map(str::to_string)
+            .collect();
+        let delimiter = Some("-".to_string());
+        let max_keys = 1;
+        let limit = max_keys_plus_one(max_keys, true) as usize;
+        let mut marker: Option<String> = None;
+        let mut page_prefixes = Vec::new();
+        let mut pages = 0;
+
+        loop {
+            pages += 1;
+            assert!(pages <= 8, "versioned prefix pagination did not terminate");
+
+            let start = marker
+                .as_deref()
+                .map(|marker| all.partition_point(|key| key.as_str() <= marker))
+                .unwrap_or(0);
+            let window: Vec<String> = all[start..].iter().take(limit).cloned().collect();
+            let disk_has_more = window.len() == limit;
+            let last_scanned = window.last().cloned();
+
+            let get_objects = fold_delimiter_page(&window, "", "-");
+            let (objects, prefixes, is_truncated, next_marker, _) = super::list_objects_paginate_versions(
+                get_objects,
+                super::VersionedListPagination {
+                    delimiter: &delimiter,
+                    max_keys,
+                    disk_has_more,
+                    cache_id: None,
+                    marker: marker.as_deref(),
+                    last_scanned_key: last_scanned.as_deref(),
+                    raw_keys: window.iter().map(String::as_str),
+                },
+            );
+
+            assert_eq!(objects.len() + prefixes.len(), 1, "a page must respect max-keys");
+            page_prefixes.push(prefixes);
+
+            if !is_truncated {
+                assert!(next_marker.is_none());
+                break;
+            }
+
+            let next = next_marker.expect("truncated page must carry a continuation key marker");
+            if let Some(previous) = &marker {
+                assert!(next > *previous, "the continuation marker must strictly advance");
+            }
+            marker = Some(next);
+        }
+
+        assert_eq!(
+            page_prefixes,
+            vec![
+                vec!["alpha-".to_string()],
+                vec!["beta-".to_string()],
+                vec!["charlie-".to_string()]
+            ]
+        );
+    }
+
+    #[test]
+    fn versioned_marker_filter_does_not_change_no_delimiter_directories() {
+        let mut objects = vec![folded_prefix_object("data-")];
+        super::filter_versions_common_prefixes_after_marker(&mut objects, None, Some("z"));
+
+        assert_eq!(objects.len(), 1);
+        assert!(super::versioned_prefix_page_boundary_key(&objects, std::iter::once("data-"), None, 1, true).is_none());
+    }
+
     // End-to-end pagination over 5000 `data-*` + 100 `other-*` with delimiter '-'.
     // Every full page re-folds to a single common prefix (< max_keys), the exact
     // trigger for ECA-03 / #944. Asserts pagination terminates in a bounded number
@@ -8903,6 +9208,38 @@ mod test {
         assert_eq!(parsed, object);
     }
 
+    #[test]
+    fn persistent_listing_snapshots_cannot_restore_legacy_encrypted_fingerprints() {
+        let mut object = ObjectInfo {
+            name: "encrypted".to_owned(),
+            etag: Some("fbfcbcc6d2f035411e6268c8b119593e".to_owned()),
+            user_defined: Arc::new(HashMap::from([(
+                "x-amz-server-side-encryption-customer-algorithm".to_owned(),
+                "AES256".to_owned(),
+            )])),
+            ..Default::default()
+        };
+        let snapshot = PersistentListMetadataObject::from_object_info(&object);
+        assert!(snapshot.to_object_info("bucket").etag.is_none());
+        let mut old_snapshot = snapshot.clone();
+        old_snapshot.etag = object.etag.clone();
+        let old_contents = format!(
+            "# rustfs-listobjects-key-only-v2\n{}\n",
+            encode_persistent_list_metadata_object(&old_snapshot)
+        );
+        let old_index = parse_persistent_key_only_index(&old_contents);
+        assert_ne!(old_index.format_version, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION);
+
+        rustfs_utils::http::metadata_compat::insert_str(
+            Arc::make_mut(&mut object.user_defined),
+            rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
+            "v1".to_owned(),
+        );
+        assert_eq!(PersistentListMetadataObject::from_object_info(&object).etag, object.etag);
+        object.user_defined = Arc::new(HashMap::new());
+        assert_eq!(PersistentListMetadataObject::from_object_info(&object).etag, object.etag);
+    }
+
     #[test]
     fn persistent_key_only_index_parses_legacy_keys_and_metadata_snapshots() {
         let object = PersistentListMetadataObject {
@@ -9644,6 +9981,67 @@ mod test {
         }
     }
 
+    // #6898: what an abandoned DeleteObjects left on every disk (rollback
+    // backup plus marked data dir) no longer blocks DeleteBucket after an empty
+    // recursive listing of the bucket.
+    #[tokio::test]
+    async fn empty_recursive_listing_purges_stranded_delete_rollback_backups() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "listing-purge-rollback-backup";
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created with authoritative metadata");
+        let transaction = uuid::Uuid::new_v4();
+        for object in ["repro/obj-00001.bin", "repro/obj-00002.bin"] {
+            let data_dir = uuid::Uuid::new_v4();
+            for dir in &dirs {
+                let object_dir = dir.path().join(bucket).join(object);
+                let residue = object_dir.join(data_dir.to_string());
+                tokio::fs::create_dir_all(&residue)
+                    .await
+                    .expect("committed data dir should be created");
+                tokio::fs::write(residue.join("part.1"), b"stale")
+                    .await
+                    .expect("stale part should be written");
+                tokio::fs::write(
+                    residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+                    [],
+                )
+                .await
+                .expect("committed delete marker should be written");
+                let backup = object_dir.join(transaction.to_string());
+                tokio::fs::create_dir_all(&backup)
+                    .await
+                    .expect("rollback dir should be created");
+                tokio::fs::write(backup.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), b"rollback metadata")
+                    .await
+                    .expect("rollback backup should be written");
+            }
+        }
+
+        let result = store
+            .clone()
+            .list_objects_generic(bucket, "", None, None, 1000, false)
+            .await
+            .expect("recursive bucket listing should succeed");
+        assert!(result.objects.is_empty());
+        for dir in &dirs {
+            assert!(
+                !dir.path().join(bucket).join("repro").exists(),
+                "the empty listing should reclaim the stranded delete residue"
+            );
+        }
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("the bucket is empty once the residue is reclaimed");
+    }
+
     #[tokio::test]
     async fn empty_recursive_bucket_listing_purges_orphan_directory_prefixes() {
         use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};

crates/ecstore/src/store/object.rs

@@ -2456,6 +2456,27 @@ fn return_batch_delete_lock_error_with_accounting(
     (deleted, errors, vec![None; objects.len()])
 }
 
+const EVENT_DETACHED_DELETE_FAILED: &str = "detached_delete_failed";
+const LOG_SUBSYSTEM_STORE_DELETE: &str = "store_delete";
+
+/// A detached delete only fails to join when it panicked or the runtime is
+/// shutting down; report either as a failed delete to the waiting caller.
+fn detached_delete_failed(operation: &'static str, err: tokio::task::JoinError) -> Error {
+    error!(
+        event = EVENT_DETACHED_DELETE_FAILED,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_STORE_DELETE,
+        operation,
+        reason = if err.is_panic() { "task_panicked" } else { "task_cancelled" },
+        "Detached delete task failed"
+    );
+    if err.is_cancelled() {
+        Error::OperationCanceled
+    } else {
+        Error::Unexpected
+    }
+}
+
 fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
     let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
     object_names.sort_unstable();
@@ -4798,19 +4819,54 @@ impl ECStore {
         purged
     }
 
+    /// Delete one object (or a prefix) on behalf of a caller that may be
+    /// dropped mid-way, such as an S3 request whose client disconnects.
+    ///
+    /// The delete runs to completion on a detached task that owns its locks
+    /// and both on-disk phases, so the object is left either intact or fully
+    /// deleted, never with its metadata gone and its data still in place.
     pub async fn delete_object_with_tier_delete_journal(
         self: &Arc<Self>,
         bucket: &str,
         object: &str,
         opts: ObjectOptions,
     ) -> Result<ObjectInfo> {
-        let result = self
-            .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
-            .await;
-        if result.is_ok() {
-            list_objects::observe_list_objects_mutation(self, bucket).await;
-        }
-        result
+        self.delete_object_with_tier_delete_journal_and_guards(bucket, object, opts, None)
+            .await
+    }
+
+    /// [`Self::delete_object_with_tier_delete_journal`] for a caller that
+    /// holds namespace locks covering the delete and passes only their fences
+    /// in `opts`, such as the bucket lifecycle write lock of a recursive delete.
+    ///
+    /// The guards move into the detached task and are released only after
+    /// the delete has finished. Dropping the caller therefore cannot let a
+    /// writer into the scope they exclude while the delete is still running.
+    pub async fn delete_object_with_tier_delete_journal_and_guards(
+        self: &Arc<Self>,
+        bucket: &str,
+        object: &str,
+        opts: ObjectOptions,
+        guards: impl IntoIterator<Item = rustfs_lock::NamespaceLockGuard>,
+    ) -> Result<ObjectInfo> {
+        let guards: Vec<_> = guards.into_iter().collect();
+        let store = Arc::clone(self);
+        let bucket = bucket.to_owned();
+        let object = object.to_owned();
+        self.ctx
+            .run_detached_mutation(async move {
+                let result = store
+                    .handle_delete_object_with_journal(&bucket, &object, opts, Some(Arc::clone(&store)))
+                    .await;
+                if result.is_ok() {
+                    list_objects::observe_list_objects_mutation(&store, &bucket).await;
+                }
+                // Keep: the only use of `guards` here, it moves them into the task so the locks outlive a dropped caller.
+                drop(guards);
+                result
+            })
+            .await
+            .unwrap_or_else(|err| Err(detached_delete_failed("delete_object", err)))
     }
 
     pub async fn delete_objects_with_tier_delete_journal(
@@ -4819,30 +4875,43 @@ impl ECStore {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        let result = self
-            .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
+        let (deleted, errors, _) = self
+            .delete_objects_with_tier_delete_journal_and_accounting(bucket, objects, opts)
             .await;
-        let success_count = result.1.iter().filter(|err| err.is_none()).count();
-        if success_count > 0 {
-            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
-        }
-        result
+        (deleted, errors)
     }
 
+    /// Batch form of [`Self::delete_object_with_tier_delete_journal`]: the
+    /// whole batch, including its object locks, runs on one detached task.
     pub async fn delete_objects_with_tier_delete_journal_and_accounting(
         self: &Arc<Self>,
         bucket: &str,
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
-        let result = self
-            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
-            .await;
-        let success_count = result.1.iter().filter(|err| err.is_none()).count();
-        if success_count > 0 {
-            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
-        }
-        result
+        let store = Arc::clone(self);
+        let bucket = bucket.to_owned();
+        let object_count = objects.len();
+        self.ctx
+            .run_detached_mutation(async move {
+                let result = store
+                    .handle_delete_objects_with_journal_and_accounting(&bucket, objects, opts, Some(Arc::clone(&store)))
+                    .await;
+                let success_count = result.1.iter().filter(|err| err.is_none()).count();
+                if success_count > 0 {
+                    list_objects::observe_list_objects_mutations(&store, &bucket, success_count).await;
+                }
+                result
+            })
+            .await
+            .unwrap_or_else(|err| {
+                let err = detached_delete_failed("delete_objects", err);
+                (
+                    vec![DeletedObject::default(); object_count],
+                    vec![Some(err); object_count],
+                    vec![None; object_count],
+                )
+            })
     }
 
     #[instrument(skip(self))]
@@ -5282,13 +5351,41 @@ impl ECStore {
         tier_journal_api: Option<Arc<ECStore>>,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         Box::pin(async move {
+            let count = objects.len();
+            let mut deleted = vec![DeletedObject::default(); count];
+            let mut errors: Vec<Option<Error>> = (0..count).map(|_| None).collect();
+            let mut accounting = vec![None; count];
+            let mut valid_objects = Vec::with_capacity(count);
+            let mut valid_indices = Vec::with_capacity(count);
+            // Validate the literal key before encoding it or taking locks. A
+            // dot-segment key must never reach filesystem path resolution.
+            for (index, object) in objects.into_iter().enumerate() {
+                match check_del_obj_args(bucket, &object.object_name) {
+                    Ok(()) => {
+                        valid_indices.push(index);
+                        valid_objects.push(object);
+                    }
+                    Err(err) => errors[index] = Some(err),
+                }
+            }
+            if valid_objects.is_empty() {
+                return (deleted, errors, accounting);
+            }
             let mut opts = opts;
             let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
-            let result = self
-                .handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
+            let (valid_deleted, valid_errors, valid_accounting) = self
+                .handle_delete_objects_with_journal_and_accounting_inner(bucket, valid_objects, opts, tier_journal_api)
                 .await;
             enqueue_recorded_tier_free_versions(self, receipt_sink).await;
-            result
+            for (index, ((object, error), receipt)) in valid_indices
+                .into_iter()
+                .zip(valid_deleted.into_iter().zip(valid_errors).zip(valid_accounting))
+            {
+                deleted[index] = object;
+                errors[index] = error;
+                accounting[index] = receipt;
+            }
+            (deleted, errors, accounting)
         })
         .await
     }
@@ -9025,6 +9122,58 @@ mod tests {
         (dirs, store)
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn batch_delete_rejects_dot_segments_and_keeps_valid_result_positions() {
+        let bucket = "batch-delete-key-validation";
+        let (_dirs, store) = multipool_version_test_store(bucket).await;
+        let options = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+        let victim = store
+            .put_object(bucket, "victim/object", &mut PutObjReader::from_vec(b"victim".to_vec()), &options)
+            .await
+            .expect("write protected-prefix control object");
+        store
+            .put_object(bucket, "allowed/object", &mut PutObjReader::from_vec(b"allowed".to_vec()), &options)
+            .await
+            .expect("write valid deletion control object");
+        let keys = [
+            "allowed/../victim/object",
+            "allowed/object",
+            "a/./b",
+            "../other-bucket/object",
+        ];
+        let objects = keys
+            .iter()
+            .map(|key| ObjectToDelete {
+                object_name: (*key).to_owned(),
+                ..Default::default()
+            })
+            .collect();
+        let (deleted, errors) = store.handle_delete_objects(bucket, objects, options).await;
+        assert_eq!(deleted.len(), keys.len());
+        assert_eq!(errors.len(), keys.len());
+        for index in [0, 2, 3] {
+            assert!(
+                matches!(&errors[index], Some(StorageError::ObjectNameInvalid(_, _))),
+                "invalid key {}: {:?}",
+                keys[index],
+                errors[index]
+            );
+            assert!(!deleted[index].delete_marker);
+        }
+        assert!(errors[1].is_none(), "valid deletion must still succeed: {:?}", errors[1]);
+        assert!(deleted[1].delete_marker);
+        assert_eq!(deleted[1].object_name, "allowed/object");
+        let after = store
+            .get_object_info(bucket, "victim/object", &ObjectOptions::default())
+            .await
+            .expect("the resolved victim path must remain visible");
+        assert_eq!(after.version_id, victim.version_id);
+        assert_eq!(after.etag, victim.etag);
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
         let bucket = "retired-marker-c11";

crates/heal/src/error.rs

@@ -141,7 +141,7 @@ impl Error {
                     return true;
                 }
                 err.is_quorum_error()
-                    || matches!(err, EcstoreError::Io(error) if is_recoverable_internode_error(error))
+                    || matches!(err, EcstoreError::Io(error) if is_recoverable_io_error(error))
                     || matches!(
                         err,
                         EcstoreError::DiskNotFound
@@ -171,11 +171,11 @@ impl Error {
                         | DiskError::FaultyRemoteDisk
                         | DiskError::FaultyDisk
                         | DiskError::RemoteClientUnavailable(_)
-                ) || matches!(err, DiskError::Io(error) if is_recoverable_internode_error(error))
+                ) || matches!(err, DiskError::Io(error) if is_recoverable_io_error(error))
                     || is_recoverable_heal_error_message(&err.to_string())
             }
             Error::TaskExecutionFailed { message } | Error::Other(message) => is_recoverable_heal_error_message(message),
-            Error::Io(err) => is_recoverable_internode_error(err) || is_recoverable_heal_error_message(&err.to_string()),
+            Error::Io(err) => is_recoverable_io_error(err) || is_recoverable_heal_error_message(&err.to_string()),
             _ => false,
         }
     }
@@ -222,7 +222,15 @@ impl Error {
     }
 }
 
-fn is_recoverable_internode_error(error: &std::io::Error) -> bool {
+fn is_recoverable_io_error(error: &std::io::Error) -> bool {
+    // ObjectIO preserves store errors as IO sources; their quorum identity must survive this boundary.
+    if error
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<EcstoreError>())
+        .is_some_and(EcstoreError::is_quorum_error)
+    {
+        return true;
+    }
     // A peer restart can cancel an RPC after a partial repair. Replay it within
     // the existing heal retry budget; task cancellation remains terminal.
     if DiskError::io_error_is_rpc_cancelled(error) {
@@ -301,6 +309,37 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
+    #[test]
+    fn namespace_lock_quorum_is_recoverable_across_io_error_conversions() {
+        for (required, achieved) in [(2, 1), (1, 0)] {
+            let quorum = || EcstoreError::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "bucket".to_owned(),
+                object: "object".to_owned(),
+                required,
+                achieved,
+            };
+            for error in [
+                Error::Storage(quorum()),
+                Error::Io(std::io::Error::from(quorum())),
+                Error::Storage(EcstoreError::from(std::io::Error::from(quorum()))),
+                Error::Disk(DiskError::from(std::io::Error::from(quorum()))),
+            ] {
+                assert!(
+                    error.is_recoverable_object_heal(),
+                    "typed lock quorum failure must remain retryable: {error:?}"
+                );
+            }
+        }
+
+        let message = "Namespace lock quorum unavailable for write lock on bucket/object: required 2, achieved 1";
+        assert!(!Error::Io(std::io::Error::other(message)).is_recoverable_object_heal());
+        let denied = EcstoreError::Lock(rustfs_lock::LockError::PermissionDenied {
+            reason: message.to_owned(),
+        });
+        assert!(!Error::Storage(EcstoreError::from(std::io::Error::from(denied))).is_recoverable_object_heal());
+    }
+
     #[test]
     fn cancelled_rpc_is_recoverable_across_storage_error_conversions() {
         let status = tonic::Status::cancelled("operation was canceled");

crates/heal/src/heal/erasure_healer.rs

@@ -337,6 +337,12 @@ impl ErasureSetHealer {
             return HealObjectOutcome::Failed;
         };
 
+        if let EcstoreError::Lock(lock_error) = se
+            && !lock_error.is_fatal()
+        {
+            return HealObjectOutcome::Transient;
+        }
+
         // Genuine object/version absence: nothing left to heal, treat as handled.
         if matches!(
             se,
@@ -2142,6 +2148,46 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn retryable_lock_failures_are_transient_but_fatal_lock_errors_are_not() {
+        use rustfs_lock::LockError;
+
+        assert!(matches!(
+            ErasureSetHealer::classify_heal_object_error(&Error::Storage(EcstoreError::Lock(LockError::timeout(
+                "bucket/object",
+                std::time::Duration::from_secs(5)
+            )))),
+            HealObjectOutcome::Transient
+        ));
+        assert!(matches!(
+            ErasureSetHealer::classify_heal_object_error(&Error::Storage(EcstoreError::Lock(LockError::QuorumNotReached {
+                required: 3,
+                achieved: 1,
+            }))),
+            HealObjectOutcome::Transient
+        ));
+        assert!(matches!(
+            ErasureSetHealer::classify_heal_object_error(&Error::Storage(EcstoreError::Lock(LockError::already_locked(
+                "bucket/object",
+                "node-2"
+            )))),
+            HealObjectOutcome::Transient
+        ));
+        assert!(matches!(
+            ErasureSetHealer::classify_heal_object_error(&Error::Storage(EcstoreError::Lock(LockError::InsufficientNodes {
+                required: 3,
+                available: 1,
+            }))),
+            HealObjectOutcome::Transient
+        ));
+        assert!(matches!(
+            ErasureSetHealer::classify_heal_object_error(&Error::Storage(EcstoreError::Lock(LockError::permission_denied(
+                "lock policy"
+            )))),
+            HealObjectOutcome::Failed
+        ));
+    }
+
     #[test]
     fn dangling_delete_grace_is_transient() {
         assert!(matches!(

crates/heal/src/heal/manager.rs

@@ -45,6 +45,7 @@ use tokio::{
 };
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
+use uuid::Uuid;
 
 use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
 
@@ -1903,6 +1904,10 @@ impl HealManager {
         }
     }
 
+    pub(crate) async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
+        self.storage.mrf_bucket_incarnation_retired(bucket, expected).await
+    }
+
     async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
         &self,
         mut request: HealRequest,

crates/heal/src/heal/mrf_queue.rs

@@ -1366,8 +1366,10 @@ impl MrfRuntime {
             rustfs_common::mrf_channel::consume_recorded_verified_mrf_repair_events_for(bucket.as_ref(), &mut anchors);
         }
         let remaining: HashSet<_> = anchors.into_iter().collect();
+        let previous_replay_anchor_count = self.durable_replay_anchors.len();
         self.durable_replay_anchors.retain(|anchor| remaining.contains(anchor));
-        self.dirty |= self.partial_writes.retain_unproven(&remaining);
+        let replay_anchors_changed = self.durable_replay_anchors.len() != previous_replay_anchor_count;
+        self.dirty |= replay_anchors_changed || self.partial_writes.retain_unproven(&remaining);
         for bucket in buckets {
             for event in rustfs_common::mrf_channel::take_mrf_unverified_legacy_events_for(bucket.as_ref()) {
                 if self.partial_writes.park_unverified_legacy(&event.anchor) {
@@ -1944,6 +1946,7 @@ async fn run_mrf_consumer(
                 }
             }
             _ = flush_tick.tick() => {
+                runtime.partial_writes.note_retired_generation_proofs(manager.as_ref()).await;
                 runtime.discharge_durable_replay_anchors();
                 match tick_action(
                     runtime.dirty,

crates/heal/src/heal/mrf_queue/partial_write.rs

@@ -16,10 +16,14 @@ use super::{
     HealManager, MrfConsumerConfig, MrfDurableRepairAnchor, MrfIntent, MrfLegacyRiskAcceptanceRequest, MrfQueueKey, queue_key,
     submit_mrf_heal_request,
 };
-use rustfs_common::mrf_channel::{MrfDurableAdmissionError, MrfIngressResult, release_mrf_intent, try_rearm_mrf_replay_intent};
+use rustfs_common::mrf_channel::{
+    MrfDurableAdmissionError, MrfIngressResult, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent, release_mrf_intent,
+    try_rearm_mrf_replay_intent,
+};
 use rustfs_heal_contracts::heal_channel::HealAdmissionResult;
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::sync::Arc;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tokio::time::Instant;
 use uuid::Uuid;
@@ -29,6 +33,7 @@ const MAX_OPERATOR_ACTOR_BYTES: usize = 256;
 const MAX_OPERATOR_REFERENCE_BYTES: usize = 256;
 const LIFECYCLE_CHECKPOINT_RECORD_OVERHEAD_BYTES: usize = 256;
 const MAX_UNVERIFIED_RETRY_BACKOFF: Duration = Duration::from_secs(60);
+const MRF_RETIREMENT_PROOF_RECHECK_INTERVAL: Duration = Duration::from_secs(5);
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 struct PartialWriteKey {
@@ -229,6 +234,7 @@ pub(super) struct PartialWrites {
     entries: HashMap<PartialWriteKey, Responsibility>,
     retry_order: VecDeque<PartialWriteKey>,
     retry_index: HashSet<PartialWriteKey>,
+    retirement_checks: HashMap<(Arc<str>, Uuid), (Instant, bool)>,
     bytes: usize,
 }
 
@@ -335,6 +341,66 @@ impl PartialWrites {
         self.entries.values().filter_map(|entry| entry.anchor.as_ref())
     }
 
+    pub(super) async fn note_retired_generation_proofs(&mut self, manager: &HealManager) {
+        let mut anchors_by_generation: HashMap<(Arc<str>, Uuid), Vec<MrfDurableRepairAnchor>> = HashMap::new();
+        for entry in self.entries.values() {
+            if !entry.persisted {
+                continue;
+            }
+            let Some(source_incarnation) = entry.source_bucket_incarnation_id else {
+                continue;
+            };
+            let Some(anchor) = MrfDurableRepairAnchor::from_intent(&entry.intent, source_incarnation)
+                .filter(|anchor| anchor.bucket_incarnation_id == source_incarnation)
+            else {
+                continue;
+            };
+            anchors_by_generation
+                .entry((anchor.bucket.clone(), source_incarnation))
+                .or_default()
+                .push(anchor);
+        }
+
+        self.retirement_checks
+            .retain(|generation, _| anchors_by_generation.contains_key(generation));
+        let now = Instant::now();
+        for ((bucket, incarnation), anchors) in anchors_by_generation {
+            let retired = match self.retirement_checks.get(&(bucket.clone(), incarnation)) {
+                Some((_, true)) => true,
+                Some((checked_at, false))
+                    if now.saturating_duration_since(*checked_at) < MRF_RETIREMENT_PROOF_RECHECK_INTERVAL =>
+                {
+                    false
+                }
+                _ => {
+                    let retired = manager
+                        .mrf_bucket_incarnation_retired(bucket.as_ref(), incarnation)
+                        .await
+                        .unwrap_or(false);
+                    self.retirement_checks.insert((bucket.clone(), incarnation), (now, retired));
+                    retired
+                }
+            };
+            if !retired {
+                continue;
+            }
+
+            for anchor in anchors {
+                rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
+                    kind: anchor.kind,
+                    bucket: anchor.bucket.clone(),
+                    object: anchor.object.clone(),
+                    version_id: anchor.version_id,
+                    scope: anchor.scope,
+                    delete_marker_purge: anchor.delete_marker_purge,
+                    lease: Some(anchor.lease),
+                    bucket_incarnation_id: anchor.bucket_incarnation_id,
+                    disposition: MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
+                });
+            }
+        }
+    }
+
     pub(super) fn park_unverified_legacy(&mut self, anchor: &MrfDurableRepairAnchor) -> bool {
         let key = PartialWriteKey::from_anchor(anchor);
         let Some(entry) = self.entries.get_mut(&key) else {
@@ -945,6 +1011,7 @@ mod tests {
     use super::*;
     use crate::heal::mrf_queue::MrfLegacyRiskAcceptanceRequest;
     use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
+    use crate::heal::storage_api::storage::{BucketOperations as _, DeleteBucketOptions};
     use rustfs_common::mrf_channel::{MrfKind, MrfScope, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent};
     use serial_test::serial;
     use std::sync::Arc;
@@ -1861,6 +1928,94 @@ mod tests {
         assert!(runtime.partial_writes.intent_for_responsibility(current_id).is_none());
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial]
+    async fn durable_mrf_anchor_retires_only_after_exact_bucket_generation_is_deleted() {
+        use super::super::{MrfQueue, MrfRuntime};
+
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .prefix("rustfs_mrf_retirement_proof")
+            .build()
+            .await;
+        let bucket = "mrf-retired-generation";
+        env.make_bucket(bucket, false).await;
+        let incarnation = env
+            .ecstore
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("current generation identity");
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+        let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None));
+
+        let mut item = intent("durable-object");
+        item.bucket = Arc::from(bucket);
+        let mut partial_writes = PartialWrites::default();
+        partial_writes
+            .admit_with_source_incarnation(item.clone(), Some(incarnation), 1, 8192)
+            .expect("retain exact old-generation responsibility");
+        partial_writes.mark_persisted();
+        let key = PartialWriteKey::new(&item, Some(incarnation));
+        let anchor = MrfDurableRepairAnchor::from_intent(&item, incarnation).expect("exact retirement anchor");
+        partial_writes.entries.get_mut(&key).expect("durable responsibility").anchor = Some(anchor);
+
+        assert!(
+            !manager
+                .mrf_bucket_incarnation_retired(bucket, incarnation)
+                .await
+                .expect("live generation is queryable"),
+            "a live generation cannot retire its MRF anchors"
+        );
+        env.ecstore
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("normal bucket deletion publishes durable retirement evidence");
+
+        partial_writes.note_retired_generation_proofs(manager.as_ref()).await;
+        let config = MrfConsumerConfig::default();
+        let mut runtime = MrfRuntime {
+            partial_writes,
+            queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
+            config,
+            checkpoint_owner: Uuid::new_v4(),
+            next_checkpoint_sequence: 1,
+            new_since_flush: 0,
+            dirty: false,
+            journal_on_disk: true,
+            retain_replay_journal: false,
+            durable_replay_anchors: Vec::new(),
+            replay_cleanup: None,
+            runtime_checkpoint: None,
+            backoff_until: None,
+        };
+        runtime.discharge_durable_replay_anchors();
+        assert_eq!(
+            runtime.partial_writes.depth(),
+            0,
+            "retirement proof should discharge the exact old anchor"
+        );
+        assert!(runtime.dirty, "discharging durable responsibility must publish a successor checkpoint");
+
+        let manually_removed_bucket = "mrf-manually-removed-volume";
+        env.make_bucket(manually_removed_bucket, false).await;
+        let manual_incarnation = env
+            .ecstore
+            .bucket_incarnation_id_from_disk(manually_removed_bucket)
+            .await
+            .expect("manual-removal bucket generation identity");
+        for disk_path in &env.disk_paths {
+            tokio::fs::remove_dir_all(disk_path.join(manually_removed_bucket))
+                .await
+                .expect("simulate out-of-band volume removal");
+        }
+        assert!(
+            !manager
+                .mrf_bucket_incarnation_retired(manually_removed_bucket, manual_incarnation)
+                .await
+                .expect("missing retirement proof is authoritative absence of proof"),
+            "out-of-band disk removal must not be mistaken for a durable retirement record"
+        );
+    }
+
     #[test]
     fn lifecycle_listing_uses_stable_bounded_cursors() {
         let mut writes = PartialWrites::default();

crates/heal/src/heal/storage.rs

@@ -459,6 +459,13 @@ pub trait HealStorageAPI: Send + Sync {
         Ok(None)
     }
 
+    /// Whether the exact old bucket generation has a durable deletion record.
+    /// Alternate backends must fail closed unless they can validate the same
+    /// deployment, bucket and incarnation identity.
+    async fn mrf_bucket_incarnation_retired(&self, _bucket: &str, _expected: Uuid) -> Result<bool> {
+        Ok(false)
+    }
+
     /// Aggregate usage-cache baselines for the requested buckets.
     async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
         Ok(None)
@@ -1178,6 +1185,13 @@ impl HealStorageAPI for ECStoreHealStorage {
             .map_err(Error::Storage)
     }
 
+    async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
+        self.ecstore
+            .is_bucket_incarnation_retired(bucket, expected)
+            .await
+            .map_err(Error::Storage)
+    }
+
     async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
         if buckets.is_empty() {
             return Ok(None);

crates/heal/src/heal/storage_api.rs

@@ -53,6 +53,9 @@ pub(crate) mod storage {
         BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations,
         StorageAdminApi,
     };
+
+    #[cfg(test)]
+    pub(crate) use super::storage_contracts::DeleteBucketOptions;
 }
 
 #[cfg(test)]

crates/iam/Cargo.toml

@@ -30,6 +30,7 @@ workspace = true
 
 [features]
 default = []
+test-util = []
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/iam/src/federation/mapper.rs

@@ -0,0 +1,500 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{FederatedAuthorization, VerifiedFederatedIdentity};
+use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
+use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
+use std::{collections::HashMap, sync::Arc};
+use tracing::debug;
+
+#[derive(Clone)]
+pub(crate) struct FederatedAuthorizationRule {
+    provider_id: String,
+    claim_name: String,
+    claim_prefix: String,
+    role_policy: String,
+    groups_claim: String,
+    roles_claim: String,
+}
+
+impl FederatedAuthorizationRule {
+    pub(crate) fn new(
+        provider_id: String,
+        claim_name: String,
+        claim_prefix: String,
+        role_policy: String,
+        groups_claim: String,
+        roles_claim: String,
+    ) -> Self {
+        Self {
+            provider_id,
+            claim_name,
+            claim_prefix,
+            role_policy,
+            groups_claim,
+            roles_claim,
+        }
+    }
+
+    fn as_ref(&self) -> FederatedAuthorizationRuleRef<'_> {
+        FederatedAuthorizationRuleRef::new(
+            &self.claim_name,
+            &self.claim_prefix,
+            &self.role_policy,
+            &self.groups_claim,
+            &self.roles_claim,
+        )
+    }
+}
+
+#[derive(Clone, Copy)]
+pub(crate) struct FederatedAuthorizationRuleRef<'a> {
+    claim_name: &'a str,
+    claim_prefix: &'a str,
+    role_policy: &'a str,
+    groups_claim: &'a str,
+    roles_claim: &'a str,
+}
+
+impl<'a> FederatedAuthorizationRuleRef<'a> {
+    pub(crate) fn new(
+        claim_name: &'a str,
+        claim_prefix: &'a str,
+        role_policy: &'a str,
+        groups_claim: &'a str,
+        roles_claim: &'a str,
+    ) -> Self {
+        Self {
+            claim_name,
+            claim_prefix,
+            role_policy,
+            groups_claim,
+            roles_claim,
+        }
+    }
+}
+
+/// Immutable provider-specific inputs for core federation authorization mapping.
+#[derive(Clone, Default)]
+pub struct FederatedAuthorizationRules {
+    providers: Arc<HashMap<String, FederatedAuthorizationRule>>,
+}
+
+impl FederatedAuthorizationRules {
+    pub(crate) fn new(rules: impl IntoIterator<Item = FederatedAuthorizationRule>) -> Self {
+        Self {
+            providers: Arc::new(rules.into_iter().map(|rule| (rule.provider_id.clone(), rule)).collect()),
+        }
+    }
+}
+
+/// Maps a verified external identity into RustFS authorization data.
+#[derive(Clone)]
+pub struct CoreFederatedAuthorizationMapper {
+    rules: FederatedAuthorizationRules,
+}
+
+impl CoreFederatedAuthorizationMapper {
+    pub fn new(rules: FederatedAuthorizationRules) -> Self {
+        Self { rules }
+    }
+
+    pub(crate) fn map(&self, identity: VerifiedFederatedIdentity) -> FederatedAuthorization {
+        let (provider, claims) = identity.into_parts();
+        let provider_id = provider.into_string();
+        let Some(rule) = self.rules.providers.get(&provider_id).map(FederatedAuthorizationRule::as_ref) else {
+            return FederatedAuthorization {
+                provider_id,
+                claims,
+                policies: Vec::new(),
+                group_claim_policies: Vec::new(),
+                groups: Vec::new(),
+                roles_claim_key: None,
+                roles: Vec::new(),
+            };
+        };
+        let (policies, groups) = Self::map_policies_and_groups(&provider_id, rule, &claims.groups, &claims.raw, true);
+        let group_claim_policies = Self::group_claim_policy_names(rule, &claims.groups, &claims.raw);
+        let roles_claim_key = (!rule.roles_claim.trim().is_empty()).then(|| rule.roles_claim.trim().to_string());
+        let roles = roles_claim_key
+            .as_deref()
+            .map(|claim_name| role_values(&claims.raw, claim_name))
+            .unwrap_or_default();
+
+        FederatedAuthorization {
+            provider_id,
+            claims,
+            policies,
+            group_claim_policies,
+            groups,
+            roles_claim_key,
+            roles,
+        }
+    }
+
+    pub(crate) fn map_policies_and_groups(
+        provider_id: &str,
+        rule: FederatedAuthorizationRuleRef<'_>,
+        source_groups: &[String],
+        raw_claims: &HashMap<String, serde_json::Value>,
+        emit_diagnostics: bool,
+    ) -> (Vec<String>, Vec<String>) {
+        let has_role_policy = !rule.role_policy.trim().is_empty();
+        let mut policies: Vec<String> = if has_role_policy {
+            rule.role_policy
+                .split(',')
+                .map(str::trim)
+                .filter(|policy| !policy.is_empty())
+                .map(ToOwned::to_owned)
+                .collect()
+        } else {
+            source_groups
+                .iter()
+                .map(|group| policy_name(rule.claim_prefix, group))
+                .collect()
+        };
+
+        if !has_role_policy && rule.claim_name != rule.groups_claim {
+            policies.extend(
+                claim_values(raw_claims, rule.claim_name)
+                    .into_iter()
+                    .map(|value| policy_name(rule.claim_prefix, &value)),
+            );
+        }
+        policies.sort();
+        policies.dedup();
+
+        let mut groups = source_groups.to_vec();
+        groups.sort();
+        groups.dedup();
+
+        if emit_diagnostics {
+            record_mapping_diagnostics(provider_id, rule, raw_claims, &policies, &groups);
+        }
+
+        (policies, groups)
+    }
+
+    pub(crate) fn group_claim_policy_names(
+        rule: FederatedAuthorizationRuleRef<'_>,
+        source_groups: &[String],
+        raw_claims: &HashMap<String, serde_json::Value>,
+    ) -> Vec<String> {
+        if !rule.role_policy.trim().is_empty() {
+            return Vec::new();
+        }
+
+        let explicit_policy_names: Vec<String> = if rule.claim_name != rule.groups_claim {
+            claim_values(raw_claims, rule.claim_name)
+                .iter()
+                .map(|value| policy_name(rule.claim_prefix, value))
+                .collect()
+        } else {
+            Vec::new()
+        };
+        let mut group_policies: Vec<String> = source_groups
+            .iter()
+            .map(|group| policy_name(rule.claim_prefix, group))
+            .filter(|policy| !explicit_policy_names.contains(policy))
+            .collect();
+        group_policies.sort();
+        group_policies.dedup();
+        group_policies
+    }
+}
+
+fn record_mapping_diagnostics(
+    provider_id: &str,
+    rule: FederatedAuthorizationRuleRef<'_>,
+    raw_claims: &HashMap<String, serde_json::Value>,
+    policies: &[String],
+    groups: &[String],
+) {
+    let (claim_name_lookup, claim_name_type) = claim_lookup_details(raw_claims, rule.claim_name);
+    let (groups_claim_lookup, groups_claim_type) = claim_lookup_details(raw_claims, rule.groups_claim);
+    let (roles_claim_lookup, roles_claim_type) = claim_lookup_details(raw_claims, rule.roles_claim);
+    let claim_name_value_count = claim_values(raw_claims, rule.claim_name).len();
+    let groups_claim_value_count = claim_values(raw_claims, rule.groups_claim).len();
+    let roles_claim_value_count = claim_values(raw_claims, rule.roles_claim).len();
+
+    debug!(
+        event = EVENT_OIDC_DIAGNOSTICS,
+        component = LOG_COMPONENT_IAM,
+        subsystem = LOG_SUBSYSTEM_OIDC,
+        result = "claims_policy_mapped",
+        provider_id,
+        claim_name = %rule.claim_name,
+        claim_prefix = %rule.claim_prefix,
+        groups_claim = %rule.groups_claim,
+        roles_claim = %rule.roles_claim,
+        role_policy_configured = !rule.role_policy.trim().is_empty(),
+        policy_count = policies.len(),
+        group_count = groups.len(),
+        raw_claim_key_count = raw_claims.len(),
+        claim_name_lookup,
+        claim_name_type,
+        claim_name_value_count,
+        groups_claim_lookup,
+        groups_claim_type,
+        groups_claim_value_count,
+        roles_claim_lookup,
+        roles_claim_type,
+        roles_claim_value_count,
+        "oidc claims mapped to policies"
+    );
+}
+
+fn policy_name(prefix: &str, value: &str) -> String {
+    format!("{prefix}{value}")
+}
+
+fn claim_values(claims: &HashMap<String, serde_json::Value>, claim_name: &str) -> Vec<String> {
+    match get_claim_case_insensitive(claims, claim_name) {
+        ClaimLookup::Found(serde_json::Value::Array(values)) => values
+            .iter()
+            .filter_map(|value| value.as_str().map(ToOwned::to_owned))
+            .collect(),
+        ClaimLookup::Found(serde_json::Value::String(value)) => value.split(',').map(str::trim).map(ToOwned::to_owned).collect(),
+        ClaimLookup::Missing | ClaimLookup::Ambiguous | ClaimLookup::Found(_) => Vec::new(),
+    }
+}
+
+fn role_values(claims: &HashMap<String, serde_json::Value>, claim_name: &str) -> Vec<String> {
+    match get_claim_case_insensitive(claims, claim_name) {
+        ClaimLookup::Found(serde_json::Value::Array(values)) => values
+            .iter()
+            .filter_map(|value| value.as_str().map(ToOwned::to_owned))
+            .collect(),
+        ClaimLookup::Found(serde_json::Value::String(value)) => value
+            .split(',')
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+            .map(ToOwned::to_owned)
+            .collect(),
+        ClaimLookup::Missing | ClaimLookup::Ambiguous | ClaimLookup::Found(_) => Vec::new(),
+    }
+}
+
+fn claim_lookup_details(claims: &HashMap<String, serde_json::Value>, claim_name: &str) -> (&'static str, &'static str) {
+    match get_claim_case_insensitive(claims, claim_name) {
+        ClaimLookup::Found(value) => ("found", claim_value_type(value)),
+        ClaimLookup::Missing => ("missing", "none"),
+        ClaimLookup::Ambiguous => ("ambiguous", "none"),
+    }
+}
+
+fn claim_value_type(value: &serde_json::Value) -> &'static str {
+    match value {
+        serde_json::Value::Null => "null",
+        serde_json::Value::Bool(_) => "bool",
+        serde_json::Value::Number(_) => "number",
+        serde_json::Value::String(_) => "string",
+        serde_json::Value::Array(_) => "array",
+        serde_json::Value::Object(_) => "object",
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity};
+    use serde_json::json;
+
+    fn mapper(rule: FederatedAuthorizationRule) -> CoreFederatedAuthorizationMapper {
+        CoreFederatedAuthorizationMapper::new(FederatedAuthorizationRules::new([rule]))
+    }
+
+    fn identity(provider_id: &str) -> VerifiedFederatedIdentity {
+        VerifiedFederatedIdentity::from_claims(
+            FederatedProviderRef::new(provider_id.to_string()),
+            FederatedClaims {
+                sub: "subject".to_string(),
+                email: "user@example.test".to_string(),
+                username: "user".to_string(),
+                groups: vec![
+                    "source-ops".to_string(),
+                    "source-developers".to_string(),
+                    "source-ops".to_string(),
+                ],
+                raw: HashMap::from([
+                    ("Policy".to_string(), json!(["explicit", "source-ops"])),
+                    ("Roles".to_string(), json!("reader, writer, ,auditor")),
+                ]),
+            },
+        )
+    }
+
+    #[test]
+    fn claim_mapping_preserves_policy_group_role_and_case_behavior() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "policy".to_string(),
+            "mapped-".to_string(),
+            String::new(),
+            "groups".to_string(),
+            "roles".to_string(),
+        ));
+
+        let authorization = mapper.map(identity("corp"));
+
+        assert_eq!(
+            authorization.policies,
+            ["mapped-explicit", "mapped-source-developers", "mapped-source-ops"]
+        );
+        assert_eq!(authorization.group_claim_policies, ["mapped-source-developers"]);
+        assert_eq!(authorization.groups, ["source-developers", "source-ops"]);
+        assert_eq!(authorization.roles_claim_key.as_deref(), Some("roles"));
+        assert_eq!(authorization.roles, ["reader", "writer", "auditor"]);
+    }
+
+    #[test]
+    fn role_policy_mode_keeps_groups_without_group_derived_policies() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "policy".to_string(),
+            "mapped-".to_string(),
+            " readonly, readwrite, readonly ".to_string(),
+            "groups".to_string(),
+            String::new(),
+        ));
+
+        let authorization = mapper.map(identity("corp"));
+
+        assert_eq!(authorization.policies, ["readonly", "readwrite"]);
+        assert!(authorization.group_claim_policies.is_empty());
+        assert_eq!(authorization.groups, ["source-developers", "source-ops"]);
+    }
+
+    #[test]
+    fn missing_provider_rules_preserve_identity_without_authorization_context() {
+        let authorization =
+            CoreFederatedAuthorizationMapper::new(FederatedAuthorizationRules::default()).map(identity("missing"));
+
+        assert_eq!(authorization.provider_id, "missing");
+        assert_eq!(authorization.claims.username, "user");
+        assert!(!authorization.has_authorization_context());
+    }
+
+    #[test]
+    fn exact_claim_name_wins_over_case_insensitive_candidates() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "policy".to_string(),
+            "mapped-".to_string(),
+            String::new(),
+            "groups".to_string(),
+            String::new(),
+        ));
+        let identity = VerifiedFederatedIdentity::from_claims(
+            FederatedProviderRef::new("corp".to_string()),
+            FederatedClaims {
+                sub: String::new(),
+                email: String::new(),
+                username: String::new(),
+                groups: Vec::new(),
+                raw: HashMap::from([
+                    ("policy".to_string(), json!("exact")),
+                    ("Policy".to_string(), json!("fallback")),
+                ]),
+            },
+        );
+
+        let authorization = mapper.map(identity);
+
+        assert_eq!(authorization.policies, ["mapped-exact"]);
+    }
+
+    #[test]
+    fn ambiguous_case_insensitive_claim_name_is_ignored() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "pOlIcY".to_string(),
+            "mapped-".to_string(),
+            String::new(),
+            "groups".to_string(),
+            String::new(),
+        ));
+        let identity = VerifiedFederatedIdentity::from_claims(
+            FederatedProviderRef::new("corp".to_string()),
+            FederatedClaims {
+                sub: String::new(),
+                email: String::new(),
+                username: String::new(),
+                groups: vec!["source-ops".to_string()],
+                raw: HashMap::from([
+                    ("Policy".to_string(), json!("first")),
+                    ("POLICY".to_string(), json!("second")),
+                ]),
+            },
+        );
+
+        let authorization = mapper.map(identity);
+
+        assert_eq!(authorization.policies, ["mapped-source-ops"]);
+        assert_eq!(claim_lookup_details(&authorization.claims.raw, "pOlIcY"), ("ambiguous", "none"));
+        assert_eq!(claim_lookup_details(&authorization.claims.raw, "missing"), ("missing", "none"));
+    }
+
+    #[test]
+    fn comma_separated_claim_values_keep_existing_empty_value_behavior() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "Policy".to_string(),
+            "mapped-".to_string(),
+            String::new(),
+            "groups".to_string(),
+            String::new(),
+        ));
+        let identity = VerifiedFederatedIdentity::from_claims(
+            FederatedProviderRef::new("corp".to_string()),
+            FederatedClaims {
+                sub: String::new(),
+                email: String::new(),
+                username: String::new(),
+                groups: Vec::new(),
+                raw: HashMap::from([("Policy".to_string(), json!(" explicit, ,audit "))]),
+            },
+        );
+
+        let authorization = mapper.map(identity);
+
+        assert_eq!(authorization.policies, ["mapped-", "mapped-audit", "mapped-explicit"]);
+    }
+
+    #[test]
+    fn role_array_values_keep_existing_empty_value_behavior() {
+        let mapper = mapper(FederatedAuthorizationRule::new(
+            "corp".to_string(),
+            "policy".to_string(),
+            String::new(),
+            String::new(),
+            "groups".to_string(),
+            "roles".to_string(),
+        ));
+        let identity = VerifiedFederatedIdentity::from_claims(
+            FederatedProviderRef::new("corp".to_string()),
+            FederatedClaims {
+                sub: String::new(),
+                email: String::new(),
+                username: String::new(),
+                groups: Vec::new(),
+                raw: HashMap::from([("roles".to_string(), json!(["reader", "", 7, "writer"]))]),
+            },
+        );
+
+        let authorization = mapper.map(identity);
+
+        assert_eq!(authorization.roles, ["reader", "", "writer"]);
+    }
+}

crates/iam/src/federation/mod.rs

@@ -14,18 +14,21 @@
 
 mod binding;
 mod error;
+mod mapper;
 mod model;
 pub mod oidc;
 mod provider;
-mod registry;
 mod transaction;
 
 pub use binding::FederatedSessionBinding;
 pub use error::{FederatedSessionBindingError, FederationError, Result};
+pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRules};
+pub(crate) use mapper::{FederatedAuthorizationRule, FederatedAuthorizationRuleRef};
+pub(crate) use model::OpaqueLogoutContinuation;
 pub use model::{
-    FederatedAuthorization, FederatedClaims, FederatedCodeExchange, FederatedLoginSession, FederatedSession,
-    FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM,
+    FederatedAuthorization, FederatedClaims, FederatedLoginSession, FederatedProviderRef, FederatedSession,
+    FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, VerifiedFederatedIdentity,
 };
-pub use provider::FederatedIdentityProvider;
-pub use registry::FederatedIdentityRegistry;
+pub(crate) use provider::{FederatedProviderQuery, StandardOidcAuthentication, VerifiedFederatedCodeExchange};
+pub use provider::{FederatedProviderView, FederatedRedirectPolicy};
 pub use transaction::FederatedIdentityService;

crates/iam/src/federation/model.rs

@@ -15,7 +15,7 @@
 use rustfs_credentials::Credentials;
 use rustfs_utils::HashAlgorithm;
 use serde_json::Value;
-use std::collections::HashMap;
+use std::{collections::HashMap, fmt};
 
 pub const OIDC_VIRTUAL_PARENT_CLAIM: &str = "x-rustfs-internal-oidc-parent";
 
@@ -42,6 +42,89 @@ impl FederatedClaims {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FederatedProviderRef {
+    provider_id: String,
+}
+
+impl FederatedProviderRef {
+    /// Retains the configured provider instance ID exactly as selected by the authenticator.
+    pub(crate) fn new(provider_id: String) -> Self {
+        Self { provider_id }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.provider_id
+    }
+
+    pub(crate) fn into_string(self) -> String {
+        self.provider_id
+    }
+}
+
+#[derive(Clone)]
+pub struct VerifiedFederatedIdentity {
+    provider: FederatedProviderRef,
+    claims: FederatedClaims,
+}
+
+impl fmt::Debug for VerifiedFederatedIdentity {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("VerifiedFederatedIdentity")
+            .field("provider", &self.provider)
+            .field("issuer_present", &self.issuer().is_some())
+            .field("subject_present", &!self.claims.sub.is_empty())
+            .field("email_present", &!self.claims.email.is_empty())
+            .field("username_present", &!self.claims.username.is_empty())
+            .field("source_group_count", &self.claims.groups.len())
+            .field("attribute_count", &self.claims.raw.len())
+            .finish()
+    }
+}
+
+impl VerifiedFederatedIdentity {
+    pub(crate) fn from_claims(provider: FederatedProviderRef, claims: FederatedClaims) -> Self {
+        Self { provider, claims }
+    }
+
+    pub fn provider(&self) -> &FederatedProviderRef {
+        &self.provider
+    }
+
+    pub fn issuer(&self) -> Option<&str> {
+        self.claims.raw.get("iss").and_then(Value::as_str)
+    }
+
+    pub fn subject(&self) -> &str {
+        &self.claims.sub
+    }
+
+    pub fn email(&self) -> &str {
+        &self.claims.email
+    }
+
+    pub fn username(&self) -> &str {
+        &self.claims.username
+    }
+
+    pub fn source_groups(&self) -> &[String] {
+        &self.claims.groups
+    }
+
+    pub fn attributes(&self) -> &HashMap<String, Value> {
+        &self.claims.raw
+    }
+
+    pub fn session_identity(&self) -> String {
+        self.claims.session_identity()
+    }
+
+    pub(crate) fn into_parts(self) -> (FederatedProviderRef, FederatedClaims) {
+        (self.provider, self.claims)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct FederatedAuthorization {
     pub provider_id: String,
@@ -81,11 +164,30 @@ impl FederatedAuthorization {
     }
 }
 
-#[derive(Debug)]
-pub struct FederatedCodeExchange {
-    pub authorization: FederatedAuthorization,
-    pub redirect_after: Option<String>,
-    pub id_token: String,
+/// Carries the provider-bound ID token between code exchange and logout token creation.
+pub(crate) struct OpaqueLogoutContinuation {
+    provider: FederatedProviderRef,
+    id_token: String,
+}
+
+impl OpaqueLogoutContinuation {
+    pub(crate) fn new(provider: FederatedProviderRef, id_token: String) -> Self {
+        Self { provider, id_token }
+    }
+
+    pub(crate) fn into_parts(self) -> (FederatedProviderRef, String) {
+        (self.provider, self.id_token)
+    }
+}
+
+impl fmt::Debug for OpaqueLogoutContinuation {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("OpaqueLogoutContinuation")
+            .field("provider", &self.provider)
+            .field("id_token", &"[REDACTED]")
+            .finish()
+    }
 }
 
 #[derive(Debug)]
@@ -208,4 +310,67 @@ mod tests {
 
         assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent());
     }
+
+    #[test]
+    fn verified_identity_preserves_exact_values_and_distinct_groups() {
+        let attributes = HashMap::from([
+            ("iss".to_string(), Value::String(" https://issuer.example/path ".to_string())),
+            ("department".to_string(), Value::String("identity-attribute-poison".to_string())),
+        ]);
+        let identity = VerifiedFederatedIdentity {
+            provider: FederatedProviderRef::new(" Corp Provider ".to_string()),
+            claims: FederatedClaims {
+                sub: " subject ".to_string(),
+                email: " email@example.test ".to_string(),
+                username: " username ".to_string(),
+                groups: vec![" source-b ".to_string(), "source-a".to_string(), " source-b ".to_string()],
+                raw: attributes.clone(),
+            },
+        };
+        let debug = format!("{identity:?}");
+
+        assert_eq!(identity.provider().as_str(), " Corp Provider ");
+        assert_eq!(identity.issuer(), Some(" https://issuer.example/path "));
+        assert_eq!(identity.subject(), " subject ");
+        assert_eq!(identity.email(), " email@example.test ");
+        assert_eq!(identity.username(), " username ");
+        assert_eq!(identity.source_groups(), [" source-b ", "source-a", " source-b "]);
+        assert_eq!(identity.attributes(), &attributes);
+        assert!(!debug.contains(" subject "));
+        assert!(!debug.contains(" email@example.test "));
+        assert!(!debug.contains(" username "));
+        assert!(!debug.contains(" source-b "));
+        assert!(!debug.contains("identity-attribute-poison"));
+
+        let authorization = FederatedAuthorization {
+            provider_id: " Corp Provider ".to_string(),
+            claims: FederatedClaims {
+                sub: "subject".to_string(),
+                email: String::new(),
+                username: String::new(),
+                groups: vec!["different-source-group".to_string()],
+                raw: HashMap::new(),
+            },
+            policies: Vec::new(),
+            group_claim_policies: Vec::new(),
+            groups: vec!["mapped-a".to_string(), "mapped-b".to_string()],
+            roles_claim_key: None,
+            roles: Vec::new(),
+        };
+        assert_eq!(authorization.claims.groups, ["different-source-group"]);
+        assert_eq!(authorization.groups, ["mapped-a", "mapped-b"]);
+    }
+
+    #[test]
+    fn logout_continuation_binds_provider_and_redacts_token() {
+        let continuation =
+            OpaqueLogoutContinuation::new(FederatedProviderRef::new("standard_oidc".to_string()), "secret-id-token".to_string());
+        let continuation_debug = format!("{continuation:?}");
+        let (provider, id_token) = continuation.into_parts();
+
+        assert_eq!(provider.as_str(), "standard_oidc");
+        assert_eq!(id_token, "secret-id-token");
+        assert!(!continuation_debug.contains("secret-id-token"));
+        assert!(continuation_debug.contains("[REDACTED]"));
+    }
 }

crates/iam/src/federation/oidc/claims.rs

@@ -12,111 +12,39 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::config::roles_claim_key;
 use crate::{
-    federation::{FederatedAuthorization, FederatedClaims},
-    oidc::{OidcClaims, OidcSys},
+    federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity},
+    oidc::OidcClaims,
 };
-use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
 
-fn string_list_claim(claims: &OidcClaims, claim_name: &str) -> Vec<String> {
-    match get_claim_case_insensitive(&claims.raw, claim_name) {
-        ClaimLookup::Found(serde_json::Value::Array(values)) => values
-            .iter()
-            .filter_map(|value| value.as_str().map(ToOwned::to_owned))
-            .collect(),
-        ClaimLookup::Found(serde_json::Value::String(value)) => value
-            .split(',')
-            .map(str::trim)
-            .filter(|value| !value.is_empty())
-            .map(ToOwned::to_owned)
-            .collect(),
-        ClaimLookup::Missing | ClaimLookup::Ambiguous | ClaimLookup::Found(_) => Vec::new(),
-    }
-}
-
-pub(super) fn authorization(oidc: &OidcSys, provider_id: String, claims: OidcClaims) -> FederatedAuthorization {
-    let (policies, groups) = oidc.map_claims_to_policies(&provider_id, &claims);
-    let group_claim_policies = oidc.group_claim_policy_names(&provider_id, &claims);
-    let roles_claim_key = roles_claim_key(oidc, &provider_id);
-    let roles = roles_claim_key
-        .as_deref()
-        .map(|claim_name| string_list_claim(&claims, claim_name))
-        .unwrap_or_default();
-
-    FederatedAuthorization {
-        provider_id,
-        claims: FederatedClaims {
+pub(super) fn verified_identity(provider: FederatedProviderRef, claims: OidcClaims) -> VerifiedFederatedIdentity {
+    VerifiedFederatedIdentity::from_claims(
+        provider,
+        FederatedClaims {
             sub: claims.sub,
             email: claims.email,
             username: claims.username,
             groups: claims.groups,
             raw: claims.raw,
         },
-        policies,
-        group_claim_policies,
-        groups,
-        roles_claim_key,
-        roles,
-    }
+    )
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::oidc::{make_test_sys, test_config};
     use serde_json::json;
     use std::collections::HashMap;
 
     #[test]
-    fn string_list_claim_matches_existing_array_and_csv_behavior() {
-        let claims = OidcClaims {
-            raw: HashMap::from([
-                ("array_roles".to_string(), json!(["reader", 7, "writer"])),
-                ("csv_roles".to_string(), json!("reader, writer, ,auditor")),
-            ]),
-            ..Default::default()
-        };
-
-        assert_eq!(string_list_claim(&claims, "array_roles"), ["reader", "writer"]);
-        assert_eq!(string_list_claim(&claims, "csv_roles"), ["reader", "writer", "auditor"]);
-    }
-
-    #[test]
-    fn string_list_claim_preserves_exact_and_ambiguous_match_behavior() {
-        let exact = OidcClaims {
-            raw: HashMap::from([
-                ("Roles".to_string(), json!(["mixed-case"])),
-                ("roles".to_string(), json!(["exact-match"])),
-            ]),
-            ..Default::default()
-        };
-        assert_eq!(string_list_claim(&exact, "roles"), ["exact-match"]);
-
-        let ambiguous = OidcClaims {
-            raw: HashMap::from([
-                ("Roles".to_string(), json!(["mixed-case"])),
-                ("ROLES".to_string(), json!(["upper-case"])),
-            ]),
-            ..Default::default()
-        };
-        assert!(string_list_claim(&ambiguous, "roles").is_empty());
-    }
-
-    #[test]
-    fn authorization_preserves_verified_claims_and_keeps_source_groups_distinct() {
-        let mut config = test_config("corp");
-        config.claim_prefix = "mapped-".to_string();
-        config.roles_claim = "roles".to_string();
-        let oidc = make_test_sys(vec![config]);
+    fn verified_identity_preserves_provider_and_claim_values() {
         let raw = HashMap::from([
             ("iss".to_string(), json!("https://corp.example.test")),
             ("department".to_string(), json!("engineering")),
             ("roles".to_string(), json!(["reader", "admin"])),
         ]);
-        let authorization = authorization(
-            &oidc,
-            "corp".to_string(),
+        let identity = verified_identity(
+            FederatedProviderRef::new("corp".to_string()),
             OidcClaims {
                 sub: " subject-123 ".to_string(),
                 email: " user@example.test ".to_string(),
@@ -130,16 +58,11 @@ mod tests {
             },
         );
 
-        assert_eq!(authorization.provider_id, "corp");
-        assert_eq!(authorization.claims.sub, " subject-123 ");
-        assert_eq!(authorization.claims.email, " user@example.test ");
-        assert_eq!(authorization.claims.username, " user ");
-        assert_eq!(authorization.claims.groups, ["source-ops", "source-developers", "source-ops"]);
-        assert_eq!(authorization.claims.raw, raw);
-        assert_eq!(authorization.policies, ["mapped-source-developers", "mapped-source-ops"]);
-        assert_eq!(authorization.group_claim_policies, ["mapped-source-developers", "mapped-source-ops"]);
-        assert_eq!(authorization.groups, ["source-developers", "source-ops"]);
-        assert_eq!(authorization.roles_claim_key.as_deref(), Some("roles"));
-        assert_eq!(authorization.roles, ["reader", "admin"]);
+        assert_eq!(identity.provider().as_str(), "corp");
+        assert_eq!(identity.subject(), " subject-123 ");
+        assert_eq!(identity.email(), " user@example.test ");
+        assert_eq!(identity.username(), " user ");
+        assert_eq!(identity.source_groups(), ["source-ops", "source-developers", "source-ops"]);
+        assert_eq!(identity.attributes(), &raw);
     }
 }

crates/iam/src/federation/oidc/config.rs

@@ -12,16 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::oidc::{OidcProviderConfig, OidcSys};
+use crate::{
+    federation::{FederatedAuthorizationRule, FederatedAuthorizationRules, FederatedRedirectPolicy},
+    oidc::{OidcConfigSnapshot, OidcSys},
+};
 
-pub(super) fn provider_config<'a>(oidc: &'a OidcSys, id: &str) -> Option<&'a OidcProviderConfig> {
-    oidc.get_provider_config(id)
+pub(crate) fn authorization_rules(oidc: &OidcSys) -> FederatedAuthorizationRules {
+    FederatedAuthorizationRules::new(oidc.provider_configs().map(|config| {
+        FederatedAuthorizationRule::new(
+            config.id.clone(),
+            config.claim_name.clone(),
+            config.claim_prefix.clone(),
+            config.role_policy.clone(),
+            config.groups_claim.clone(),
+            config.roles_claim.clone(),
+        )
+    }))
 }
 
-pub(super) fn roles_claim_key(oidc: &OidcSys, provider_id: &str) -> Option<String> {
-    provider_config(oidc, provider_id)
-        .map(|config| config.roles_claim.trim().to_string())
-        .filter(|claim| !claim.is_empty())
+/// Read-only access to the active standard OIDC configuration.
+///
+/// This interface stays separate from authentication and generic provider
+/// views because its snapshots include OIDC-specific fields and secrets.
+pub trait OidcConfigQuery: Send + Sync {
+    fn config_snapshot(&self) -> OidcConfigSnapshot;
+}
+
+pub(super) fn redirect_policy(oidc: &OidcSys, provider_id: &str) -> Option<FederatedRedirectPolicy> {
+    oidc.get_provider_config(provider_id).map(|config| FederatedRedirectPolicy {
+        redirect_uri: config.redirect_uri.clone(),
+        allow_request_origin: config.redirect_uri_dynamic,
+    })
 }
 
 #[cfg(test)]
@@ -29,8 +50,19 @@ mod tests {
     use super::*;
 
     #[test]
-    fn roles_claim_key_requires_explicit_provider_config() {
-        let oidc = OidcSys::empty().expect("empty OIDC configuration should be valid");
-        assert_eq!(roles_claim_key(&oidc, "default"), None);
+    fn redirect_policy_keeps_configured_uri_and_dynamic_setting() {
+        let mut config = crate::oidc::test_config("default");
+        config.redirect_uri = Some("https://console.example.test/callback".to_string());
+        config.redirect_uri_dynamic = false;
+        let oidc = crate::oidc::make_test_sys(vec![config]);
+
+        assert_eq!(
+            redirect_policy(&oidc, "default"),
+            Some(FederatedRedirectPolicy {
+                redirect_uri: Some("https://console.example.test/callback".to_string()),
+                allow_request_origin: false,
+            })
+        );
+        assert_eq!(redirect_policy(&oidc, "missing"), None);
     }
 }

crates/iam/src/federation/oidc/discovery.rs

@@ -12,12 +12,52 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::oidc::{OidcProviderSummary, OidcSys};
+use crate::{federation::FederatedProviderView, oidc::OidcSys};
 
-pub(super) fn list_providers(oidc: &OidcSys) -> Vec<OidcProviderSummary> {
-    oidc.list_providers()
+fn provider_view(provider: &crate::oidc::OidcProviderConfig) -> FederatedProviderView {
+    FederatedProviderView {
+        provider_id: provider.id.clone(),
+        display_name: provider.display_name.clone(),
+    }
 }
 
-pub(super) fn list_visible_providers(oidc: &OidcSys) -> Vec<OidcProviderSummary> {
-    oidc.list_visible_providers()
+pub(super) fn list_providers(oidc: &OidcSys) -> Vec<FederatedProviderView> {
+    oidc.provider_configs().map(provider_view).collect()
+}
+
+pub(super) fn list_visible_providers(oidc: &OidcSys) -> Vec<FederatedProviderView> {
+    oidc.provider_configs()
+        .filter(|provider| !provider.hide_from_ui)
+        .map(provider_view)
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn provider_views_preserve_names_and_visibility() {
+        let mut visible = crate::oidc::test_config("default");
+        visible.display_name = "Company SSO".to_string();
+        let mut hidden = crate::oidc::test_config("workload");
+        hidden.hide_from_ui = true;
+        let oidc = crate::oidc::make_test_sys(vec![visible, hidden]);
+
+        let all = list_providers(&oidc);
+        assert_eq!(all.len(), 2);
+        assert!(
+            all.iter()
+                .any(|provider| { provider.provider_id == "default" && provider.display_name == "Company SSO" })
+        );
+        assert!(all.iter().any(|provider| provider.provider_id == "workload"));
+
+        assert_eq!(
+            list_visible_providers(&oidc),
+            vec![FederatedProviderView {
+                provider_id: "default".to_string(),
+                display_name: "Company SSO".to_string(),
+            }]
+        );
+    }
 }

crates/iam/src/federation/oidc/flow.rs

@@ -12,68 +12,98 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::config::OidcConfigQuery;
 use super::{claims, config, discovery, http};
 use crate::{
-    federation::{FederatedAuthorization, FederatedCodeExchange, FederatedIdentityProvider, FederationError, Result},
-    oidc::{OidcProviderConfig, OidcProviderSummary, OidcSys},
+    federation::{
+        CoreFederatedAuthorizationMapper, FederatedAuthorizationRules, FederatedIdentityService, FederatedProviderQuery,
+        FederatedProviderRef, FederatedProviderView, FederatedRedirectPolicy, FederationError, OpaqueLogoutContinuation, Result,
+        StandardOidcAuthentication, VerifiedFederatedCodeExchange,
+    },
+    oidc::{OidcConfigSnapshot, OidcSys},
 };
 use std::sync::Arc;
 
 pub struct StandardOidcAdapter {
     oidc: Arc<OidcSys>,
+    authorization_rules: FederatedAuthorizationRules,
 }
 
 impl StandardOidcAdapter {
     pub fn new(oidc: Arc<OidcSys>) -> Self {
-        Self { oidc }
+        let authorization_rules = config::authorization_rules(&oidc);
+        Self {
+            oidc,
+            authorization_rules,
+        }
+    }
+
+    pub fn authorization_rules(&self) -> FederatedAuthorizationRules {
+        self.authorization_rules.clone()
+    }
+
+    pub fn into_service(self: Arc<Self>, mapper: CoreFederatedAuthorizationMapper) -> FederatedIdentityService {
+        let authentication: Arc<dyn StandardOidcAuthentication> = self.clone();
+        let provider_query: Arc<dyn FederatedProviderQuery> = self;
+        FederatedIdentityService::from_standard_oidc_parts(provider_query, authentication, mapper)
+    }
+}
+
+impl OidcConfigQuery for StandardOidcAdapter {
+    fn config_snapshot(&self) -> OidcConfigSnapshot {
+        self.oidc.config_snapshot()
     }
 }
 
 #[async_trait::async_trait]
-impl FederatedIdentityProvider for StandardOidcAdapter {
-    fn has_providers(&self) -> bool {
-        self.oidc.has_providers()
-    }
-
-    fn list_providers(&self) -> Vec<OidcProviderSummary> {
-        discovery::list_providers(&self.oidc)
-    }
-
-    fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
-        discovery::list_visible_providers(&self.oidc)
-    }
-
-    fn provider_config(&self, id: &str) -> Option<&OidcProviderConfig> {
-        config::provider_config(&self.oidc, id)
-    }
-
+impl StandardOidcAuthentication for StandardOidcAdapter {
     async fn authorize_url(&self, provider_id: &str, redirect_uri: &str, redirect_after: Option<String>) -> Result<String> {
         http::authorize_url(&self.oidc, provider_id, redirect_uri, redirect_after).await
     }
 
-    async fn exchange_code(&self, state: &str, code: &str, redirect_uri: &str) -> Result<FederatedCodeExchange> {
+    async fn exchange_identity(&self, state: &str, code: &str, redirect_uri: &str) -> Result<VerifiedFederatedCodeExchange> {
         let (oidc_claims, provider_id, session, id_token) = http::exchange_code(&self.oidc, state, code, redirect_uri).await?;
-        Ok(FederatedCodeExchange {
-            authorization: claims::authorization(&self.oidc, provider_id, oidc_claims),
-            redirect_after: session.redirect_after,
-            id_token,
-        })
+        let provider = FederatedProviderRef::new(provider_id);
+        Ok(VerifiedFederatedCodeExchange::new(
+            claims::verified_identity(provider.clone(), oidc_claims),
+            session.redirect_after,
+            OpaqueLogoutContinuation::new(provider, id_token),
+        ))
     }
 
-    async fn verify_web_identity_token(&self, jwt: &str) -> Result<FederatedAuthorization> {
+    async fn verify_identity(&self, jwt: &str) -> Result<crate::federation::VerifiedFederatedIdentity> {
         let (oidc_claims, provider_id) = self
             .oidc
             .verify_web_identity_token(jwt)
             .await
             .map_err(FederationError::TokenVerification)?;
-        Ok(claims::authorization(&self.oidc, provider_id, oidc_claims))
+        Ok(claims::verified_identity(FederatedProviderRef::new(provider_id), oidc_claims))
     }
 
-    async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String> {
-        http::create_logout_token(&self.oidc, provider_id, id_token).await
+    async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> Result<String> {
+        let (provider, id_token) = continuation.into_parts();
+        http::create_logout_token(&self.oidc, provider.as_str(), &id_token).await
     }
 
     async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>> {
         http::build_logout_url(&self.oidc, logout_token, post_logout_redirect_uri).await
     }
 }
+
+impl FederatedProviderQuery for StandardOidcAdapter {
+    fn has_providers(&self) -> bool {
+        self.oidc.has_providers()
+    }
+
+    fn list_providers(&self) -> Vec<FederatedProviderView> {
+        discovery::list_providers(&self.oidc)
+    }
+
+    fn list_visible_providers(&self) -> Vec<FederatedProviderView> {
+        discovery::list_visible_providers(&self.oidc)
+    }
+
+    fn redirect_policy(&self, provider_id: &str) -> Option<FederatedRedirectPolicy> {
+        config::redirect_policy(&self.oidc, provider_id)
+    }
+}

crates/iam/src/federation/oidc/mod.rs

@@ -18,4 +18,5 @@ mod discovery;
 mod flow;
 mod http;
 
+pub use config::OidcConfigQuery;
 pub use flow::StandardOidcAdapter;

crates/iam/src/federation/provider.rs

@@ -12,26 +12,80 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::{FederatedAuthorization, FederatedCodeExchange, Result};
-use crate::oidc::{OidcProviderConfig, OidcProviderSummary};
+use super::{OpaqueLogoutContinuation, Result, VerifiedFederatedIdentity};
+use serde::{Deserialize, Serialize};
+
+/// Provider data exposed to login discovery consumers.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct FederatedProviderView {
+    pub provider_id: String,
+    pub display_name: String,
+}
+
+/// Redirect inputs needed by the browser authorization flow.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FederatedRedirectPolicy {
+    pub redirect_uri: Option<String>,
+    pub allow_request_origin: bool,
+}
+
+pub(crate) struct VerifiedFederatedCodeExchange {
+    identity: VerifiedFederatedIdentity,
+    redirect_after: Option<String>,
+    logout_continuation: OpaqueLogoutContinuation,
+}
+
+impl VerifiedFederatedCodeExchange {
+    pub(crate) fn new(
+        identity: VerifiedFederatedIdentity,
+        redirect_after: Option<String>,
+        logout_continuation: OpaqueLogoutContinuation,
+    ) -> Self {
+        Self {
+            identity,
+            redirect_after,
+            logout_continuation,
+        }
+    }
+
+    pub(crate) fn into_parts(self) -> (VerifiedFederatedIdentity, Option<String>, OpaqueLogoutContinuation) {
+        (self.identity, self.redirect_after, self.logout_continuation)
+    }
+}
 
 #[async_trait::async_trait]
-pub trait FederatedIdentityProvider: Send + Sync {
-    fn has_providers(&self) -> bool;
-
-    fn list_providers(&self) -> Vec<OidcProviderSummary>;
-
-    fn list_visible_providers(&self) -> Vec<OidcProviderSummary>;
-
-    fn provider_config(&self, id: &str) -> Option<&OidcProviderConfig>;
-
+pub(crate) trait StandardOidcAuthentication: Send + Sync {
     async fn authorize_url(&self, provider_id: &str, redirect_uri: &str, redirect_after: Option<String>) -> Result<String>;
-
-    async fn exchange_code(&self, state: &str, code: &str, redirect_uri: &str) -> Result<FederatedCodeExchange>;
-
-    async fn verify_web_identity_token(&self, jwt: &str) -> Result<FederatedAuthorization>;
-
-    async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String>;
-
+    async fn exchange_identity(&self, state: &str, code: &str, redirect_uri: &str) -> Result<VerifiedFederatedCodeExchange>;
+    async fn verify_identity(&self, jwt: &str) -> Result<VerifiedFederatedIdentity>;
+    async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> Result<String>;
     async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>>;
 }
+
+pub(crate) trait FederatedProviderQuery: Send + Sync {
+    fn has_providers(&self) -> bool;
+
+    fn list_providers(&self) -> Vec<FederatedProviderView>;
+
+    fn list_visible_providers(&self) -> Vec<FederatedProviderView>;
+
+    fn redirect_policy(&self, provider_id: &str) -> Option<FederatedRedirectPolicy>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn provider_view_preserves_login_json_shape() {
+        let view = FederatedProviderView {
+            provider_id: "default".to_string(),
+            display_name: "Company SSO".to_string(),
+        };
+
+        assert_eq!(
+            serde_json::to_string(&[view]).expect("provider view should serialize"),
+            r#"[{"provider_id":"default","display_name":"Company SSO"}]"#
+        );
+    }
+}

crates/iam/src/federation/registry.rs

@@ -1,31 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::FederatedIdentityProvider;
-use std::sync::Arc;
-
-/// Immutable registry of built-in federation adapters.
-pub struct FederatedIdentityRegistry {
-    standard_oidc: Arc<dyn FederatedIdentityProvider>,
-}
-
-impl FederatedIdentityRegistry {
-    pub fn new(standard_oidc: Arc<dyn FederatedIdentityProvider>) -> Self {
-        Self { standard_oidc }
-    }
-
-    pub(crate) fn standard_oidc(&self) -> &dyn FederatedIdentityProvider {
-        self.standard_oidc.as_ref()
-    }
-}

crates/iam/src/federation/transaction.rs

@@ -13,14 +13,15 @@
 // limitations under the License.
 
 use super::{
-    FederatedIdentityRegistry, FederatedLoginSession, FederatedSession, FederatedSessionBinding, FederatedSessionTransaction,
-    FederationError, Result,
+    CoreFederatedAuthorizationMapper, FederatedAuthorization, FederatedLoginSession, FederatedProviderQuery,
+    FederatedProviderView, FederatedRedirectPolicy, FederatedSession, FederatedSessionBinding, FederatedSessionTransaction,
+    FederationError, OpaqueLogoutContinuation, Result, StandardOidcAuthentication,
 };
-use crate::oidc::{OidcProviderConfig, OidcProviderSummary};
+use std::sync::Arc;
 
 const DEFAULT_OIDC_PROVIDER_ID: &str = "default";
 
-fn sorted_provider_summaries(mut providers: Vec<OidcProviderSummary>) -> Vec<OidcProviderSummary> {
+fn sorted_provider_views(mut providers: Vec<FederatedProviderView>) -> Vec<FederatedProviderView> {
     providers.sort_by(|left, right| {
         (left.provider_id != DEFAULT_OIDC_PROVIDER_ID)
             .cmp(&(right.provider_id != DEFAULT_OIDC_PROVIDER_ID))
@@ -30,33 +31,68 @@ fn sorted_provider_summaries(mut providers: Vec<OidcProviderSummary>) -> Vec<Oid
 }
 
 pub struct FederatedIdentityService {
-    registry: FederatedIdentityRegistry,
+    provider_query: Arc<dyn FederatedProviderQuery>,
+    authentication: Arc<dyn StandardOidcAuthentication>,
+    mapper: CoreFederatedAuthorizationMapper,
+}
+
+struct RuntimeCodeExchange {
+    authorization: FederatedAuthorization,
+    redirect_after: Option<String>,
+    logout_continuation: OpaqueLogoutContinuation,
 }
 
 impl FederatedIdentityService {
-    pub fn new(registry: FederatedIdentityRegistry) -> Self {
-        Self { registry }
+    async fn exchange_code(&self, state: &str, code: &str, redirect_uri: &str) -> Result<RuntimeCodeExchange> {
+        let exchange = self.authentication.exchange_identity(state, code, redirect_uri).await?;
+        let (identity, redirect_after, logout_continuation) = exchange.into_parts();
+        Ok(RuntimeCodeExchange {
+            authorization: self.mapper.map(identity),
+            redirect_after,
+            logout_continuation,
+        })
+    }
+
+    async fn verify_identity(&self, jwt: &str) -> Result<FederatedAuthorization> {
+        Ok(self.mapper.map(self.authentication.verify_identity(jwt).await?))
+    }
+
+    async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> Result<String> {
+        self.authentication.create_logout_token(continuation).await
+    }
+
+    pub(super) fn from_standard_oidc_parts(
+        provider_query: Arc<dyn FederatedProviderQuery>,
+        authentication: Arc<dyn StandardOidcAuthentication>,
+        mapper: CoreFederatedAuthorizationMapper,
+    ) -> Self {
+        Self {
+            provider_query,
+            authentication,
+            mapper,
+        }
     }
 
     pub fn has_providers(&self) -> bool {
-        self.registry.standard_oidc().has_providers()
+        self.provider_query.has_providers()
     }
 
-    pub fn list_providers(&self) -> Vec<OidcProviderSummary> {
-        sorted_provider_summaries(self.registry.standard_oidc().list_providers())
+    pub fn list_providers(&self) -> Vec<FederatedProviderView> {
+        let providers = self.provider_query.list_providers();
+        sorted_provider_views(providers)
     }
 
-    pub fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
-        sorted_provider_summaries(self.registry.standard_oidc().list_visible_providers())
+    pub fn list_visible_providers(&self) -> Vec<FederatedProviderView> {
+        let providers = self.provider_query.list_visible_providers();
+        sorted_provider_views(providers)
     }
 
-    pub fn get_provider_config(&self, id: &str) -> Option<&OidcProviderConfig> {
-        self.registry.standard_oidc().provider_config(id)
+    pub fn redirect_policy(&self, provider_id: &str) -> Option<FederatedRedirectPolicy> {
+        self.provider_query.redirect_policy(provider_id)
     }
 
     pub async fn authorize_url(&self, provider_id: &str, redirect_uri: &str, redirect_after: Option<String>) -> Result<String> {
-        self.registry
-            .standard_oidc()
+        self.authentication
             .authorize_url(provider_id, redirect_uri, redirect_after)
             .await
     }
@@ -69,20 +105,14 @@ impl FederatedIdentityService {
         duration_seconds: usize,
         binding: &dyn FederatedSessionBinding,
     ) -> Result<FederatedLoginSession> {
-        let exchange = self.registry.standard_oidc().exchange_code(state, code, redirect_uri).await?;
-        let provider_id = exchange.authorization.provider_id.clone();
+        let exchange = self.exchange_code(state, code, redirect_uri).await?;
         let transaction = FederatedSessionTransaction {
             authorization: exchange.authorization,
             duration_seconds,
             session_policy: None,
         };
         let credentials = binding.bind(&transaction).await?;
-        let logout_token = self
-            .registry
-            .standard_oidc()
-            .create_logout_token(&provider_id, &exchange.id_token)
-            .await?;
-
+        let logout_token = self.create_logout_token(exchange.logout_continuation).await?;
         Ok(FederatedLoginSession {
             session: FederatedSession {
                 credentials,
@@ -100,7 +130,7 @@ impl FederatedIdentityService {
         session_policy: Option<String>,
         binding: &dyn FederatedSessionBinding,
     ) -> Result<FederatedSession> {
-        let authorization = self.registry.standard_oidc().verify_web_identity_token(jwt).await?;
+        let authorization = self.verify_identity(jwt).await?;
         if !authorization.has_authorization_context() {
             tracing::warn!(
                 provider_id = %authorization.provider_id,
@@ -136,8 +166,7 @@ impl FederatedIdentityService {
     }
 
     pub async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>> {
-        self.registry
-            .standard_oidc()
+        self.authentication
             .build_logout_url(logout_token, post_logout_redirect_uri)
             .await
     }
@@ -147,9 +176,9 @@ impl FederatedIdentityService {
 mod tests {
     use super::*;
     use crate::federation::{
-        FederatedAuthorization, FederatedClaims, FederatedCodeExchange, FederatedIdentityProvider, FederatedSessionBindingError,
+        FederatedAuthorizationRule, FederatedAuthorizationRules, FederatedClaims, FederatedProviderQuery, FederatedProviderRef,
+        FederatedSessionBindingError, VerifiedFederatedCodeExchange, VerifiedFederatedIdentity,
     };
-    use crate::oidc::{OidcProviderConfig, OidcProviderSummary};
     use rustfs_credentials::Credentials;
     use std::sync::{Arc, Mutex};
 
@@ -168,6 +197,8 @@ mod tests {
         web_provider_id: &'static str,
         failure: ProviderFailure,
         events: Arc<Mutex<Vec<&'static str>>>,
+        authorize_calls: Mutex<Vec<(String, String, Option<String>)>>,
+        logout_url_calls: Mutex<Vec<(String, String)>>,
         expected_logout: (&'static str, &'static str),
         listed_provider_ids: Vec<&'static str>,
         visible_provider_ids: Vec<&'static str>,
@@ -182,6 +213,8 @@ mod tests {
                 web_provider_id: DEFAULT_OIDC_PROVIDER_ID,
                 failure: ProviderFailure::None,
                 events,
+                authorize_calls: Mutex::new(Vec::new()),
+                logout_url_calls: Mutex::new(Vec::new()),
                 expected_logout: (DEFAULT_OIDC_PROVIDER_ID, "id-token"),
                 listed_provider_ids: Vec::new(),
                 visible_provider_ids: Vec::new(),
@@ -192,108 +225,139 @@ mod tests {
             self.events.lock().expect("event log should not be poisoned").push(event);
         }
 
-        fn authorization(&self, provider_id: &str) -> FederatedAuthorization {
-            FederatedAuthorization {
-                provider_id: provider_id.to_string(),
-                claims: FederatedClaims {
+        fn identity(&self, provider_id: &str) -> VerifiedFederatedIdentity {
+            VerifiedFederatedIdentity::from_claims(
+                FederatedProviderRef::new(provider_id.to_string()),
+                FederatedClaims {
                     sub: "subject".to_string(),
                     email: String::new(),
                     username: "user".to_string(),
-                    groups: vec!["source-group".to_string()],
+                    groups: if self.with_group {
+                        vec!["developers".to_string()]
+                    } else {
+                        Vec::new()
+                    },
                     raw: Default::default(),
                 },
-                policies: if self.with_policy {
-                    vec!["readwrite".to_string()]
-                } else {
-                    Vec::new()
-                },
-                group_claim_policies: Vec::new(),
-                groups: if self.with_group {
-                    vec!["developers".to_string()]
-                } else {
-                    Vec::new()
-                },
-                roles_claim_key: None,
-                roles: Vec::new(),
-            }
+            )
         }
     }
 
-    fn provider_summaries(provider_ids: &[&str]) -> Vec<OidcProviderSummary> {
+    fn provider_views(provider_ids: &[&str]) -> Vec<FederatedProviderView> {
         provider_ids
             .iter()
-            .map(|provider_id| OidcProviderSummary {
+            .map(|provider_id| FederatedProviderView {
                 provider_id: (*provider_id).to_string(),
                 display_name: (*provider_id).to_string(),
             })
             .collect()
     }
 
-    #[async_trait::async_trait]
-    impl FederatedIdentityProvider for TestProvider {
+    impl FederatedProviderQuery for TestProvider {
         fn has_providers(&self) -> bool {
             true
         }
 
-        fn list_providers(&self) -> Vec<OidcProviderSummary> {
-            provider_summaries(&self.listed_provider_ids)
+        fn list_providers(&self) -> Vec<FederatedProviderView> {
+            provider_views(&self.listed_provider_ids)
         }
 
-        fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
-            provider_summaries(&self.visible_provider_ids)
+        fn list_visible_providers(&self) -> Vec<FederatedProviderView> {
+            provider_views(&self.visible_provider_ids)
         }
 
-        fn provider_config(&self, _id: &str) -> Option<&OidcProviderConfig> {
-            None
+        fn redirect_policy(&self, provider_id: &str) -> Option<FederatedRedirectPolicy> {
+            (provider_id == DEFAULT_OIDC_PROVIDER_ID).then_some(FederatedRedirectPolicy {
+                redirect_uri: None,
+                allow_request_origin: true,
+            })
         }
+    }
 
-        async fn authorize_url(
-            &self,
-            _provider_id: &str,
-            _redirect_uri: &str,
-            _redirect_after: Option<String>,
-        ) -> Result<String> {
+    #[async_trait::async_trait]
+    impl StandardOidcAuthentication for TestProvider {
+        async fn authorize_url(&self, provider_id: &str, redirect_uri: &str, redirect_after: Option<String>) -> Result<String> {
+            self.authorize_calls
+                .lock()
+                .expect("authorize call log should not be poisoned")
+                .push((provider_id.to_string(), redirect_uri.to_string(), redirect_after));
             Ok("https://identity.example/authorize".to_string())
         }
 
-        async fn exchange_code(&self, _state: &str, _code: &str, _redirect_uri: &str) -> Result<FederatedCodeExchange> {
+        async fn exchange_identity(
+            &self,
+            _state: &str,
+            _code: &str,
+            _redirect_uri: &str,
+        ) -> Result<VerifiedFederatedCodeExchange> {
             self.record("exchange");
             if self.failure == ProviderFailure::Exchange {
                 return Err(FederationError::CodeExchange("exchange failed".to_string()));
             }
-            Ok(FederatedCodeExchange {
-                authorization: self.authorization(self.browser_provider_id),
-                redirect_after: Some("/browser".to_string()),
-                id_token: "id-token".to_string(),
-            })
+            let provider = FederatedProviderRef::new(self.browser_provider_id.to_string());
+            Ok(VerifiedFederatedCodeExchange::new(
+                self.identity(self.browser_provider_id),
+                Some("/browser".to_string()),
+                OpaqueLogoutContinuation::new(provider, "id-token".to_string()),
+            ))
         }
 
-        async fn verify_web_identity_token(&self, _jwt: &str) -> Result<FederatedAuthorization> {
+        async fn verify_identity(&self, _jwt: &str) -> Result<VerifiedFederatedIdentity> {
             self.record("verify");
             if self.failure == ProviderFailure::Verification {
                 return Err(FederationError::TokenVerification("verification failed".to_string()));
             }
-            Ok(self.authorization(self.web_provider_id))
+            Ok(self.identity(self.web_provider_id))
         }
 
-        async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String> {
+        async fn create_logout_token(&self, continuation: OpaqueLogoutContinuation) -> Result<String> {
             self.record("logout");
-            assert_eq!((provider_id, id_token), self.expected_logout);
+            let (provider, id_token) = continuation.into_parts();
+            assert_eq!((provider.as_str(), id_token.as_str()), self.expected_logout);
             if self.failure == ProviderFailure::Logout {
                 return Err(FederationError::Logout("logout failed".to_string()));
             }
             Ok("logout-token".to_string())
         }
 
-        async fn build_logout_url(&self, _logout_token: &str, _post_logout_redirect_uri: &str) -> Result<Option<String>> {
+        async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>> {
+            self.logout_url_calls
+                .lock()
+                .expect("logout URL call log should not be poisoned")
+                .push((logout_token.to_string(), post_logout_redirect_uri.to_string()));
             Ok(Some("https://identity.example/logout".to_string()))
         }
     }
 
+    fn standard_service(provider: Arc<TestProvider>) -> FederatedIdentityService {
+        let mut provider_ids = vec![provider.browser_provider_id, provider.web_provider_id];
+        provider_ids.sort_unstable();
+        provider_ids.dedup();
+        let role_policy = if provider.with_policy { "readwrite" } else { "" };
+        let rules = FederatedAuthorizationRules::new(provider_ids.into_iter().map(|provider_id| {
+            FederatedAuthorizationRule::new(
+                provider_id.to_string(),
+                "groups".to_string(),
+                String::new(),
+                role_policy.to_string(),
+                "groups".to_string(),
+                String::new(),
+            )
+        }));
+        let authentication: Arc<dyn StandardOidcAuthentication> = provider.clone();
+        let provider_query: Arc<dyn FederatedProviderQuery> = provider;
+        FederatedIdentityService::from_standard_oidc_parts(
+            provider_query,
+            authentication,
+            CoreFederatedAuthorizationMapper::new(rules),
+        )
+    }
+
     struct RecordingBinding {
         fail: bool,
         events: Arc<Mutex<Vec<&'static str>>>,
         transactions: Mutex<Vec<(String, usize, Option<String>)>>,
+        mapped_policies: Mutex<Vec<Vec<String>>>,
     }
 
     impl RecordingBinding {
@@ -302,6 +366,7 @@ mod tests {
                 fail: false,
                 events,
                 transactions: Mutex::new(Vec::new()),
+                mapped_policies: Mutex::new(Vec::new()),
             }
         }
     }
@@ -318,6 +383,10 @@ mod tests {
                 transaction.duration_seconds,
                 transaction.session_policy.clone(),
             ));
+            self.mapped_policies
+                .lock()
+                .expect("mapped policies should not be poisoned")
+                .push(transaction.authorization.policies.clone());
             if self.fail {
                 return Err(FederatedSessionBindingError::Internal("binding failed".to_string()));
             }
@@ -334,7 +403,7 @@ mod tests {
         let mut provider = TestProvider::new(events);
         provider.listed_provider_ids = vec!["zeta", "hidden", DEFAULT_OIDC_PROVIDER_ID, "alpha"];
         provider.visible_provider_ids = vec!["zeta", DEFAULT_OIDC_PROVIDER_ID, "alpha"];
-        let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
+        let service = standard_service(Arc::new(provider));
 
         assert_eq!(
             service
@@ -353,7 +422,7 @@ mod tests {
             [DEFAULT_OIDC_PROVIDER_ID, "alpha", "zeta"]
         );
         assert_eq!(
-            sorted_provider_summaries(provider_summaries(&["zeta", "alpha"]))
+            sorted_provider_views(provider_views(&["zeta", "alpha"]))
                 .into_iter()
                 .map(|provider| provider.provider_id)
                 .collect::<Vec<_>>(),
@@ -361,6 +430,48 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn authorization_and_logout_urls_preserve_service_arguments() {
+        let provider = Arc::new(TestProvider::new(Arc::new(Mutex::new(Vec::new()))));
+        let service = standard_service(provider.clone());
+
+        let authorize_url = service
+            .authorize_url(
+                "corp",
+                "https://console.example.com/oauth/callback",
+                Some("/browser/path?tab=objects".to_string()),
+            )
+            .await
+            .expect("authorization URL should be returned");
+        let logout_url = service
+            .build_logout_url("opaque-logout-token", "https://console.example.com/signed-out")
+            .await
+            .expect("logout URL should be returned");
+
+        assert_eq!(authorize_url, "https://identity.example/authorize");
+        assert_eq!(logout_url.as_deref(), Some("https://identity.example/logout"));
+        assert_eq!(
+            provider
+                .authorize_calls
+                .lock()
+                .expect("authorize call log should not be poisoned")
+                .as_slice(),
+            [(
+                "corp".to_string(),
+                "https://console.example.com/oauth/callback".to_string(),
+                Some("/browser/path?tab=objects".to_string()),
+            )]
+        );
+        assert_eq!(
+            provider
+                .logout_url_calls
+                .lock()
+                .expect("logout URL call log should not be poisoned")
+                .as_slice(),
+            [("opaque-logout-token".to_string(), "https://console.example.com/signed-out".to_string(),)]
+        );
+    }
+
     #[tokio::test]
     async fn callback_and_web_identity_preserve_provider_and_transaction_boundaries() {
         let events = Arc::new(Mutex::new(Vec::new()));
@@ -368,9 +479,8 @@ mod tests {
         provider.browser_provider_id = "corp";
         provider.web_provider_id = "partner";
         provider.expected_logout = ("corp", "id-token");
-        let provider = Arc::new(provider);
         let binding = Arc::new(RecordingBinding::new(events.clone()));
-        let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
+        let service = standard_service(Arc::new(provider));
 
         let login = service
             .complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref())
@@ -406,14 +516,52 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn standard_runtime_maps_before_common_browser_and_web_identity_flows() {
+        let events = Arc::new(Mutex::new(Vec::new()));
+        let mut provider = TestProvider::new(events.clone());
+        provider.browser_provider_id = "corp";
+        provider.web_provider_id = "partner";
+        provider.expected_logout = ("corp", "id-token");
+        let binding = Arc::new(RecordingBinding::new(events.clone()));
+        let service = standard_service(Arc::new(provider));
+
+        let login = service
+            .complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref())
+            .await
+            .expect("standard callback flow should complete");
+        assert_eq!(login.session.authorization.provider_id, "corp");
+        assert_eq!(login.session.authorization.policies, ["readwrite"]);
+        assert_eq!(
+            events.lock().expect("event log should not be poisoned").as_slice(),
+            ["exchange", "bind", "logout"]
+        );
+        events.lock().expect("event log should not be poisoned").clear();
+
+        let web_identity = service
+            .assume_role_with_web_identity("jwt", 7200, Some("session-policy".to_string()), binding.as_ref())
+            .await
+            .expect("standard web identity flow should complete");
+        assert_eq!(web_identity.authorization.provider_id, "partner");
+        assert_eq!(web_identity.authorization.policies, ["readwrite"]);
+        assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
+        assert_eq!(
+            binding
+                .mapped_policies
+                .lock()
+                .expect("mapped policies should not be poisoned")
+                .as_slice(),
+            [vec!["readwrite".to_string()], vec!["readwrite".to_string()]]
+        );
+    }
+
     #[tokio::test]
     async fn web_identity_without_policy_or_group_is_not_bound() {
         let events = Arc::new(Mutex::new(Vec::new()));
         let mut provider = TestProvider::new(events.clone());
         provider.with_policy = false;
-        let provider = Arc::new(provider);
         let binding = Arc::new(RecordingBinding::new(events.clone()));
-        let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
+        let service = standard_service(Arc::new(provider));
 
         let error = service
             .assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())
@@ -430,16 +578,15 @@ mod tests {
         let mut provider = TestProvider::new(events.clone());
         provider.with_policy = false;
         provider.with_group = true;
-        let provider = Arc::new(provider);
         let binding = Arc::new(RecordingBinding::new(events.clone()));
-        let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
+        let service = standard_service(Arc::new(provider));
 
         let session = service
             .assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())
             .await
             .expect("a mapped group is an authorization context");
 
-        assert!(session.authorization.policies.is_empty());
+        assert_eq!(session.authorization.policies, ["developers"]);
         assert_eq!(session.authorization.groups, ["developers"]);
         assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
     }
@@ -457,7 +604,7 @@ mod tests {
             let mut binding = RecordingBinding::new(events.clone());
             binding.fail = binding_failure;
             let binding = Arc::new(binding);
-            let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
+            let service = standard_service(Arc::new(provider));
 
             let error = service
                 .complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref())
@@ -496,7 +643,7 @@ mod tests {
             let mut binding = RecordingBinding::new(events.clone());
             binding.fail = binding_failure;
             let binding = Arc::new(binding);
-            let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
+            let service = standard_service(Arc::new(provider));
 
             let error = service
                 .assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())

crates/iam/src/lib.rs

@@ -20,10 +20,11 @@ use store::object::ObjectStore;
 use sys::IamSys;
 use tracing::{debug, error, info, instrument, warn};
 
-const LOG_COMPONENT_IAM: &str = "iam";
+pub(crate) const LOG_COMPONENT_IAM: &str = "iam";
 const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
-const LOG_SUBSYSTEM_OIDC: &str = "oidc";
+pub(crate) const LOG_SUBSYSTEM_OIDC: &str = "oidc";
 const EVENT_IAM_STATE: &str = "iam_state";
+pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
 const EVENT_OIDC_STATE: &str = "oidc_state";
 
 pub mod cache;
@@ -332,7 +333,7 @@ async fn init_oidc_sys_with_extra_root_ca_provider_inner(
                     event = EVENT_OIDC_STATE,
                     component = LOG_COMPONENT_IAM,
                     subsystem = LOG_SUBSYSTEM_OIDC,
-                    provider_count = sys.list_providers().len(),
+                    provider_count = sys.provider_configs().count(),
                     state = "ready",
                     "OIDC runtime ready"
                 );

crates/iam/src/manager.rs

@@ -3345,27 +3345,23 @@ mod tests {
 
     #[test]
     fn test_extract_jwt_claims_basic() {
-        let user_identity = UserIdentity {
-            version: 1,
-            credentials: Credentials {
-                access_key: "test-access-key".to_string(),
-                secret_key: "test-secret-key".to_string(),
-                session_token: "invalid-token".to_string(), // Invalid token for testing error handling
-                expiration: None,
-                status: "enabled".to_string(),
-                parent_user: "".to_string(),
-                groups: None,
-                claims: Some({
-                    let mut claims = HashMap::new();
-                    claims.insert("sub".to_string(), json!("test-user"));
-                    claims.insert("aud".to_string(), json!("test-audience"));
-                    claims
-                }),
-                name: None,
-                description: None,
-            },
-            update_at: Some(OffsetDateTime::now_utc()),
-        };
+        let user_identity = UserIdentity::new(Credentials {
+            access_key: "test-access-key".to_string(),
+            secret_key: "test-secret-key".to_string(),
+            session_token: "invalid-token".to_string(), // Invalid token for testing error handling
+            expiration: None,
+            status: "enabled".to_string(),
+            parent_user: "".to_string(),
+            groups: None,
+            claims: Some({
+                let mut claims = HashMap::new();
+                claims.insert("sub".to_string(), json!("test-user"));
+                claims.insert("aud".to_string(), json!("test-audience"));
+                claims
+            }),
+            name: None,
+            description: None,
+        });
 
         let result = extract_jwt_claims(&user_identity);
         // In test environment without proper JWT setup, this should fail
@@ -3374,22 +3370,18 @@ mod tests {
 
     #[test]
     fn test_extract_jwt_claims_no_claims() {
-        let user_identity = UserIdentity {
-            version: 1,
-            credentials: Credentials {
-                access_key: "test-access-key".to_string(),
-                secret_key: "test-secret-key".to_string(),
-                session_token: "".to_string(), // Empty token
-                expiration: None,
-                status: "enabled".to_string(),
-                parent_user: "".to_string(),
-                groups: None,
-                claims: None,
-                name: None,
-                description: None,
-            },
-            update_at: Some(OffsetDateTime::now_utc()),
-        };
+        let user_identity = UserIdentity::new(Credentials {
+            access_key: "test-access-key".to_string(),
+            secret_key: "test-secret-key".to_string(),
+            session_token: "".to_string(), // Empty token
+            expiration: None,
+            status: "enabled".to_string(),
+            parent_user: "".to_string(),
+            groups: None,
+            claims: None,
+            name: None,
+            description: None,
+        });
 
         let result = extract_jwt_claims(&user_identity);
         // Should fail with empty session token
@@ -3448,11 +3440,7 @@ mod tests {
             description: None,
         };
 
-        let user_identity = UserIdentity {
-            version: 1,
-            credentials,
-            update_at: Some(OffsetDateTime::now_utc()),
-        };
+        let user_identity = UserIdentity::new(credentials);
 
         // Test basic structure
         assert_eq!(user_identity.version, 1);

crates/iam/src/oidc.rs

@@ -18,7 +18,11 @@
 //! `openidconnect` crate for standards-compliant discovery, token exchange,
 //! and ID token verification.
 
-use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
+use crate::{
+    EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC,
+    federation::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRuleRef},
+    oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
+};
 use openidconnect::core::{
     CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm,
 };
@@ -47,9 +51,6 @@ use tokio::time::sleep;
 use tracing::{debug, error, warn};
 use url::Url;
 
-const LOG_COMPONENT_IAM: &str = "iam";
-const LOG_SUBSYSTEM_OIDC: &str = "oidc";
-const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
 const EVENT_OIDC_HTTP: &str = "oidc_http";
 const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
@@ -682,7 +683,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
     }
 }
 
-// ---- Public types (unchanged API) ----
+// ---- Public types ----
 
 const REDACTED_SECRET: &str = "***redacted***";
 
@@ -753,6 +754,31 @@ pub struct SourcedOidcProviderConfig {
     pub source: OidcProviderConfigSource,
 }
 
+/// Immutable OIDC configuration entries returned by configuration queries.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct OidcConfigSnapshot {
+    providers: Vec<SourcedOidcProviderConfig>,
+}
+
+impl OidcConfigSnapshot {
+    pub fn new(mut providers: Vec<SourcedOidcProviderConfig>) -> Self {
+        providers.sort_by(|left, right| {
+            (left.config.id != "default")
+                .cmp(&(right.config.id != "default"))
+                .then_with(|| left.config.id.cmp(&right.config.id))
+        });
+        Self { providers }
+    }
+
+    pub fn providers(&self) -> &[SourcedOidcProviderConfig] {
+        &self.providers
+    }
+
+    pub fn into_providers(self) -> Vec<SourcedOidcProviderConfig> {
+        self.providers
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct OidcProviderValidationResult {
     pub issuer: String,
@@ -760,12 +786,8 @@ pub struct OidcProviderValidationResult {
     pub token_endpoint: Option<String>,
 }
 
-/// Summary info about a provider, returned to the console.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct OidcProviderSummary {
-    pub provider_id: String,
-    pub display_name: String,
-}
+/// Compatibility name for the provider summary returned by OIDC list APIs.
+pub type OidcProviderSummary = crate::federation::FederatedProviderView;
 
 /// Claims extracted from an OIDC ID token.
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -932,7 +954,7 @@ impl ProviderState {
 
 /// Global OIDC manager for all configured providers.
 pub struct OidcSys {
-    configs: HashMap<String, OidcProviderConfig>,
+    configs: HashMap<String, SourcedOidcProviderConfig>,
     provider_states: RwLock<HashMap<String, ProviderState>>,
     state_store: OidcStateStore,
     http_client: ReqwestHttpClient,
@@ -967,22 +989,22 @@ impl OidcSys {
 
     async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result<Self, String> {
         let server_config = crate::server_config::current_server_config();
-        let parsed_configs = load_effective_oidc_provider_configs(server_config.as_ref());
+        let parsed_configs = load_oidc_config_snapshot(server_config.as_ref());
         let mut configs = HashMap::new();
         let mut provider_states = HashMap::new();
 
-        for sourced_config in parsed_configs {
-            let config = sourced_config.config;
+        for sourced_config in parsed_configs.into_providers() {
+            let config = &sourced_config.config;
             if !config.enabled {
                 debug!(provider = %config.id, "OIDC provider disabled");
                 continue;
             }
 
-            match Self::discover_provider(&config, &http_client).await {
+            match Self::discover_provider(config, &http_client).await {
                 Ok(state) => {
                     debug!(provider = %config.id, "OIDC provider discovered");
                     provider_states.insert(config.id.clone(), state);
-                    configs.insert(config.id.clone(), config);
+                    configs.insert(config.id.clone(), sourced_config);
                 }
                 Err(e) => {
                     error!(
@@ -1026,29 +1048,35 @@ impl OidcSys {
         !self.configs.is_empty()
     }
 
-    /// List all providers (including hidden ones). Used by site-replication and admin config.
+    pub(crate) fn provider_configs(&self) -> impl Iterator<Item = &OidcProviderConfig> {
+        self.configs.values().map(|provider| &provider.config)
+    }
+
+    /// List all providers, including providers hidden from the login UI.
     pub fn list_providers(&self) -> Vec<OidcProviderSummary> {
-        self.configs
-            .values()
-            .map(|c| OidcProviderSummary {
-                provider_id: c.id.clone(),
-                display_name: c.display_name.clone(),
+        self.provider_configs()
+            .map(|config| OidcProviderSummary {
+                provider_id: config.id.clone(),
+                display_name: config.display_name.clone(),
             })
             .collect()
     }
 
-    /// List only visible providers (excludes those with `hide_from_ui = true`).
+    /// List providers visible in the login UI.
     pub fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
-        self.configs
-            .values()
-            .filter(|c| !c.hide_from_ui)
-            .map(|c| OidcProviderSummary {
-                provider_id: c.id.clone(),
-                display_name: c.display_name.clone(),
+        self.provider_configs()
+            .filter(|config| !config.hide_from_ui)
+            .map(|config| OidcProviderSummary {
+                provider_id: config.id.clone(),
+                display_name: config.display_name.clone(),
             })
             .collect()
     }
 
+    pub fn config_snapshot(&self) -> OidcConfigSnapshot {
+        OidcConfigSnapshot::new(self.configs.values().cloned().collect())
+    }
+
     /// Build the PKCE authorization URL for a provider, store state in the state store.
     pub async fn authorize_url(
         &self,
@@ -1057,8 +1085,7 @@ impl OidcSys {
         redirect_after: Option<String>,
     ) -> Result<String, String> {
         let config = self
-            .configs
-            .get(provider_id)
+            .get_provider_config(provider_id)
             .ok_or_else(|| format!("unknown OIDC provider: {provider_id}"))?;
         let state = self.ensure_provider_state(provider_id, config).await?;
 
@@ -1116,8 +1143,7 @@ impl OidcSys {
             .ok_or_else(|| "invalid or expired OIDC state".to_string())?;
 
         let config = self
-            .configs
-            .get(&session.provider_id)
+            .get_provider_config(&session.provider_id)
             .ok_or_else(|| format!("unknown provider: {}", session.provider_id))?;
         let provider_state = self.get_provider_state(&session.provider_id)?;
         let issuer = provider_state.metadata.issuer().to_string();
@@ -1451,8 +1477,7 @@ impl OidcSys {
             .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?;
 
         let config = self
-            .configs
-            .get(&session.provider_id)
+            .get_provider_config(&session.provider_id)
             .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?;
         let state = self.ensure_provider_state(&session.provider_id, config).await?;
         let Some(end_session_endpoint) = state.metadata.console()?.additional_metadata().end_session_endpoint.clone() else {
@@ -1478,88 +1503,10 @@ impl OidcSys {
 
     /// Map OIDC claims to rustfs policy names.
     pub fn map_claims_to_policies(&self, provider_id: &str, claims: &OidcClaims) -> (Vec<String>, Vec<String>) {
-        let config = match self.configs.get(provider_id) {
-            Some(c) => c,
-            None => return (vec![], vec![]),
+        let Some(rule) = self.authorization_rule(provider_id) else {
+            return (Vec::new(), Vec::new());
         };
-
-        let mut policies = Vec::new();
-        let mut groups = Vec::new();
-
-        // Role-policy and claim-based authorization are separate OIDC modes. When a
-        // role policy is configured, group claims still provide group context but
-        // must not also become policy names.
-        let has_role_policy = !config.role_policy.trim().is_empty();
-        if has_role_policy {
-            for policy in config.role_policy.split(',') {
-                let policy = policy.trim();
-                if !policy.is_empty() {
-                    policies.push(policy.to_string());
-                }
-            }
-        }
-
-        for group in &claims.groups {
-            groups.push(group.clone());
-            if !has_role_policy {
-                policies.push(claim_policy_name(&config.claim_prefix, group));
-            }
-        }
-
-        if !has_role_policy && config.claim_name != config.groups_claim {
-            for val in extract_groups_claim(&claims.raw, &config.claim_name) {
-                policies.push(claim_policy_name(&config.claim_prefix, &val));
-            }
-        }
-
-        // Deduplicate
-        policies.sort();
-        policies.dedup();
-        groups.sort();
-        groups.dedup();
-
-        let mut raw_claim_keys: Vec<&str> = claims.raw.keys().map(String::as_str).collect();
-        raw_claim_keys.sort_unstable();
-        let (claim_name_lookup, claim_name_raw_value) = claim_lookup_for_log(&claims.raw, &config.claim_name);
-        let (groups_claim_lookup, groups_claim_raw_value) = claim_lookup_for_log(&claims.raw, &config.groups_claim);
-        let (roles_claim_lookup, roles_claim_raw_value) = claim_lookup_for_log(&claims.raw, &config.roles_claim);
-        let claim_name_values = extract_groups_claim(&claims.raw, &config.claim_name);
-        let groups_claim_values = extract_groups_claim(&claims.raw, &config.groups_claim);
-        let roles_claim_values = extract_groups_claim(&claims.raw, &config.roles_claim);
-
-        debug!(
-            event = EVENT_OIDC_DIAGNOSTICS,
-            component = LOG_COMPONENT_IAM,
-            subsystem = LOG_SUBSYSTEM_OIDC,
-            result = "claims_policy_mapped",
-            provider_id = %provider_id,
-            claim_name = %config.claim_name,
-            claim_prefix = %config.claim_prefix,
-            groups_claim = %config.groups_claim,
-            roles_claim = %config.roles_claim,
-            role_policy = %config.role_policy,
-            policy_count = policies.len(),
-            group_count = groups.len(),
-            policies = ?policies,
-            groups = ?groups,
-            raw_claim_keys = ?raw_claim_keys,
-            raw_claims = ?claims.raw,
-            claim_name_lookup = %claim_name_lookup,
-            claim_name_type = claim_value_type_for_log(claim_name_raw_value),
-            claim_name_value = ?claim_name_raw_value,
-            claim_name_values = ?claim_name_values,
-            groups_claim_lookup = %groups_claim_lookup,
-            groups_claim_type = claim_value_type_for_log(groups_claim_raw_value),
-            groups_claim_value = ?groups_claim_raw_value,
-            groups_claim_values = ?groups_claim_values,
-            roles_claim_lookup = %roles_claim_lookup,
-            roles_claim_type = claim_value_type_for_log(roles_claim_raw_value),
-            roles_claim_value = ?roles_claim_raw_value,
-            roles_claim_values = ?roles_claim_values,
-            "oidc claims mapped to policies"
-        );
-
-        (policies, groups)
+        CoreFederatedAuthorizationMapper::map_policies_and_groups(provider_id, rule, &claims.groups, &claims.raw, true)
     }
 
     /// Policy names produced only by the canonical groups in claim-based mode: the groups claim
@@ -1571,30 +1518,22 @@ impl OidcSys {
     /// dedicated policy claim are never included, so those configurations keep requiring every
     /// policy to resolve.
     pub fn group_claim_policy_names(&self, provider_id: &str, claims: &OidcClaims) -> Vec<String> {
-        let Some(config) = self.configs.get(provider_id) else {
+        let Some(rule) = self.authorization_rule(provider_id) else {
             return Vec::new();
         };
-        if !config.role_policy.trim().is_empty() {
-            return Vec::new();
-        }
+        CoreFederatedAuthorizationMapper::group_claim_policy_names(rule, &claims.groups, &claims.raw)
+    }
 
-        let explicit_policy_names: Vec<String> = if config.claim_name != config.groups_claim {
-            extract_groups_claim(&claims.raw, &config.claim_name)
-                .iter()
-                .map(|value| claim_policy_name(&config.claim_prefix, value))
-                .collect()
-        } else {
-            Vec::new()
-        };
-        let mut policies: Vec<String> = claims
-            .groups
-            .iter()
-            .map(|group| claim_policy_name(&config.claim_prefix, group))
-            .filter(|policy| !explicit_policy_names.contains(policy))
-            .collect();
-        policies.sort();
-        policies.dedup();
-        policies
+    fn authorization_rule(&self, provider_id: &str) -> Option<FederatedAuthorizationRuleRef<'_>> {
+        self.get_provider_config(provider_id).map(|config| {
+            FederatedAuthorizationRuleRef::new(
+                &config.claim_name,
+                &config.claim_prefix,
+                &config.role_policy,
+                &config.groups_claim,
+                &config.roles_claim,
+            )
+        })
     }
 
     /// Verify a raw JWT (id_token) for the AssumeRoleWithWebIdentity flow.
@@ -1678,7 +1617,7 @@ impl OidcSys {
                 && issuer_host == provider_host
                 && issuer_port == provider_port
                 && issuer_path == provider_path
-                && let Some(config) = self.configs.get(id)
+                && let Some(config) = self.get_provider_config(id)
             {
                 return Some((id.clone(), config.clone(), state.clone()));
             }
@@ -1747,7 +1686,7 @@ impl OidcSys {
 
     /// Get a provider config by ID.
     pub fn get_provider_config(&self, id: &str) -> Option<&OidcProviderConfig> {
-        self.configs.get(id)
+        self.configs.get(id).map(|provider| &provider.config)
     }
 
     /// Parse all OIDC provider configs from environment variables.
@@ -2225,6 +2164,10 @@ pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>
     merge_oidc_provider_configs(env_configs, persisted_configs)
 }
 
+pub fn load_oidc_config_snapshot(server_config: Option<&ServerConfig>) -> OidcConfigSnapshot {
+    OidcConfigSnapshot::new(load_effective_oidc_provider_configs(server_config))
+}
+
 pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
     validate_oidc_provider_config_with_extra_root_ca(config, None).await
 }
@@ -2387,10 +2330,6 @@ fn extract_string_claim(claims: &HashMap<String, serde_json::Value>, key: &str)
     }
 }
 
-fn claim_policy_name(claim_prefix: &str, value: &str) -> String {
-    format!("{claim_prefix}{value}")
-}
-
 /// Extract a groups/array claim from raw claims with case-insensitive fallback. Handles both string arrays and single strings.
 fn extract_groups_claim(claims: &HashMap<String, serde_json::Value>, key: &str) -> Vec<String> {
     match get_claim_case_insensitive(claims, key) {
@@ -2415,32 +2354,14 @@ fn extract_canonical_group_values(
     groups
 }
 
-fn claim_lookup_for_log<'a>(
-    claims: &'a HashMap<String, serde_json::Value>,
-    key: &str,
-) -> (&'static str, Option<&'a serde_json::Value>) {
-    match get_claim_case_insensitive(claims, key) {
-        ClaimLookup::Found(value) => ("found", Some(value)),
-        ClaimLookup::Missing => ("missing", None),
-        ClaimLookup::Ambiguous => ("ambiguous", None),
-    }
-}
-
-fn claim_value_type_for_log(value: Option<&serde_json::Value>) -> &'static str {
-    match value {
-        Some(serde_json::Value::Null) => "null",
-        Some(serde_json::Value::Bool(_)) => "bool",
-        Some(serde_json::Value::Number(_)) => "number",
-        Some(serde_json::Value::String(_)) => "string",
-        Some(serde_json::Value::Array(_)) => "array",
-        Some(serde_json::Value::Object(_)) => "object",
-        None => "none",
-    }
-}
-
-#[cfg(test)]
-pub(crate) fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
-    let configs = configs.into_iter().map(|config| (config.id.clone(), config)).collect();
+#[cfg(any(test, feature = "test-util"))]
+#[doc(hidden)]
+pub fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
+    let configs = configs
+        .into_iter()
+        .map(test_sourced_config)
+        .map(|provider| (provider.config.id.clone(), provider))
+        .collect();
     OidcSys {
         configs,
         provider_states: RwLock::new(HashMap::new()),
@@ -2449,8 +2370,17 @@ pub(crate) fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
     }
 }
 
-#[cfg(test)]
-pub(crate) fn test_config(id: &str) -> OidcProviderConfig {
+#[cfg(any(test, feature = "test-util"))]
+fn test_sourced_config(config: OidcProviderConfig) -> SourcedOidcProviderConfig {
+    SourcedOidcProviderConfig {
+        config,
+        source: OidcProviderConfigSource::Persisted,
+    }
+}
+
+#[cfg(any(test, feature = "test-util"))]
+#[doc(hidden)]
+pub fn test_config(id: &str) -> OidcProviderConfig {
     OidcProviderConfig {
         id: id.to_string(),
         enabled: true,
@@ -3041,7 +2971,7 @@ mod tests {
             .await
             .expect("initial OIDC discovery should succeed");
         let sys = OidcSys {
-            configs: HashMap::from([(config.id.clone(), config.clone())]),
+            configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]),
             provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])),
             state_store: OidcStateStore::new(),
             http_client,
@@ -3175,7 +3105,7 @@ mod tests {
                 let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).unwrap());
                 let discovered = OidcSys::discover_provider(&config, &http_client).await.unwrap();
                 let sys = OidcSys {
-                    configs: HashMap::from([(config.id.clone(), config)]),
+                    configs: HashMap::from([(config.id.clone(), test_sourced_config(config))]),
                     provider_states: RwLock::new(HashMap::from([("console".into(), discovered)])),
                     state_store: OidcStateStore::new(),
                     http_client,
@@ -3403,13 +3333,13 @@ mod tests {
                 .expect("workload discovery must succeed");
             assert!(state.metadata.authorization_endpoint().is_none());
             let sys = OidcSys {
-                configs: HashMap::from([(config.id.clone(), config.clone())]),
+                configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]),
                 provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])),
                 state_store: OidcStateStore::new(),
                 http_client,
             };
             assert!(sys.has_providers());
-            assert!(sys.list_visible_providers().is_empty());
+            assert!(sys.provider_configs().all(|provider| provider.hide_from_ui));
             let error = sys
                 .authorize_url(&config.id, "https://console.example.com/callback", None)
                 .await
@@ -4033,7 +3963,7 @@ mod tests {
     fn test_oidc_sys_empty() {
         let sys = OidcSys::empty().expect("failed to initialize empty OIDC system");
         assert!(!sys.has_providers());
-        assert!(sys.list_providers().is_empty());
+        assert!(sys.config_snapshot().providers().is_empty());
     }
 
     #[test]
@@ -4183,7 +4113,7 @@ mod tests {
             )
             .await;
         let sys = OidcSys {
-            configs: HashMap::from([(provider_id.to_string(), config)]),
+            configs: HashMap::from([(provider_id.to_string(), test_sourced_config(config))]),
             provider_states: RwLock::new(HashMap::from([(
                 provider_id.to_string(),
                 ProviderState {
@@ -4432,8 +4362,8 @@ mod tests {
         let listed = sys.list_visible_providers();
 
         assert_eq!(listed.len(), 1);
-        assert!(listed.iter().any(|p| p.provider_id == "dex"));
-        assert!(!listed.iter().any(|p| p.provider_id == "kubernetes"));
+        assert!(listed.iter().any(|provider| provider.provider_id == "dex"));
+        assert!(!listed.iter().any(|provider| provider.provider_id == "kubernetes"));
     }
 
     #[test]
@@ -4449,44 +4379,28 @@ mod tests {
     }
 
     #[test]
-    fn test_list_providers_includes_hidden_for_replication() {
+    fn test_config_snapshot_includes_hidden_for_replication() {
+        let visible = test_config("dex");
+        let mut hidden = test_config("kubernetes");
+        hidden.hide_from_ui = true;
+
+        let sys = make_test_sys(vec![visible, hidden]);
+
+        assert_eq!(sys.config_snapshot().providers().len(), 2);
+    }
+
+    #[test]
+    fn test_list_providers_includes_hidden_for_compatibility() {
         let visible = test_config("dex");
         let mut hidden = test_config("kubernetes");
         hidden.hide_from_ui = true;
 
         let sys = make_test_sys(vec![visible, hidden]);
 
-        // Unfiltered list returns all (used by site-replication)
         assert_eq!(sys.list_providers().len(), 2);
-        // UI-filtered list hides the hidden one
         assert_eq!(sys.list_visible_providers().len(), 1);
     }
 
-    #[test]
-    fn test_list_providers_all_visible_by_default() {
-        let a = test_config("okta");
-        let b = test_config("dex");
-
-        let sys = make_test_sys(vec![a, b]);
-        let listed = sys.list_visible_providers();
-
-        assert_eq!(listed.len(), 2);
-    }
-
-    #[test]
-    fn test_list_visible_providers_all_hidden() {
-        let mut a = test_config("k8s-a");
-        a.hide_from_ui = true;
-        let mut b = test_config("k8s-b");
-        b.hide_from_ui = true;
-
-        let sys = make_test_sys(vec![a, b]);
-        let listed = sys.list_visible_providers();
-
-        assert!(listed.is_empty());
-        assert!(sys.has_providers());
-    }
-
     #[test]
     fn test_hide_from_ui_default_is_false() {
         let config = test_config("default");
@@ -4723,17 +4637,32 @@ mod tests {
     }
 
     #[test]
-    fn test_list_providers() {
+    fn test_config_snapshot_lists_providers() {
         let mut config = test_config("keycloak");
         config.display_name = "Keycloak SSO".to_string();
 
         let sys = make_test_sys(vec![config]);
 
         assert!(sys.has_providers());
-        let summaries = sys.list_providers();
-        assert_eq!(summaries.len(), 1);
-        assert_eq!(summaries[0].provider_id, "keycloak");
-        assert_eq!(summaries[0].display_name, "Keycloak SSO");
+        let snapshot = sys.config_snapshot();
+        assert_eq!(snapshot.providers().len(), 1);
+        assert_eq!(snapshot.providers()[0].config.id, "keycloak");
+        assert_eq!(snapshot.providers()[0].config.display_name, "Keycloak SSO");
+        assert_eq!(snapshot.providers()[0].source, OidcProviderConfigSource::Persisted);
+    }
+
+    #[test]
+    fn test_active_config_snapshot_puts_default_first() {
+        let sys = make_test_sys(vec![test_config("zeta"), test_config("default"), test_config("alpha")]);
+
+        assert_eq!(
+            sys.config_snapshot()
+                .providers()
+                .iter()
+                .map(|provider| provider.config.id.as_str())
+                .collect::<Vec<_>>(),
+            ["default", "alpha", "zeta"]
+        );
     }
 
     #[test]

crates/iam/src/storage_api.rs

@@ -91,6 +91,7 @@ pub(crate) mod crate_boundary {
 
 pub(crate) mod object_store {
     pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
+    pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
 }
 
 pub(crate) mod runtime {

crates/iam/src/store/object.rs

@@ -42,6 +42,7 @@ use tracing::{debug, error, warn};
 
 use crate::storage_api::object_store::{
     HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
+    read_config_preserve_empty_with_metadata_opts,
 };
 
 pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
@@ -414,9 +415,14 @@ impl ObjectStore {
         &self,
         path: impl AsRef<str> + Send,
         mode: LoadMode,
+        preserve_empty: bool,
     ) -> Result<(Vec<u8>, IamObjectInfo)> {
         let path_ref = path.as_ref();
-        let (data, obj) = read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?;
+        let (data, obj) = if preserve_empty {
+            read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path_ref, &mode.read_opts()).await?
+        } else {
+            read_iam_config_with_metadata(self.object_api.clone(), path_ref, &mode.read_opts()).await?
+        };
         let outcome = Self::decrypt_data_with_source(&data)?;
         self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
 
@@ -614,8 +620,9 @@ impl ObjectStore {
     /// Parameterized core of [`Store::load_user_identity`].
     async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
         let mut u: UserIdentity = self
-            .load_iam_config_with(get_user_identity_path(name, user_type), mode)
+            .load_iamconfig_bytes_with_metadata(get_user_identity_path(name, user_type), mode, true)
             .await
+            .and_then(|(data, _)| serde_json::from_slice(&data).map_err(Error::from))
             .map_err(|err| {
                 if is_err_config_not_found(&err) {
                     debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
@@ -695,7 +702,7 @@ impl ObjectStore {
     /// Parameterized core of [`Store::load_policy`].
     async fn load_policy_with(&self, name: &str, mode: LoadMode) -> Result<PolicyDoc> {
         let (data, obj) = self
-            .load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode)
+            .load_iamconfig_bytes_with_metadata(get_policy_doc_path(name), mode, false)
             .await
             .map_err(|err| {
                 if is_err_config_not_found(&err) {

crates/iam/src/sys.rs

@@ -2667,11 +2667,7 @@ mod tests {
                     name: None,
                     description: None,
                 };
-                let parent_identity = UserIdentity {
-                    version: 1,
-                    credentials: creds,
-                    update_at: Some(OffsetDateTime::now_utc()),
-                };
+                let parent_identity = UserIdentity::new(creds);
                 let mut users = HashMap::new();
                 users.insert(PARENT_USER.to_string(), parent_identity);
 
@@ -2703,11 +2699,7 @@ mod tests {
                 name: None,
                 description: None,
             };
-            let parent_identity = UserIdentity {
-                version: 1,
-                credentials: creds,
-                update_at: Some(OffsetDateTime::now_utc()),
-            };
+            let parent_identity = UserIdentity::new(creds);
             let mut users = HashMap::new();
             users.insert(PARENT_USER.to_string(), parent_identity);
 
@@ -2820,16 +2812,12 @@ mod tests {
     async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
         let iam_sys = test_iam_sys().await;
         let member = "group-stamp-member";
-        let identity = UserIdentity {
-            version: 1,
-            credentials: Credentials {
-                access_key: member.to_string(),
-                secret_key: "longenoughsecret".to_string(),
-                status: "on".to_string(),
-                ..Default::default()
-            },
-            update_at: Some(OffsetDateTime::now_utc()),
-        };
+        let identity = UserIdentity::new(Credentials {
+            access_key: member.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: "on".to_string(),
+            ..Default::default()
+        });
         iam_sys.store.cache.with_write_lock(|cache| {
             cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
             // The startup load publishes every entity with the load time.
@@ -2888,6 +2876,59 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn service_account_preserves_long_secret_keys_after_rotation_and_reload() {
+        ensure_test_global_credentials();
+        let iam_sys = test_iam_sys().await;
+
+        for length in [41, 90, 128, 256, 257, 4096] {
+            let access_key = format!("long-secret-{length}");
+            let secret_key = format!("{}a", "s".repeat(length - 1));
+            let (created, _) = iam_sys
+                .new_service_account("parent-user", None, service_account_opts(&access_key, &secret_key))
+                .await
+                .expect("create service account with a long secret");
+            assert_eq!(created.secret_key, secret_key);
+            get_claims_from_token_with_secret_allow_missing_exp(&created.session_token, &secret_key)
+                .expect("the complete secret verifies the created session token");
+
+            let rotated_secret = format!("{}b", "s".repeat(length - 1));
+            iam_sys
+                .update_service_account(
+                    &access_key,
+                    UpdateServiceAccountOpts {
+                        secret_key: Some(rotated_secret.clone()),
+                        session_policy: None,
+                        status: None,
+                        name: None,
+                        description: None,
+                        expiration: None,
+                        parent_user: None,
+                        allow_site_replicator_account: false,
+                    },
+                )
+                .await
+                .expect("rotate the complete long secret");
+
+            let persisted = iam_sys
+                .store
+                .api
+                .load_user_identity(&access_key, UserType::Svc)
+                .await
+                .expect("load the persisted service account identity");
+            assert_eq!(persisted.credentials.secret_key, rotated_secret);
+            get_claims_from_token_with_secret_allow_missing_exp(&persisted.credentials.session_token, &rotated_secret)
+                .expect("the rotated secret verifies the persisted session token");
+            assert!(
+                get_claims_from_token_with_secret_allow_missing_exp(&persisted.credentials.session_token, &secret_key).is_err()
+            );
+            assert!(
+                get_claims_from_token_with_secret_allow_missing_exp(&persisted.credentials.session_token, &rotated_secret[..40])
+                    .is_err()
+            );
+        }
+    }
+
     #[tokio::test]
     async fn new_service_account_rejects_parent_access_key() {
         ensure_test_global_credentials();

crates/iam/tests/ecstore_test_compat/mod.rs

@@ -22,7 +22,11 @@
 #[allow(unused_imports)]
 pub(crate) mod fixture {
     pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
+    pub(crate) use rustfs_ecstore::api::config::com::save_config;
+    pub(crate) use rustfs_ecstore::api::global::set_object_store_resolver;
     pub(crate) use rustfs_ecstore::api::layout::SetupType;
+    pub(crate) use rustfs_ecstore::api::runtime::{InstanceContext, first_cluster_node_is_local, object_store_handle};
+    pub(crate) use rustfs_ecstore::api::storage::ECStore;
 
     // `update_erasure_type` is a write-side global facade entry. Its use is
     // restricted to reviewed storage_api boundaries; this test-only module is

crates/iam/tests/iam_bootstrap_no_lock_test.rs

@@ -33,12 +33,21 @@
 
 mod ecstore_test_compat;
 
-use ecstore_test_compat::fixture::{SetupType, update_erasure_type};
+use ecstore_test_compat::fixture::{
+    ECStore, InstanceContext, SetupType, first_cluster_node_is_local, object_store_handle, save_config,
+    set_object_store_resolver, update_erasure_type,
+};
+use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
 use rustfs_iam::cache::Cache;
-use rustfs_iam::store::object::ObjectStore;
-use rustfs_iam::store::{GroupInfo, Store};
+use rustfs_iam::error::Error as IamError;
+use rustfs_iam::manager::{IamCache, IamState};
+use rustfs_iam::store::object::{IAM_CONFIG_POLICIES_PREFIX, IAM_CONFIG_USERS_PREFIX, ObjectStore};
+use rustfs_iam::store::{GroupInfo, Store, UserType};
+use rustfs_policy::auth::UserIdentity;
 use serial_test::serial;
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
+use std::sync::{Arc, Mutex, OnceLock, Weak};
 
 const TEST_GROUP: &str = "seq-restart-group";
 const TEST_MEMBERS: [&str; 2] = ["alice", "bob"];
@@ -69,7 +78,8 @@ async fn load_all_bypasses_namespace_lock_quorum() {
             .build()
             .await
             .ecstore;
-        let store = ObjectStore::new(ecstore);
+        let _route_guard = TestStoreRouteGuard::new(&ecstore);
+        let store = ObjectStore::new(ecstore.clone());
 
         // Seed IAM data while namespace locks still work (single-node mode).
         store
@@ -128,3 +138,447 @@ async fn load_all_bypasses_namespace_lock_quorum() {
     })
     .await;
 }
+
+const IDENTITY_PARENT: &str = "fixture-parent";
+const IDENTITY_CHILD: &str = "fixture-service-child";
+
+static TEST_STORE_ROUTE: Mutex<Option<Weak<ECStore>>> = Mutex::new(None);
+static TEST_STORE_RESOLVER: OnceLock<()> = OnceLock::new();
+
+struct TestStoreRouteGuard;
+
+impl TestStoreRouteGuard {
+    fn new(ecstore: &Arc<ECStore>) -> Self {
+        TEST_STORE_RESOLVER.get_or_init(|| {
+            assert!(
+                set_object_store_resolver(Arc::new(|| {
+                    TEST_STORE_ROUTE
+                        .lock()
+                        .expect("lock test store route")
+                        .as_ref()
+                        .and_then(Weak::upgrade)
+                })),
+                "test binary must own its resolver registration"
+            );
+        });
+        let mut route = TEST_STORE_ROUTE.lock().expect("lock test store route");
+        assert!(route.is_none(), "another test store route is active");
+        *route = Some(Arc::downgrade(ecstore));
+        Self
+    }
+}
+
+impl Drop for TestStoreRouteGuard {
+    fn drop(&mut self) {
+        *TEST_STORE_ROUTE.lock().expect("lock test store route for cleanup") = None;
+    }
+}
+
+struct IdentityStoreFixture {
+    previous_store: Option<Arc<ECStore>>,
+    ecstore: Arc<ECStore>,
+    manager: IamCache<ObjectStore>,
+    _route_guard: TestStoreRouteGuard,
+    _temp_dir: tempfile::TempDir,
+}
+
+impl IdentityStoreFixture {
+    async fn new() -> Self {
+        let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_identity_fixture_").expect("create identity fixture directory");
+        let ctx = Arc::new(InstanceContext::new());
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(temp_dir.path())
+            .init_bucket_metadata(false)
+            .instance_ctx(ctx.clone())
+            .build()
+            .await;
+        ctx.set_endpoints(env.endpoint_pools.clone());
+        let ecstore = env.ecstore;
+        let previous_store = object_store_handle();
+        let route_guard = TestStoreRouteGuard::new(&ecstore);
+        let selected = object_store_handle().expect("fixture route selects its ECStore");
+        assert!(Arc::ptr_eq(&selected, &ecstore), "fixture route must select the isolated ECStore");
+        drop(selected);
+        assert!(first_cluster_node_is_local().await, "isolated fixture has a local first endpoint");
+        temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], rustfs_iam::build_iam_sys(ecstore.clone()))
+            .await
+            .expect("initialize IAM metadata through production owner");
+        let store = ObjectStore::new(ecstore.clone());
+        let parent = UserIdentity::new(Credentials {
+            access_key: IDENTITY_PARENT.to_string(),
+            secret_key: "fixture-parent-secret".to_string(),
+            expiration: None,
+            session_token: String::new(),
+            status: "on".to_string(),
+            ..Default::default()
+        });
+        let child = UserIdentity::new(Credentials {
+            access_key: IDENTITY_CHILD.to_string(),
+            secret_key: "fixture-child-secret".to_string(),
+            expiration: None,
+            session_token: String::new(),
+            status: "on".to_string(),
+            parent_user: IDENTITY_PARENT.to_string(),
+            claims: Some(HashMap::from([(
+                IAM_POLICY_CLAIM_NAME_SA.to_string(),
+                serde_json::json!("inherited-policy"),
+            )])),
+            ..Default::default()
+        });
+        assert!(child.credentials.is_service_account(), "child fixture must be a service account");
+        store
+            .save_user_identity(IDENTITY_PARENT, UserType::Reg, parent, None)
+            .await
+            .expect("persist parent identity");
+        store
+            .save_user_identity(IDENTITY_CHILD, UserType::Svc, child, None)
+            .await
+            .expect("persist service child identity");
+        let manager = IamCache {
+            cache: Cache::default(),
+            api: store,
+            state: Arc::new(AtomicU8::new(IamState::Ready as u8)),
+            loading: Arc::new(AtomicBool::new(false)),
+            roles: HashMap::new(),
+            send_chan: tokio::sync::mpsc::channel::<i64>(1).0,
+            last_timestamp: AtomicI64::new(0),
+            sync_failures: AtomicU64::new(0),
+            sync_successes: AtomicU64::new(0),
+            last_sync_duration_millis: AtomicU64::new(0),
+        };
+        manager
+            .load_user(IDENTITY_PARENT)
+            .await
+            .expect("load persisted parent into cache");
+        manager
+            .load_user(IDENTITY_CHILD)
+            .await
+            .expect("load persisted service child into cache");
+        Self {
+            previous_store,
+            ecstore,
+            manager,
+            _route_guard: route_guard,
+            _temp_dir: temp_dir,
+        }
+    }
+
+    async fn assert_parent_and_child(&self) {
+        let parent = self
+            .manager
+            .api
+            .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+            .await
+            .expect("read parent from real storage");
+        assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
+        assert_eq!(parent.credentials.parent_user, "");
+        let child = self
+            .manager
+            .api
+            .load_user_identity(IDENTITY_CHILD, UserType::Svc)
+            .await
+            .expect("read child from real storage");
+        assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
+        assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
+        assert!(child.credentials.is_service_account());
+        let cached_parent = self.manager.get_user(IDENTITY_PARENT).await.expect("parent remains in cache");
+        assert_eq!(cached_parent.credentials.access_key, IDENTITY_PARENT);
+        assert_eq!(cached_parent.credentials.parent_user, "");
+        let cached_child = self
+            .manager
+            .get_user(IDENTITY_CHILD)
+            .await
+            .expect("service child remains in cache");
+        assert_eq!(cached_child.credentials.access_key, IDENTITY_CHILD);
+        assert_eq!(cached_child.credentials.parent_user, IDENTITY_PARENT);
+        assert!(cached_child.credentials.is_service_account());
+    }
+
+    async fn overwrite_parent_bytes(&self, bytes: &[u8]) {
+        let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
+        save_config(self.ecstore.clone(), &path, bytes.to_vec())
+            .await
+            .expect("overwrite real parent identity bytes");
+    }
+
+    async fn assert_child_storage_and_identity_cache(&self) {
+        let child = self
+            .manager
+            .api
+            .load_user_identity(IDENTITY_CHILD, UserType::Svc)
+            .await
+            .expect("notification must preserve persisted service child");
+        assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
+        assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
+        assert!(child.credentials.is_service_account());
+        let parent = self
+            .manager
+            .get_user(IDENTITY_PARENT)
+            .await
+            .expect("failed notification preserves cached parent");
+        assert_eq!(parent.credentials.access_key, IDENTITY_PARENT);
+        assert_eq!(parent.credentials.parent_user, "");
+        let child = self
+            .manager
+            .get_user(IDENTITY_CHILD)
+            .await
+            .expect("failed notification preserves cached child");
+        assert_eq!(child.credentials.access_key, IDENTITY_CHILD);
+        assert_eq!(child.credentials.parent_user, IDENTITY_PARENT);
+        assert!(child.credentials.is_service_account());
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_store_fixture_roundtrips_parent_and_service_child() {
+    let fixture = IdentityStoreFixture::new().await;
+    let before = fixture.previous_store.clone();
+    let selected = object_store_handle().expect("fixture route selects its ECStore");
+    assert!(Arc::ptr_eq(&selected, &fixture.ecstore));
+    drop(selected);
+    fixture.assert_parent_and_child().await;
+    drop(fixture);
+    assert!(TEST_STORE_ROUTE.lock().expect("inspect cleared test store route").is_none());
+    match (before, object_store_handle()) {
+        (None, None) => {}
+        (Some(before), Some(after)) => assert!(Arc::ptr_eq(&before, &after), "fixture drop restores prior store identity"),
+        _ => panic!("fixture drop must restore the prior store option"),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_notification_valid_parent_preserves_child_storage_and_cache() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.assert_parent_and_child().await;
+    fixture
+        .manager
+        .user_notification_handler(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect("refresh valid parent notification");
+    fixture.assert_parent_and_child().await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_store_fixture_raw_valid_overwrite_is_observed() {
+    let fixture = IdentityStoreFixture::new().await;
+    let parent = fixture
+        .manager
+        .api
+        .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect("load valid parent before raw overwrite");
+    let mut raw = serde_json::to_value(parent).expect("serialize valid parent fixture");
+    raw["credentials"]["name"] = serde_json::json!("updated-name");
+    let path = format!("{}{IDENTITY_PARENT}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
+    save_config(
+        fixture.ecstore.clone(),
+        &path,
+        serde_json::to_vec(&raw).expect("encode raw parent fixture"),
+    )
+    .await
+    .expect("overwrite parent through real config storage");
+    fixture
+        .manager
+        .load_user(IDENTITY_PARENT)
+        .await
+        .expect("reload parent after raw overwrite");
+    let stored = fixture
+        .manager
+        .api
+        .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect("read overwritten parent from storage");
+    assert_eq!(stored.credentials.name.as_deref(), Some("updated-name"));
+    let cached = fixture
+        .manager
+        .get_user(IDENTITY_PARENT)
+        .await
+        .expect("read overwritten parent from cache");
+    assert_eq!(cached.credentials.name.as_deref(), Some("updated-name"));
+    fixture.assert_parent_and_child().await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_corrupt_reads_preserve_crypto_error() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
+    let locked = fixture
+        .manager
+        .api
+        .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect_err("corrupt stored identity must fail locked read");
+    assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
+    let mut users = HashMap::new();
+    let no_lock = fixture
+        .manager
+        .api
+        .load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
+        .await
+        .expect_err("corrupt stored identity must fail no-lock read");
+    assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
+    assert!(users.is_empty(), "failed no-lock read must not populate users");
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_corrupt_notification_preserves_child_storage_and_cache() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.assert_parent_and_child().await;
+    fixture.overwrite_parent_bytes(&[0xff, 0xfe, 0xfd]).await;
+    let notification = fixture
+        .manager
+        .user_notification_handler(IDENTITY_PARENT, UserType::Reg)
+        .await;
+    fixture.assert_child_storage_and_identity_cache().await;
+    let error = notification.expect_err("corrupt parent notification must return an error");
+    assert!(
+        matches!(error, IamError::CryptoError(_)),
+        "preserve notification crypto error, got {error:?}"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_invalid_schema_preserves_decode_error() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.assert_parent_and_child().await;
+    fixture
+        .overwrite_parent_bytes(br#"{"version":"not-an-integer","credentials":{}}"#)
+        .await;
+    let error = fixture
+        .manager
+        .api
+        .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect_err("invalid identity field type must fail decode");
+    assert_identity_data_error(error);
+    let mut users = HashMap::new();
+    let error = fixture
+        .manager
+        .api
+        .load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
+        .await
+        .expect_err("invalid identity field type must fail no-lock decode");
+    assert_identity_data_error(error);
+    assert!(users.is_empty(), "failed decode must not populate users");
+    let notification = fixture
+        .manager
+        .user_notification_handler(IDENTITY_PARENT, UserType::Reg)
+        .await;
+    fixture.assert_child_storage_and_identity_cache().await;
+    assert_identity_data_error(notification.expect_err("invalid parent notification must return decode error"));
+}
+
+fn assert_identity_data_error(error: IamError) {
+    let IamError::Io(inner) = error else {
+        panic!("identity decode must preserve Io serde error, got {error:?}");
+    };
+    let cause = inner
+        .get_ref()
+        .and_then(|cause| cause.downcast_ref::<serde_json::Error>())
+        .expect("identity decode error retains serde source");
+    assert!(cause.is_data(), "identity field type must produce serde Data error");
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_physically_missing_is_no_such_user() {
+    let fixture = IdentityStoreFixture::new().await;
+    let name = "physically-missing-identity";
+    let error = fixture
+        .manager
+        .api
+        .load_user_identity(name, UserType::Reg)
+        .await
+        .expect_err("missing identity must be absent");
+    assert!(
+        matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
+        "preserve absent name, got {error:?}"
+    );
+    let mut users = HashMap::new();
+    let error = fixture
+        .manager
+        .api
+        .load_user_no_lock(name, UserType::Reg, &mut users)
+        .await
+        .expect_err("missing no-lock identity must be absent");
+    assert!(
+        matches!(error, IamError::NoSuchUser(ref missing) if missing == name),
+        "preserve absent name, got {error:?}"
+    );
+    assert!(users.is_empty(), "missing read must not populate users");
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_empty_reads_preserve_crypto_error() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.overwrite_parent_bytes(&[]).await;
+    let locked = fixture
+        .manager
+        .api
+        .load_user_identity(IDENTITY_PARENT, UserType::Reg)
+        .await
+        .expect_err("empty stored identity must fail locked read");
+    assert!(matches!(locked, IamError::CryptoError(_)), "preserve crypto error, got {locked:?}");
+    let mut users = HashMap::new();
+    let no_lock = fixture
+        .manager
+        .api
+        .load_user_no_lock(IDENTITY_PARENT, UserType::Reg, &mut users)
+        .await
+        .expect_err("empty stored identity must fail no-lock read");
+    assert!(matches!(no_lock, IamError::CryptoError(_)), "preserve crypto error, got {no_lock:?}");
+    assert!(users.is_empty(), "failed no-lock read must not populate users");
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn identity_empty_notification_preserves_child_storage_and_cache() {
+    let fixture = IdentityStoreFixture::new().await;
+    fixture.assert_parent_and_child().await;
+    fixture.overwrite_parent_bytes(&[]).await;
+    let notification = fixture
+        .manager
+        .user_notification_handler(IDENTITY_PARENT, UserType::Reg)
+        .await;
+    fixture.assert_child_storage_and_identity_cache().await;
+    let error = notification.expect_err("empty parent notification must return an error");
+    assert!(
+        matches!(error, IamError::CryptoError(_)),
+        "preserve notification crypto error, got {error:?}"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn policy_empty_payload_retains_no_such_policy() {
+    let fixture = IdentityStoreFixture::new().await;
+    let name = "empty-policy";
+    let path = format!("{}{name}/policy.json", IAM_CONFIG_POLICIES_PREFIX.as_str());
+    save_config(fixture.ecstore.clone(), &path, Vec::new())
+        .await
+        .expect("save real empty policy payload");
+    let locked = fixture
+        .manager
+        .api
+        .load_policy(name)
+        .await
+        .err()
+        .expect("empty policy must retain absent-policy behavior");
+    assert!(matches!(locked, IamError::NoSuchPolicy), "preserve policy error, got {locked:?}");
+    let mut policies = HashMap::new();
+    let no_lock = fixture
+        .manager
+        .api
+        .load_policy_doc_no_lock(name, &mut policies)
+        .await
+        .expect_err("empty no-lock policy must retain absent-policy behavior");
+    assert!(matches!(no_lock, IamError::NoSuchPolicy), "preserve policy error, got {no_lock:?}");
+    assert!(policies.is_empty(), "failed no-lock policy read must not populate policies");
+}

crates/io-metrics/src/lib.rs

@@ -281,8 +281,9 @@ pub use deadlock_metrics::{
 // Lock metrics exports
 pub use lock_metrics::{
     LockMetricsSummary, record_contention_event, record_early_release, record_lock_hold_time, record_lock_optimization_enabled,
-    record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
-    record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold, record_spin_attempt, record_spin_count_change,
+    record_object_lock_acquire_failure, record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled,
+    record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold,
+    record_spin_attempt, record_spin_count_change,
 };
 
 pub use process_lock_metrics::{

crates/io-metrics/src/lock_metrics.rs

@@ -62,6 +62,25 @@ pub fn record_contention_event() {
     counter!("rustfs_lock_contentions").increment(1);
 }
 
+/// Record a failed object namespace lock acquisition with bounded labels.
+#[inline(always)]
+pub fn record_object_lock_acquire_failure(
+    op: &'static str,
+    mode: &'static str,
+    resource_class: &'static str,
+    reason: &'static str,
+) {
+    use metrics::counter;
+    counter!(
+        "rustfs_object_lock_acquire_failures_total",
+        "op" => op,
+        "mode" => mode,
+        "resource_class" => resource_class,
+        "reason" => reason
+    )
+    .increment(1);
+}
+
 /// Record a remote lock RPC that exceeded its caller's deadline.
 #[inline(always)]
 pub fn record_remote_lock_rpc_timeout(peer: &str, op: &'static str) {
@@ -244,6 +263,7 @@ mod tests {
             record_remote_lock_rpc_detached("lock", "detached");
             record_remote_lock_rpc_late_completion("lock", "success");
             record_remote_lock_late_release("released");
+            record_object_lock_acquire_failure("put_object_commit", "write", "system_metadata", "timeout");
         });
 
         let emitted: std::collections::HashSet<String> = snapshotter
@@ -267,10 +287,35 @@ mod tests {
             "rustfs_remote_lock_rpc_detached_total",
             "rustfs_remote_lock_rpc_late_completions_total",
             "rustfs_remote_lock_late_releases_total",
+            "rustfs_object_lock_acquire_failures_total",
         ] {
             assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
         }
     }
+
+    #[test]
+    fn object_lock_acquire_failure_metric_is_recorded_without_resource_labels() {
+        let recorder = SeenMetricsRecorder::default();
+        metrics::with_local_recorder(&recorder, || {
+            record_object_lock_acquire_failure("put_object_commit", "write", "system_metadata", "timeout");
+        });
+
+        let keys = recorder.counters.lock().expect("counter keys should be lockable");
+        let key = keys
+            .iter()
+            .find(|key| key.name() == "rustfs_object_lock_acquire_failures_total")
+            .expect("object lock failure counter should be emitted");
+        let labels = key.labels().map(|label| (label.key(), label.value())).collect::<Vec<_>>();
+        assert_eq!(
+            labels,
+            [
+                ("op", "put_object_commit"),
+                ("mode", "write"),
+                ("resource_class", "system_metadata"),
+                ("reason", "timeout"),
+            ]
+        );
+    }
     use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
     use std::sync::{Arc, Mutex};
 

crates/io-metrics/src/s3_http_metrics.rs

@@ -29,6 +29,7 @@ pub type S3HttpCompletionObserver = fn(S3Operation, Duration, bool);
 pub type S3HttpCompletionObserverEnabled = fn() -> bool;
 
 const METRIC: &str = "rustfs_s3_http_requests_total";
+const LOCK_FAILURE_METRIC: &str = "rustfs_s3_lock_failures_total";
 const METHODS: [&str; 10] = [
     "GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS", "PATCH", "CONNECT", "TRACE", "OTHER",
 ];
@@ -112,6 +113,21 @@ pub(crate) fn observe_s3_http_operation(op: S3Operation) {
     });
 }
 
+/// Attribute a lock-related S3 failure to the request operation without
+/// exporting bucket, object, owner, or request identifiers as metric labels.
+pub fn record_s3_http_lock_failure(resource_class: &'static str, mode: &'static str, reason: &'static str) {
+    let _ = CURRENT_OPERATION.try_with(|current| {
+        counter!(
+            LOCK_FAILURE_METRIC,
+            "op" => operation_label(current.get()),
+            "resource_class" => resource_class,
+            "mode" => mode,
+            "reason" => reason
+        )
+        .increment(1);
+    });
+}
+
 /// An external request is counted exactly once: at response headers, at a
 /// service error, or when its future is dropped before producing a response.
 /// Body-stream failures after headers use the existing streaming metrics.
@@ -196,7 +212,7 @@ pub fn s3_http_metrics_snapshot() -> Vec<S3HttpMetricSnapshot> {
 mod tests {
     use super::*;
     use metrics::with_local_recorder;
-    use metrics_util::debugging::DebuggingRecorder;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
 
     #[test]
     fn completion_observer_sees_each_dispatched_request_once() {
@@ -303,4 +319,42 @@ mod tests {
         assert_eq!(other.method, METHODS.len() - 1);
         assert_eq!(other.operation, UNKNOWN_OPERATION);
     }
+
+    #[test]
+    fn lock_failure_metric_uses_bounded_request_labels() {
+        let recorder = DebuggingRecorder::new();
+        with_local_recorder(&recorder, || {
+            let mut request = S3HttpRequestGuard::new("PUT");
+            request.in_scope(|| {
+                observe_s3_http_operation(S3Operation::PutObject);
+                record_s3_http_lock_failure("system_metadata", "write", "timeout");
+            });
+            record_s3_http_lock_failure("object_namespace", "read", "quorum");
+        });
+
+        let exported = recorder
+            .snapshotter()
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .filter(|(key, _, _, _)| key.key().name() == LOCK_FAILURE_METRIC)
+            .collect::<Vec<_>>();
+        assert_eq!(exported.len(), 1, "background lock errors must not be attributed to an S3 request");
+        let (key, _, _, value) = &exported[0];
+        assert!(matches!(value, DebugValue::Counter(1)));
+        let labels = key
+            .key()
+            .labels()
+            .map(|label| (label.key(), label.value()))
+            .collect::<std::collections::BTreeMap<_, _>>();
+        assert_eq!(
+            labels,
+            std::collections::BTreeMap::from([
+                ("mode", "write"),
+                ("op", S3Operation::PutObject.as_str()),
+                ("reason", "timeout"),
+                ("resource_class", "system_metadata"),
+            ])
+        );
+    }
 }

crates/obs/README.md

@@ -36,7 +36,7 @@ use rustfs_obs::init_obs;
 #[tokio::main]
 async fn main() {
     // Build config from environment variables, then initialise all backends.
-    let _guard = init_obs(None).await.expect("failed to initialise observability");
+    let _guard = init_obs(None, None).await.expect("failed to initialise observability");
 
     tracing::info!("RustFS started");
 
@@ -56,7 +56,7 @@ async fn main() {
 ```rust
 use rustfs_obs::init_obs;
 
-let _guard = init_obs(Some("http://otel-collector:4318".to_string()))
+let _guard = init_obs(Some("http://otel-collector:4318".to_string()), None)
 .await
 .expect("observability init failed");
 ```
@@ -136,7 +136,7 @@ With OTLP → Collector → Prometheus, the time from a failure to an alert also
 | Variable                     | Default           | Description                                             |
 |------------------------------|-------------------|---------------------------------------------------------|
 | `RUSTFS_OBS_SERVICE_NAME`    | `rustfs`          | OTel `service.name`                                     |
-| `RUSTFS_OBS_SERVICE_VERSION` | _(crate version)_ | OTel `service.version`                                  |
+| `RUSTFS_OBS_SERVICE_VERSION` | RustFS build version | OTel `service.version`; the explicit environment value overrides the version passed by the RustFS binary |
 | `RUSTFS_OBS_ENVIRONMENT`     | `development`     | Deployment environment (`production`, `development`, …) |
 
 ### Local logging

crates/obs/examples/server.rs

@@ -20,7 +20,7 @@ use tracing::{Level, info, instrument};
 #[tokio::main]
 async fn main() {
     let obs_conf = Some("http://localhost:4318".to_string());
-    let _guard = init_obs(obs_conf).await;
+    let _guard = init_obs(obs_conf, None).await;
     let span = tracing::span!(Level::INFO, "main");
     let _enter = span.enter();
     info!("Program starts");

crates/obs/src/config.rs

@@ -46,7 +46,7 @@ use rustfs_config::observability::{
 use rustfs_config::{
     APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
     DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_PROFILING_EXPORT_ENABLED,
-    DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
+    DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, USE_STDOUT, VERSION,
 };
 use rustfs_utils::{
     get_env_bool, get_env_bool_with_aliases, get_env_f64, get_env_opt_bool, get_env_opt_str, get_env_opt_u64, get_env_str,
@@ -162,7 +162,7 @@ pub struct OtelConfig {
     pub meter_interval: Option<u64>,
     /// OTel `service.name` attribute (default: `APP_NAME`).
     pub service_name: Option<String>,
-    /// OTel `service.version` attribute (default: `SERVICE_VERSION`).
+    /// OTel `service.version` attribute (default: the supplied application version or Cargo package version).
     pub service_version: Option<String>,
     /// Deployment environment tag, e.g. `production` or `development`.
     pub environment: Option<String>,
@@ -234,6 +234,10 @@ impl OtelConfig {
     /// `None` or an empty string the value is read from the
     /// `RUSTFS_OBS_ENDPOINT` environment variable instead.
     ///
+    /// The default service version is the Cargo package version. Applications
+    /// that know their display/build version should use
+    /// [`OtelConfig::extract_otel_config_from_env_with_version`].
+    ///
     /// When no endpoint is configured at all, `use_stdout` is forced to `true`
     /// so that logs are still visible during development.
     ///
@@ -250,6 +254,15 @@ impl OtelConfig {
     /// );
     /// ```
     pub fn extract_otel_config_from_env(endpoint: Option<String>) -> OtelConfig {
+        Self::extract_otel_config_from_env_with_version(endpoint, None)
+    }
+
+    /// Build an [`OtelConfig`] using `version` as the default OTel service version.
+    ///
+    /// An explicit `RUSTFS_OBS_SERVICE_VERSION` environment variable takes
+    /// precedence over `version`. When `version` is `None`, the current Cargo
+    /// package version is used.
+    pub fn extract_otel_config_from_env_with_version(endpoint: Option<String>, version: Option<String>) -> OtelConfig {
         let endpoint = match endpoint {
             Some(ep) if !ep.is_empty() => ep,
             _ => env::var(ENV_OBS_ENDPOINT).unwrap_or_default(),
@@ -312,7 +325,7 @@ impl OtelConfig {
             sample_ratio: Some(get_env_f64(ENV_OBS_SAMPLE_RATIO, SAMPLE_RATIO)),
             meter_interval: Some(get_env_u64(ENV_OBS_METER_INTERVAL, METER_INTERVAL)),
             service_name: Some(get_env_str(ENV_OBS_SERVICE_NAME, APP_NAME)),
-            service_version: Some(get_env_str(ENV_OBS_SERVICE_VERSION, SERVICE_VERSION)),
+            service_version: Some(get_env_str(ENV_OBS_SERVICE_VERSION, version.as_deref().unwrap_or(VERSION))),
             environment: Some(get_env_str(ENV_OBS_ENVIRONMENT, ENVIRONMENT)),
             // Local logging
             logger_level: Some(get_env_str(ENV_OBS_LOGGER_LEVEL, DEFAULT_LOG_LEVEL)),
@@ -459,6 +472,29 @@ mod tests {
         OtelConfig::extract_otel_config_from_env(None).profiling_export_enabled
     }
 
+    #[test]
+    fn service_version_uses_application_version_when_environment_is_unset() {
+        with_profiling_env_lock(|| {
+            temp_env::with_var_unset(ENV_OBS_SERVICE_VERSION, || {
+                let config = OtelConfig::extract_otel_config_from_env_with_version(None, Some("2.3.4".to_string()));
+                assert_eq!(config.service_version.as_deref(), Some("2.3.4"));
+
+                let config = OtelConfig::extract_otel_config_from_env_with_version(None, None);
+                assert_eq!(config.service_version.as_deref(), Some(VERSION));
+            });
+        });
+    }
+
+    #[test]
+    fn service_version_environment_override_takes_precedence_over_application_version() {
+        with_profiling_env_lock(|| {
+            temp_env::with_var(ENV_OBS_SERVICE_VERSION, Some("custom-build-42"), || {
+                let config = OtelConfig::extract_otel_config_from_env_with_version(None, Some("2.3.4".to_string()));
+                assert_eq!(config.service_version.as_deref(), Some("custom-build-42"));
+            });
+        });
+    }
+
     #[test]
     fn profiling_export_defaults_to_disabled_when_unset() {
         with_profiling_env_lock(|| {

crates/obs/src/global.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{AppConfig, GlobalError, OtelConfig, OtelGuard, telemetry::init_telemetry};
+use crate::{GlobalError, OtelConfig, OtelGuard, telemetry::init_telemetry};
 use std::sync::{Arc, Mutex};
 use tokio::sync::OnceCell;
 use tracing::{info, warn};
@@ -68,10 +68,12 @@ pub(crate) fn set_observability_metric_enabled(enabled: bool) {
 /// Initialize the observability module
 ///
 /// # Parameters
-/// - `config`: Configuration information
+/// - `endpoint`: Optional root OTLP endpoint; falls back to `RUSTFS_OBS_ENDPOINT`.
+/// - `version`: Optional application build version used as the default OTel
+///   `service.version`; `RUSTFS_OBS_SERVICE_VERSION` takes precedence.
 ///
 /// # Returns
-/// A tuple containing the logger and the telemetry guard
+/// An [`OtelGuard`] that must remain alive for the lifetime of the application.
 ///
 /// # Example
 /// ```no_run
@@ -79,16 +81,15 @@ pub(crate) fn set_observability_metric_enabled(enabled: bool) {
 ///
 /// # #[tokio::main]
 /// # async fn main() {
-/// #    match init_obs(None).await {
+/// #    match init_obs(None, None).await {
 /// #         Ok(guard) => {}
 /// #         Err(e) => { eprintln!("Failed to initialize observability: {}", e); }
 /// #     }
 /// # }
 /// ```
-pub async fn init_obs(endpoint: Option<String>) -> Result<OtelGuard, GlobalError> {
-    // Load the configuration file
-    let config = AppConfig::new_with_endpoint(endpoint);
-    init_obs_with_config(&config.observability).await
+pub async fn init_obs(endpoint: Option<String>, version: Option<String>) -> Result<OtelGuard, GlobalError> {
+    let config = OtelConfig::extract_otel_config_from_env_with_version(endpoint, version);
+    init_obs_with_config(&config).await
 }
 
 /// Initialize the observability module with an explicit [`OtelConfig`].
@@ -142,7 +143,7 @@ pub async fn init_obs_with_config(config: &OtelConfig) -> Result<OtelGuard, Glob
 /// # use rustfs_obs::{ init_obs, set_global_guard};
 ///
 /// # async fn init() -> Result<(), Box<dyn std::error::Error>> {
-/// #    let guard = match init_obs(None).await{
+/// #    let guard = match init_obs(None, None).await{
 /// #         Ok(g) => g,
 /// #         Err(e) => { return Err(Box::new(e)); }
 /// #    };

crates/obs/src/lib.rs

@@ -31,7 +31,7 @@
 //!
 //! # #[tokio::main]
 //! # async fn main() {
-//! #   let _guard = match init_obs(None).await {
+//! #   let _guard = match init_obs(None, None).await {
 //! #         Ok(g) => g,
 //! #         Err(e) => {
 //! #             panic!("Failed to initialize observability: {:?}", e);

crates/obs/src/telemetry/resource.rs

@@ -31,7 +31,7 @@ use opentelemetry_semantic_conventions::{
         SERVICE_VERSION as OTEL_SERVICE_VERSION,
     },
 };
-use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
+use rustfs_config::{APP_NAME, ENVIRONMENT, VERSION};
 use rustfs_utils::get_local_ip_with_default;
 use std::borrow::Cow;
 
@@ -40,7 +40,7 @@ use std::borrow::Cow;
 /// The resource carries the following attributes:
 /// - `service.name` — from `config.service_name`, defaulting to [`APP_NAME`].
 /// - `service.version` — from `config.service_version`, defaulting to
-///   [`SERVICE_VERSION`].
+///   [`VERSION`].
 /// - `deployment.environment` — from `config.environment`, defaulting to
 ///   [`ENVIRONMENT`].
 /// - `rustfs.node` / `service.instance.id` — the stable RustFS local node name
@@ -61,7 +61,7 @@ pub(super) fn build_resource(config: &OtelConfig) -> Resource {
             [
                 KeyValue::new(
                     OTEL_SERVICE_VERSION,
-                    Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
+                    Cow::Borrowed(config.service_version.as_deref().unwrap_or(VERSION)).to_string(),
                 ),
                 KeyValue::new(
                     DEPLOYMENT_ENVIRONMENT_NAME,
@@ -81,6 +81,22 @@ mod tests {
     use super::*;
     use opentelemetry::Key;
 
+    #[test]
+    fn build_resource_uses_configured_service_version() {
+        let config = OtelConfig {
+            service_version: Some("@a1b2c3d".to_string()),
+            ..OtelConfig::default()
+        };
+        let resource = build_resource(&config);
+
+        assert_eq!(
+            resource
+                .get(&Key::from_static_str(OTEL_SERVICE_VERSION))
+                .map(|value| value.to_string()),
+            Some("@a1b2c3d".to_string())
+        );
+    }
+
     #[tokio::test]
     async fn build_resource_uses_stable_local_node_identity() {
         let _guard = crate::node_identity::local_node_identity_test_guard().await;

crates/policy/Cargo.toml

@@ -58,6 +58,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
 time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
 serde = { workspace = true, features = ["derive", "rc"] }
 serde_json = { workspace = true, features = ["raw_value"] }
+uuid = { workspace = true, features = ["serde"] }
 thiserror.workspace = true
 strum = { workspace = true, features = ["derive"] }
 rustfs-crypto = { workspace = true }

crates/policy/src/auth/credentials.rs

@@ -26,7 +26,6 @@ use tracing::debug;
 const ACCESS_KEY_MIN_LEN: usize = 3;
 const ACCESS_KEY_MAX_LEN: usize = 128;
 const SECRET_KEY_MIN_LEN: usize = 8;
-const SECRET_KEY_MAX_LEN: usize = 40;
 
 pub const ACCOUNT_ON: &str = "on";
 pub const ACCOUNT_OFF: &str = "off";
@@ -122,7 +121,7 @@ pub fn create_new_credentials_with_metadata(
         return Err(Error::InvalidAccessKeyLength);
     }
 
-    if sk.len() < SECRET_KEY_MIN_LEN || sk.len() > SECRET_KEY_MAX_LEN {
+    if !is_secret_key_valid(sk) {
         return Err(Error::InvalidSecretKeyLength);
     }
 
@@ -450,4 +449,53 @@ mod reserved_chars_tests {
 
         assert!(matches!(err, crate::error::Error::InvalidSecretKeyLength));
     }
+
+    #[test]
+    fn credential_creation_preserves_custom_secret_keys() {
+        for length in [8, 40, 41, 90, 128, 256, 257, 4096] {
+            let secret_key = "s".repeat(length);
+            let cred = create_new_credentials_with_metadata("custom-access-key", &secret_key, &HashMap::new(), "")
+                .expect("custom secret key should be accepted");
+            assert_eq!(cred.secret_key, secret_key, "the full {length}-byte secret must be preserved");
+        }
+    }
+
+    #[test]
+    fn credential_creation_measures_the_minimum_in_utf8_bytes() {
+        let cred = create_new_credentials_with_metadata("custom-access-key", "éééé", &HashMap::new(), "")
+            .expect("eight UTF-8 bytes should be accepted");
+        assert_eq!(cred.secret_key, "éééé");
+
+        for secret_key in ["1234567", "ééé"] {
+            let err = create_new_credentials_with_metadata("custom-access-key", secret_key, &HashMap::new(), "")
+                .expect_err("fewer than eight UTF-8 bytes must be rejected");
+            assert!(matches!(err, crate::error::Error::InvalidSecretKeyLength));
+        }
+    }
+
+    #[test]
+    fn credential_creation_signs_with_the_entire_custom_secret() {
+        let mut claims = HashMap::new();
+        claims.insert(
+            "exp".to_owned(),
+            serde_json::json!((time::OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp()),
+        );
+        for length in [90, 256, 4096] {
+            let secret_key = format!("{}a", "s".repeat(length - 1));
+            let cred = create_new_credentials_with_metadata("custom-access-key", &secret_key, &claims, &secret_key)
+                .expect("long secret key should sign a session token");
+            crate::utils::extract_claims::<serde_json::Value>(&cred.session_token, &secret_key)
+                .expect("the original secret should verify the session token");
+            let changed_tail = format!("{}b", "s".repeat(length - 1));
+            assert!(crate::utils::extract_claims::<serde_json::Value>(&cred.session_token, &changed_tail).is_err());
+            assert!(crate::utils::extract_claims::<serde_json::Value>(&cred.session_token, &secret_key[..40]).is_err());
+        }
+    }
+
+    #[test]
+    fn credential_generation_keeps_the_default_lengths() {
+        let (access_key, secret_key) = super::generate_credentials().expect("generate default credentials");
+        assert_eq!(access_key.len(), 20);
+        assert_eq!(secret_key.len(), 40);
+    }
 }

crates/policy/src/auth/mod.rs

@@ -22,6 +22,33 @@ use serde_json::json;
 use std::collections::HashMap;
 use time::OffsetDateTime;
 
+/// A nonnil stored principal identifier; parsing it does not establish identity authority.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(try_from = "uuid::Uuid", into = "uuid::Uuid")]
+pub struct PrincipalIncarnation(uuid::Uuid);
+
+#[derive(Debug, thiserror::Error)]
+#[error("principal incarnation must not be nil")]
+pub struct InvalidPrincipalIncarnation;
+
+impl TryFrom<uuid::Uuid> for PrincipalIncarnation {
+    type Error = InvalidPrincipalIncarnation;
+
+    fn try_from(value: uuid::Uuid) -> Result<Self, Self::Error> {
+        if value.is_nil() {
+            Err(InvalidPrincipalIncarnation)
+        } else {
+            Ok(Self(value))
+        }
+    }
+}
+
+impl From<PrincipalIncarnation> for uuid::Uuid {
+    fn from(value: PrincipalIncarnation) -> Self {
+        value.0
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct UserIdentity {
     pub version: i64,
@@ -29,6 +56,8 @@ pub struct UserIdentity {
     /// updatedAt (RFC3339), legacy RustFS: update_at. Serialize as updatedAt
     #[serde(rename = "updatedAt", alias = "update_at", default, with = "crate::serde_datetime::option")]
     pub update_at: Option<OffsetDateTime>,
+    #[serde(default, rename = "principalIncarnation", skip_serializing_if = "Option::is_none")]
+    pub principal_incarnation: Option<PrincipalIncarnation>,
 }
 
 impl UserIdentity {
@@ -44,6 +73,7 @@ impl UserIdentity {
             version: 1,
             credentials,
             update_at: Some(OffsetDateTime::now_utc()),
+            principal_incarnation: None,
         }
     }
 
@@ -73,13 +103,124 @@ impl From<Credentials> for UserIdentity {
             version: 1,
             credentials: value,
             update_at: Some(OffsetDateTime::now_utc()),
+            principal_incarnation: None,
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::UserIdentity;
+    use super::{PrincipalIncarnation, UserIdentity};
+    use rustfs_credentials::Credentials;
+    use serde_json::json;
+    use uuid::Uuid;
+
+    const INCARNATION: &str = "8f422eba-aab4-4ec3-a7ec-d1717463b520";
+
+    #[test]
+    fn test_user_identity_legacy_fields_absent() {
+        let mut legacy = json!({"version": 1, "credentials": {"accessKey": "ak", "secretKey": "sk12345678"}});
+        let identity: UserIdentity = serde_json::from_value(legacy.clone()).expect("decode legacy identity");
+        assert!(identity.principal_incarnation.is_none());
+        legacy["principalIncarnation"] = serde_json::Value::Null;
+        let identity: UserIdentity = serde_json::from_value(legacy).expect("decode null incarnation");
+        assert!(identity.principal_incarnation.is_none());
+    }
+
+    #[test]
+    fn test_user_identity_legacy_serialization_shape() {
+        let identity = UserIdentity::default();
+        let serialized = serde_json::to_value(&identity).expect("serialize legacy identity");
+        assert_eq!(serialized, json!({"version": 0, "credentials": identity.credentials, "updatedAt": null}));
+        assert!(serialized.get("principalIncarnation").is_none());
+    }
+
+    #[test]
+    fn test_user_identity_constructors_do_not_mint_incarnation() {
+        let default = UserIdentity::default();
+        assert!(default.principal_incarnation.is_none());
+        assert_eq!(default.version, 0);
+        assert!(default.update_at.is_none());
+        let credentials = Credentials {
+            access_key: "ak".to_string(),
+            secret_key: "sk12345678".to_string(),
+            ..Default::default()
+        };
+        let expected_credentials = serde_json::to_value(&credentials).expect("serialize expected credentials");
+        for identity in [UserIdentity::new(credentials.clone()), UserIdentity::from(credentials)] {
+            assert!(identity.principal_incarnation.is_none());
+            assert_eq!(identity.version, 1);
+            assert!(identity.update_at.is_some());
+            assert_eq!(
+                serde_json::to_value(&identity.credentials).expect("serialize constructor credentials"),
+                expected_credentials
+            );
+            assert!(
+                serde_json::to_value(identity)
+                    .expect("serialize constructor identity")
+                    .get("principalIncarnation")
+                    .is_none()
+            );
+        }
+    }
+
+    #[test]
+    fn test_principal_incarnation_rejects_nil_and_malformed() {
+        assert!(PrincipalIncarnation::try_from(Uuid::nil()).is_err());
+        for value in ["00000000-0000-0000-0000-000000000000", "not-a-uuid", ""] {
+            assert!(serde_json::from_value::<PrincipalIncarnation>(json!(value)).is_err(), "reject {value}");
+            let identity = json!({"version": 1, "credentials": {}, "principalIncarnation": value});
+            assert!(serde_json::from_value::<UserIdentity>(identity).is_err(), "identity rejects {value}");
+        }
+    }
+
+    #[test]
+    fn test_principal_incarnation_roundtrip_preserves_uuid() {
+        let uuid = Uuid::parse_str(INCARNATION).expect("parse fixed incarnation");
+        let incarnation = PrincipalIncarnation::try_from(uuid).expect("accept nonnil incarnation");
+        assert_eq!(Uuid::from(incarnation).as_bytes(), uuid.as_bytes());
+        let serialized = serde_json::to_value(incarnation).expect("serialize incarnation");
+        assert_eq!(serialized, json!(INCARNATION));
+        assert_eq!(
+            serde_json::from_value::<PrincipalIncarnation>(serialized).expect("decode incarnation"),
+            incarnation
+        );
+    }
+
+    #[test]
+    fn test_user_identity_live_incarnation_roundtrip() {
+        let mut identity = UserIdentity::new(Credentials::default());
+        identity.principal_incarnation = Some(
+            PrincipalIncarnation::try_from(Uuid::parse_str(INCARNATION).expect("parse fixed incarnation"))
+                .expect("accept fixed incarnation"),
+        );
+        let serialized = serde_json::to_value(&identity).expect("serialize identity with incarnation");
+        assert_eq!(serialized["principalIncarnation"], INCARNATION);
+        let encoded = serde_json::to_string(&identity).expect("encode identity with incarnation");
+        let decoded: UserIdentity = serde_json::from_str(&encoded).expect("decode identity with incarnation");
+        assert_eq!(decoded.principal_incarnation, identity.principal_incarnation);
+        assert_eq!(serde_json::to_value(decoded).expect("reserialize identity"), serialized);
+    }
+
+    #[test]
+    fn test_user_identity_timestamp_aliases_unchanged() {
+        let mut minio = json!({"version": 1, "credentials": {}, "updatedAt": "2025-03-07T12:00:00Z"});
+        let encoded = serde_json::to_string(&minio).expect("encode updatedAt fixture");
+        let identity: UserIdentity = serde_json::from_str(&encoded).expect("decode updatedAt timestamp");
+        minio["update_at"] = minio
+            .as_object_mut()
+            .expect("identity object")
+            .remove("updatedAt")
+            .expect("existing timestamp");
+        let encoded = serde_json::to_string(&minio).expect("encode update_at fixture");
+        let legacy: UserIdentity = serde_json::from_str(&encoded).expect("decode update_at alias");
+        assert_eq!(legacy.update_at, identity.update_at);
+        assert_eq!(legacy.version, 1);
+        let serialized = serde_json::to_value(legacy).expect("serialize legacy timestamp");
+        assert_eq!(serialized["updatedAt"], "2025-03-07T12:00:00Z");
+        assert!(serialized.get("update_at").is_none());
+        assert!(serialized.get("principalIncarnation").is_none());
+    }
 
     /// Deserialize UserIdentity from MinIO-style JSON (RFC3339 updatedAt).
     #[test]

crates/policy/src/policy/action.rs

@@ -527,6 +527,9 @@ pub enum AdminAction {
     ImportBucketMetadataAction,
     #[strum(serialize = "admin:ExportBucketMetadata")]
     ExportBucketMetadataAction,
+    /// Reconcile metadata left after an operator removed all physical bucket volumes.
+    #[strum(serialize = "admin:RecoverOrphanedBucket")]
+    RecoverOrphanedBucketAction,
     #[strum(serialize = "admin:GetTableCatalog")]
     GetTableCatalogAction,
     #[strum(serialize = "admin:MigrateTableCatalog")]
@@ -682,6 +685,7 @@ impl AdminAction {
                 | AdminAction::GetReplicationMetricsAction
                 | AdminAction::ImportBucketMetadataAction
                 | AdminAction::ExportBucketMetadataAction
+                | AdminAction::RecoverOrphanedBucketAction
                 | AdminAction::GetTableCatalogAction
                 | AdminAction::MigrateTableCatalogAction
                 | AdminAction::GetTableBucketAction
@@ -909,6 +913,14 @@ mod tests {
         assert!(AdminAction::GetMetricsAction.is_valid());
     }
 
+    #[test]
+    fn orphaned_bucket_recovery_admin_action_is_valid() {
+        let action = AdminAction::try_from("admin:RecoverOrphanedBucket").expect("parse recovery action");
+        assert_eq!(action, AdminAction::RecoverOrphanedBucketAction);
+        assert!(action.is_valid());
+        assert_eq!(<&str>::from(action), "admin:RecoverOrphanedBucket");
+    }
+
     #[test]
     fn test_bucket_on_demand_migration_admin_actions_are_valid() {
         let set_action = AdminAction::try_from("admin:SetBucketOnDemandMigration").expect("parse set action");

crates/policy/src/policy/function/key_name.rs

@@ -198,6 +198,9 @@ pub enum S3KeyName {
     #[strum(serialize = "s3:object-lock-retain-until-date")]
     S3ObjectLockRetainUntilDate,
 
+    #[strum(serialize = "s3:object-lock-remaining-retention-days")]
+    S3ObjectLockRemainingRetentionDays,
+
     #[strum(serialize = "s3:object-lock-legal-hold")]
     S3ObjectLockLegalHold,
 
@@ -393,6 +396,10 @@ mod tests {
     #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
     #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
     #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
+    #[test_case(
+        "s3:object-lock-remaining-retention-days",
+        KeyName::S3(S3KeyName::S3ObjectLockRemainingRetentionDays)
+    )]
     #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
     #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
     #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]
@@ -417,6 +424,10 @@ mod tests {
     #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
     #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
     #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
+    #[test_case(
+        "s3:object-lock-remaining-retention-days",
+        KeyName::S3(S3KeyName::S3ObjectLockRemainingRetentionDays)
+    )]
     #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
     #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
     #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]
@@ -437,6 +448,10 @@ mod tests {
     #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId))]
     #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
     #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
+    #[test_case(
+        "s3:object-lock-remaining-retention-days",
+        KeyName::S3(S3KeyName::S3ObjectLockRemainingRetentionDays)
+    )]
     #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
     #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
     #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]

crates/policy/src/policy/function/number.rs

@@ -104,6 +104,7 @@ mod tests {
         key_name::KeyName::{self, *},
         key_name::S3KeyName::*,
     };
+    use std::collections::HashMap;
     use test_case::test_case;
 
     fn new_func(name: KeyName, variable: Option<String>, value: i64) -> NumberFunc {
@@ -125,6 +126,35 @@ mod tests {
         Ok(())
     }
 
+    #[test_case(r#"{"s3:object-lock-remaining-retention-days": 30}"#; "number")]
+    #[test_case(r#"{"s3:object-lock-remaining-retention-days": "30"}"#; "string")]
+    fn remaining_retention_days_deserializes(input: &str) -> Result<(), serde_json::Error> {
+        let v: NumberFunc = serde_json::from_str(input)?;
+        assert_eq!(v, new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30));
+        Ok(())
+    }
+
+    #[test_case("29", true, false, false; "below limit")]
+    #[test_case("30", true, false, true; "at limit")]
+    #[test_case("31", false, true, false; "above limit")]
+    fn remaining_retention_days_numeric_comparisons(days: &str, less_than_equals: bool, greater_than: bool, equals: bool) {
+        let limit = new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30);
+        let values = HashMap::from([("object-lock-remaining-retention-days".to_string(), vec![days.to_string()])]);
+
+        assert_eq!(limit.evaluate(i64::le, false, &values), less_than_equals);
+        assert_eq!(limit.evaluate(i64::gt, false, &values), greater_than);
+        assert_eq!(limit.evaluate(i64::eq, false, &values), equals);
+    }
+
+    #[test]
+    fn remaining_retention_days_numeric_condition_without_value_uses_if_exists() {
+        let limit = new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30);
+        let values = HashMap::new();
+
+        assert!(!limit.evaluate(i64::le, false, &values));
+        assert!(limit.evaluate(i64::le, true, &values));
+    }
+
     #[test_case(r#"{"s3:max-keys":"1"}"#, new_func(S3(S3MaxKeys), None, 1); "1")]
     #[test_case(r#"{"s3:max-keys/a":"1"}"#, new_func(S3(S3MaxKeys), Some("a".into()), 1); "2")]
     fn test_ser(expect: &str, input: NumberFunc) -> Result<(), serde_json::Error> {

crates/policy/src/policy/variables.rs

@@ -252,6 +252,13 @@ impl PolicyVariableResolver for VariableResolver {
 }
 
 pub async fn resolve_aws_variables(pattern: &str, resolver: &dyn PolicyVariableResolver) -> Vec<String> {
+    resolve_aws_variables_with_depth(pattern, resolver, 0).await
+}
+
+async fn resolve_aws_variables_with_depth(pattern: &str, resolver: &dyn PolicyVariableResolver, depth: usize) -> Vec<String> {
+    if depth >= 10 {
+        return vec![pattern.to_string()];
+    }
     let mut results = vec![pattern.to_string()];
 
     let mut changed = true;
@@ -264,7 +271,7 @@ pub async fn resolve_aws_variables(pattern: &str, resolver: &dyn PolicyVariableR
 
         let mut new_results = Vec::new();
         for result in &results {
-            let resolved = resolve_single_pass(result, resolver).await;
+            let resolved = resolve_single_pass(result, resolver, depth).await;
             if resolved.len() > 1 || (resolved.len() == 1 && &resolved[0] != result) {
                 changed = true;
             }
@@ -288,30 +295,31 @@ pub async fn resolve_aws_variables(pattern: &str, resolver: &dyn PolicyVariableR
 fn resolve_aws_variables_boxed<'a>(
     pattern: &'a str,
     resolver: &'a dyn PolicyVariableResolver,
+    depth: usize,
 ) -> std::pin::Pin<Box<dyn Future<Output = Vec<String>> + Send + 'a>> {
-    Box::pin(resolve_aws_variables(pattern, resolver))
+    Box::pin(resolve_aws_variables_with_depth(pattern, resolver, depth))
 }
 
 /// Single pass resolution of variables in a string
-async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolver) -> Vec<String> {
+async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolver, depth: usize) -> Vec<String> {
     // Find all ${...} format variables
-    let mut results = vec![pattern.to_string()];
+    let mut results = vec![(pattern.to_string(), 0)];
 
     // Process each result string
     let mut i = 0;
     while i < results.len() {
-        let mut start = 0;
+        let mut start = results[i].1;
         let mut modified = false;
 
         // Find variables in current string
-        while let Some(pos) = results[i][start..].find("${") {
+        while let Some(pos) = results[i].0[start..].find("${") {
             let actual_pos = start + pos;
 
             // Find the matching closing brace, taking into account nested braces
             let mut brace_count = 1;
             let mut end_pos = actual_pos + 2; // Start after "${"
 
-            let bytes = results[i].as_bytes();
+            let bytes = results[i].0.as_bytes();
             while end_pos < bytes.len() && brace_count > 0 {
                 match bytes[end_pos] {
                     b'{' => brace_count += 1,
@@ -324,19 +332,23 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
             }
 
             if brace_count == 0 {
-                let var_name = &results[i][actual_pos + 2..end_pos];
+                let var_name = &results[i].0[actual_pos + 2..end_pos];
 
                 // Check if this is a nested variable (contains ${...} inside)
                 if var_name.contains("${") {
                     // For nested variables like ${${a}-${b}}, we need to resolve the inner variables first
                     // Then use the resolved result as a new variable to resolve
-                    let resolved_inner = resolve_aws_variables_boxed(var_name, resolver).await;
+                    let resolved_inner = resolve_aws_variables_boxed(var_name, resolver, depth + 1).await;
+                    if resolved_inner.len() == 1 && resolved_inner[0] == var_name {
+                        start = end_pos + 1;
+                        continue;
+                    }
                     let mut new_results = Vec::new();
 
                     for resolved_var_name in resolved_inner {
-                        let prefix = &results[i][..actual_pos];
-                        let suffix = &results[i][end_pos + 1..];
-                        new_results.push(format!("{prefix}{resolved_var_name}{suffix}"));
+                        let prefix = &results[i].0[..actual_pos];
+                        let suffix = &results[i].0[end_pos + 1..];
+                        new_results.push((format!("{prefix}{resolved_var_name}{suffix}"), actual_pos + resolved_var_name.len()));
                     }
 
                     if !new_results.is_empty() {
@@ -354,11 +366,13 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
                         if !values.is_empty() {
                             // If there are multiple values, create a new result for each value
                             let mut new_results = Vec::new();
-                            let prefix = &results[i][..actual_pos];
-                            let suffix = &results[i][end_pos + 1..];
+                            let prefix = &results[i].0[..actual_pos];
+                            let suffix = &results[i].0[end_pos + 1..];
 
                             for value in values {
-                                new_results.push(format!("{prefix}{value}{suffix}"));
+                                // Resume after the substitution; a generated placeholder
+                                // belongs to the next bounded pass, not this one.
+                                new_results.push((format!("{prefix}{value}{suffix}"), actual_pos + value.len()));
                             }
 
                             results.splice(i..i + 1, new_results);
@@ -367,9 +381,9 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
                         } else {
                             // Variable resolved to empty, just remove the variable placeholder
                             let mut new_results = Vec::new();
-                            let prefix = &results[i][..actual_pos];
-                            let suffix = &results[i][end_pos + 1..];
-                            new_results.push(format!("{prefix}{suffix}"));
+                            let prefix = &results[i].0[..actual_pos];
+                            let suffix = &results[i].0[end_pos + 1..];
+                            new_results.push((format!("{prefix}{suffix}"), actual_pos));
 
                             results.splice(i..i + 1, new_results);
                             modified = true;
@@ -391,7 +405,7 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
         }
     }
 
-    results
+    results.into_iter().map(|(value, _)| value).collect()
 }
 
 #[cfg(test)]
@@ -399,6 +413,62 @@ mod tests {
     use super::*;
     use serde_json::Value;
     use std::collections::HashMap;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    struct CycleResolver {
+        calls: AtomicUsize,
+    }
+
+    #[async_trait]
+    impl PolicyVariableResolver for CycleResolver {
+        async fn resolve(&self, name: &str) -> Option<String> {
+            // Make the vulnerable implementation terminate so a regression
+            // reports an assertion failure instead of hanging the test runner.
+            if self.calls.fetch_add(1, Ordering::Relaxed) >= 100 {
+                return None;
+            }
+            match name {
+                "aws:AccountId" => Some("$$".to_string()),
+                "aws:username" => Some("{aws:AccountId}{aws:username}".to_string()),
+                _ => None,
+            }
+        }
+
+        fn is_dynamic(&self, _: &str) -> bool {
+            false
+        }
+    }
+
+    #[tokio::test]
+    async fn generated_policy_variable_cycles_are_bounded() {
+        let resolver = CycleResolver {
+            calls: AtomicUsize::new(0),
+        };
+        let result = resolve_aws_variables("${aws:AccountId}{aws:username}", &resolver).await;
+        assert!(resolver.calls.load(Ordering::Relaxed) <= 10);
+        assert_eq!(result, vec!["${aws:AccountId}{aws:username}".to_string()]);
+    }
+
+    #[tokio::test]
+    async fn a_single_pass_resolves_all_original_variables() {
+        let mut context = VariableContext::new();
+        context.username = Some("alice".to_string());
+        let resolver = VariableResolver::new(context);
+        let pattern = "${aws:username}".repeat(20);
+        assert_eq!(resolve_single_pass(&pattern, &resolver, 0).await, vec!["alice".repeat(20)]);
+    }
+
+    #[tokio::test]
+    async fn nested_policy_variables_have_a_depth_limit() {
+        let resolver = CycleResolver {
+            calls: AtomicUsize::new(0),
+        };
+        let pattern = format!("{}aws:username{}", "${".repeat(100), "}".repeat(100));
+        let result = resolve_aws_variables(&pattern, &resolver).await;
+        assert_eq!(result.len(), 1);
+        assert!(result[0].contains("${"));
+        assert!(resolver.calls.load(Ordering::Relaxed) <= 10);
+    }
 
     #[tokio::test]
     async fn test_resolve_aws_variables_with_username() {

crates/rio/Cargo.toml

@@ -80,6 +80,7 @@ rustfs-tls-runtime.workspace = true
 rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
 serde_json = { workspace = true, features = ["raw_value"] }
 md-5 = { workspace = true }
+hmac.workspace = true
 minlz.workspace = true
 tracing.workspace = true
 thiserror.workspace = true

crates/rio/src/etag_reader.rs

@@ -14,8 +14,10 @@
 
 use crate::compress_index::{Index, TryGetIndex};
 use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
+use hmac::{Hmac, KeyInit, Mac};
 use md5::{Digest, Md5};
 use pin_project_lite::pin_project;
+use sha2::Sha256;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 use tokio::io::{AsyncRead, ReadBuf};
@@ -29,6 +31,7 @@ pin_project! {
         pub finished: bool,
         pub checksum: Option<String>,
         resolved_etag: Option<String>,
+        encrypted_etag_mac: Option<Hmac<Sha256>>,
     }
 }
 
@@ -40,9 +43,21 @@ impl<R> EtagReader<R> {
             finished: false,
             checksum,
             resolved_etag: None,
+            encrypted_etag_mac: None,
         }
     }
 
+    /// Protect the content fingerprint with the object's encryption material.
+    /// Plaintext checksum verification remains the inner reader's responsibility.
+    pub fn new_encrypted(inner: R, key: [u8; 32], nonce: [u8; 12]) -> std::io::Result<Self> {
+        let mut mac = <Hmac<Sha256> as KeyInit>::new_from_slice(&key).map_err(std::io::Error::other)?;
+        mac.update(b"RustFS:encrypted-ETag:v1\0");
+        mac.update(&nonce);
+        let mut reader = Self::new(inner, None);
+        reader.encrypted_etag_mac = Some(mac);
+        Ok(reader)
+    }
+
     /// Get the final md5 value (etag) as a hex string, only compute once.
     /// Can be called multiple times, always returns the same result after finished.
     pub fn get_etag(&mut self) -> String {
@@ -108,6 +123,14 @@ impl<R> EtagResolvable for EtagReader<R> {
         true
     }
     fn try_resolve_etag(&mut self) -> Option<String> {
+        if let Some(mut mac) = self.encrypted_etag_mac.clone() {
+            if !self.finished {
+                return None;
+            }
+            mac.update(self.get_etag().as_bytes());
+            let protected = mac.finalize().into_bytes();
+            return Some(hex_simd::encode_to_string(&protected[..16], hex_simd::AsciiCase::Lower));
+        }
         // EtagReader provides its own etag, not delegating to inner
         if let Some(checksum) = &self.checksum {
             Some(checksum.clone())
@@ -148,6 +171,47 @@ mod tests {
     use std::io::Cursor;
     use tokio::io::{AsyncReadExt, BufReader};
 
+    #[tokio::test]
+    async fn encrypted_etags_hide_plaintext_fingerprints_including_empty_objects() {
+        for data in [b"".as_slice(), b"Salary 2026: 85000 EUR".as_slice()] {
+            let md5 = hex_simd::encode_to_string(Md5::digest(data), hex_simd::AsciiCase::Lower);
+            let mut tags = Vec::new();
+            for (key, nonce) in [([1; 32], [1; 12]), ([2; 32], [1; 12]), ([1; 32], [2; 12])] {
+                let mut reader = EtagReader::new_encrypted(Cursor::new(data), key, nonce).expect("create protected ETag reader");
+                assert!(reader.try_resolve_etag().is_none());
+                let mut output = Vec::new();
+                reader.read_to_end(&mut output).await.expect("read unchanged plaintext");
+                assert_eq!(output, data);
+                let tag = reader.try_resolve_etag().expect("resolve protected ETag after EOF");
+                assert_eq!(tag.len(), 32);
+                assert_ne!(tag, md5);
+                assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag));
+                tags.push(tag);
+            }
+            assert_ne!(tags[0], tags[1], "different keys must not link identical content");
+            assert_ne!(tags[0], tags[2], "different objects under a legacy key must not link identical content");
+        }
+    }
+
+    #[tokio::test]
+    async fn encrypted_etag_reader_preserves_plaintext_digest_rejection() {
+        let plaintext = crate::HashReader::from_stream(
+            Cursor::new(b"actual plaintext"),
+            16,
+            16,
+            Some("00000000000000000000000000000000".to_owned()),
+            None,
+            false,
+        )
+        .expect("create checked plaintext reader");
+        let mut reader = EtagReader::new_encrypted(plaintext, [1; 32], [2; 12]).expect("create protected ETag reader");
+        let err = reader
+            .read_to_end(&mut Vec::new())
+            .await
+            .expect_err("wrong plaintext MD5 must still be rejected");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
     #[tokio::test]
     async fn test_etag_reader_basic() {
         let data = b"hello world";

crates/rio/src/hash_reader.rs

@@ -136,6 +136,7 @@ pin_project! {
         content_sha256: Option<String>,
         content_sha256_hasher: Option<Sha256Hasher>,
         checksum_on_finish: bool,
+        pub encrypted_etag: bool,
 
         // Read only at EOF; see the timing contract in `crate::trailer`.
         trailer: Option<SharedTrailerSource>,
@@ -184,6 +185,7 @@ impl HashReader {
             content_sha256: sha256hex.clone(),
             content_sha256_hasher: sha256hex.map(|_| Sha256Hasher::new()),
             checksum_on_finish: false,
+            encrypted_etag: false,
             trailer: None,
         })
     }
@@ -224,6 +226,7 @@ impl HashReader {
             content_sha256: sha256hex.clone(),
             content_sha256_hasher: sha256hex.map(|_| Sha256Hasher::new()),
             checksum_on_finish: false,
+            encrypted_etag: false,
             trailer: None,
         })
     }
@@ -294,6 +297,7 @@ impl HashReader {
                 content_hash,
                 content_hasher,
                 checksum_on_finish: false,
+                encrypted_etag: false,
                 trailer: existing_hash_reader.get_trailer().cloned(),
             })
         } else {
@@ -322,6 +326,7 @@ impl HashReader {
                 content_sha256: sha256hex.clone(),
                 content_sha256_hasher: sha256hex.map(|_| Sha256Hasher::new()),
                 checksum_on_finish: false,
+                encrypted_etag: false,
                 trailer: None,
             })
         }

crates/scanner/src/scanner/tests/quota_reset_preservation.rs

@@ -96,7 +96,7 @@ async fn quota_reset_preservation_unknown_protocol_rejects_put_after_restart() {
         let (_directory, store, incarnation, path, bytes) = reservation_fixture().await;
         let mut quota = serde_json::json!({
             "quota_type": "Hard",
-            "reservation_protocol": 2,
+            "reservation_protocol": 3,
             "reservation_quota": 1024
         });
         match quota_shape {

crates/test-utils/src/ecstore_test_compat.rs

@@ -27,9 +27,10 @@ pub(crate) mod fixture {
     pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
     pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
     pub(crate) use rustfs_ecstore::api::object::{PutObjReader, SelectObjectSnapshot};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
     #[cfg(feature = "put-object-commit-barrier")]
     pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
-    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks, init_local_disks_with_instance_ctx};
     pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO};
 
     #[cfg(test)]

crates/test-utils/src/lib.rs

@@ -35,8 +35,9 @@ use std::sync::{Arc, Once};
 #[cfg(feature = "put-object-commit-barrier")]
 use ecstore_test_compat::fixture::ecstore_set_disk;
 use ecstore_test_compat::fixture::{
-    BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
-    PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
+    BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, MakeBucketOptions,
+    ObjectIO as _, PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
+    init_local_disks_with_instance_ctx,
 };
 use tokio_util::sync::CancellationToken;
 
@@ -90,7 +91,7 @@ pub struct TestECStoreEnv {
     pub temp_root: PathBuf,
     /// The per-disk directories (`disk1`..`diskN`) under `temp_root`.
     pub disk_paths: Vec<PathBuf>,
-    /// The store, bootstrapped exactly like the historical test setups:
+    /// By default, the store is bootstrapped exactly like the historical test setups:
     /// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps
     /// nextest's process-per-test parallelism safe).
     pub ecstore: Arc<ECStore>,
@@ -155,6 +156,7 @@ pub struct TestECStoreEnvBuilder {
     prefix: String,
     base_dir: Option<PathBuf>,
     init_bucket_metadata: bool,
+    instance_ctx: Option<Arc<InstanceContext>>,
 }
 
 impl Default for TestECStoreEnvBuilder {
@@ -164,6 +166,7 @@ impl Default for TestECStoreEnvBuilder {
             prefix: "rustfs_test_utils".to_string(),
             base_dir: None,
             init_bucket_metadata: true,
+            instance_ctx: None,
         }
     }
 }
@@ -196,6 +199,12 @@ impl TestECStoreEnvBuilder {
         self
     }
 
+    /// Use a caller-owned context for local disks and store construction.
+    pub fn instance_ctx(mut self, instance_ctx: Arc<InstanceContext>) -> Self {
+        self.instance_ctx = Some(instance_ctx);
+        self
+    }
+
     /// Build the environment. Panics on any bootstrap failure — this is test
     /// scaffolding, and a broken environment must fail the test loudly.
     pub async fn build(self) -> TestECStoreEnv {
@@ -237,14 +246,22 @@ impl TestECStoreEnvBuilder {
         };
         let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
 
-        init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
-
         // Port 0 keeps ECStore-backed integration binaries parallel-safe under
         // nextest: no fixed peer port is ever shared between test processes.
         let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr");
-        let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
-            .await
-            .expect("build test ECStore");
+        let ecstore = if let Some(instance_ctx) = self.instance_ctx {
+            init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+                .await
+                .expect("init instance local disks");
+            ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), CancellationToken::new(), instance_ctx)
+                .await
+                .expect("build instance test ECStore")
+        } else {
+            init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
+            ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
+                .await
+                .expect("build test ECStore")
+        };
 
         // The production bootstrap only persists pool.bin from the elected
         // first cluster node.  Test stores intentionally have no cluster
@@ -278,3 +295,86 @@ impl TestECStoreEnvBuilder {
         }
     }
 }
+
+#[cfg(test)]
+mod context_tests {
+    use super::{InstanceContext, TestECStoreEnv};
+    use crate::ecstore_test_compat::fixture::{read_config, save_config};
+    use std::sync::Arc;
+
+    #[tokio::test]
+    async fn explicit_context_keeps_stores_isolated() {
+        let root_a = tempfile::tempdir().expect("create context A disk root");
+        let root_b = tempfile::tempdir().expect("create context B disk root");
+        let ctx_a = Arc::new(InstanceContext::new());
+        let ctx_b = Arc::new(InstanceContext::new());
+        let env_a = TestECStoreEnv::builder()
+            .base_dir(root_a.path())
+            .init_bucket_metadata(false)
+            .instance_ctx(ctx_a.clone())
+            .build()
+            .await;
+        let env_b = TestECStoreEnv::builder()
+            .base_dir(root_b.path())
+            .init_bucket_metadata(false)
+            .instance_ctx(ctx_b.clone())
+            .build()
+            .await;
+
+        assert!(env_a.ecstore.instance_endpoints().is_none());
+        assert!(env_b.ecstore.instance_endpoints().is_none());
+        ctx_a.set_endpoints(env_a.endpoint_pools.clone());
+        let endpoints_a = env_a
+            .ecstore
+            .instance_endpoints()
+            .expect("instance A sees its caller-owned context topology");
+        assert_eq!(
+            endpoints_a.0[0]
+                .endpoints
+                .into_ref()
+                .first()
+                .expect("context A first endpoint")
+                .to_string(),
+            env_a.disk_paths[0].to_string_lossy(),
+        );
+        assert!(
+            env_b.ecstore.instance_endpoints().is_none(),
+            "publishing context A must not publish context B"
+        );
+        ctx_b.set_endpoints(env_b.endpoint_pools.clone());
+        let endpoints_b = env_b
+            .ecstore
+            .instance_endpoints()
+            .expect("instance B sees its caller-owned context topology");
+        assert_eq!(
+            endpoints_b.0[0]
+                .endpoints
+                .into_ref()
+                .first()
+                .expect("context B first endpoint")
+                .to_string(),
+            env_b.disk_paths[0].to_string_lossy(),
+        );
+        assert_eq!(
+            env_a.ecstore.instance_endpoints().expect("context A remains published").0[0]
+                .endpoints
+                .into_ref()
+                .first()
+                .expect("context A retained endpoint")
+                .to_string(),
+            env_a.disk_paths[0].to_string_lossy(),
+        );
+
+        let path = "config/test-context-isolation.json";
+        let bytes_a = br#"{"context":"a"}"#.to_vec();
+        let bytes_b = br#"{"context":"b"}"#.to_vec();
+        save_config(env_a.ecstore.clone(), path, bytes_a.clone())
+            .await
+            .expect("write context A config");
+        save_config(env_b.ecstore.clone(), path, bytes_b.clone())
+            .await
+            .expect("write context B config");
+        assert_eq!(read_config(env_a.ecstore.clone(), path).await.expect("read context A config"), bytes_a);
+        assert_eq!(read_config(env_b.ecstore.clone(), path).await.expect("read context B config"), bytes_b);
+    }
+}

crates/utils/src/http/object_encryption_keys.rs

@@ -55,11 +55,9 @@ pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal
 
 /// Reserved RustFS-branded twin of the MinIO-internal SSE key family.
 ///
-/// No RustFS writer emits these keys today — the SSE writer persists the
-/// MinIO-branded `X-Minio-Internal-Server-Side-Encryption-*` keys verbatim for
-/// interoperability — but redaction (`rustfs_filemeta`) and replication
-/// stripping treat the family as sensitive so that a future or third-party
-/// writer cannot leak sealed material through the reserved names.
+/// The SSE writer persists sealed material under the MinIO-branded names for
+/// interoperability. Redaction and replication stripping also cover this twin
+/// family, including the encrypted ETag format marker.
 pub const RUSTFS_INTERNAL_ENCRYPTION_PREFIX: &str = "x-rustfs-internal-server-side-encryption-";
 
 pub const REPLICATION_SSEC_ALGORITHM_HEADER: &str = "X-Rustfs-Replication-Ssec-Algorithm";
@@ -71,6 +69,15 @@ pub const REPLICATION_SSE_SEAL_ALGORITHM_HEADER: &str = "X-Rustfs-Replication-Se
 pub const REPLICATION_SSE_SEALED_KEY_HEADER: &str = "X-Rustfs-Replication-Server-Side-Encryption-Sealed-Key";
 pub const REPLICATION_ENCRYPTED_MULTIPART_HEADER: &str = "X-Rustfs-Replication-Encrypted-Multipart";
 
+pub const SUFFIX_OPAQUE_ENCRYPTED_ETAG: &str = "server-side-encryption-opaque-etag";
+
+/// Old encrypted objects can carry a plaintext MD5. Keyless listings expose
+/// an encrypted ETag only when its producer recorded the protected format.
+pub fn etag_can_be_listed(metadata: &HashMap<String, String>) -> bool {
+    !metadata.keys().any(|key| is_replication_stripped_encryption_key(key))
+        || super::metadata_compat::get_consistent_str(metadata, SUFFIX_OPAQUE_ENCRYPTED_ETAG) == Some("v1")
+}
+
 /// Stored SSE-C metadata keys and the wire names they replicate under.
 ///
 /// Source keys must match what `encryption_material_to_metadata` persists; the
@@ -307,6 +314,28 @@ pub fn stored_managed_encryption_key(metadata: &HashMap<String, String>) -> Opti
 mod tests {
     use super::*;
 
+    #[test]
+    fn keyless_listings_hide_unproven_encrypted_etags() {
+        assert!(etag_can_be_listed(&HashMap::from([("content-type".to_owned(), "text/plain".to_owned())])));
+        for key in [
+            SSEC_ALGORITHM_HEADER,
+            INTERNAL_ENCRYPTION_KEY_HEADER,
+            MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+            MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
+            "X-AMZ-SERVER-SIDE-ENCRYPTION",
+            "x-rustfs-internal-server-side-encryption-sealed-key",
+        ] {
+            let mut metadata = HashMap::from([(key.to_owned(), "encryption-metadata".to_owned())]);
+            assert!(!etag_can_be_listed(&metadata), "legacy encrypted metadata: {key}");
+            super::super::metadata_compat::insert_str(&mut metadata, SUFFIX_OPAQUE_ENCRYPTED_ETAG, "unknown".to_owned());
+            assert!(!etag_can_be_listed(&metadata), "unknown producer format: {key}");
+            super::super::metadata_compat::insert_str(&mut metadata, SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+            assert!(etag_can_be_listed(&metadata), "protected encrypted metadata: {key}");
+            metadata.insert("x-minio-internal-server-side-encryption-opaque-etag".to_owned(), "unknown".to_owned());
+            assert!(!etag_can_be_listed(&metadata), "conflicting format markers: {key}");
+        }
+    }
+
     #[test]
     fn ssec_transport_projection_retains_only_redacted_reader_headers() {
         let mut headers = http::HeaderMap::new();

crates/zip/Cargo.toml

@@ -49,6 +49,10 @@ rustfs-rio.workspace = true
 tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
 thiserror = { workspace = true }
 
+[target.'cfg(target_os = "macos")'.dependencies]
+# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
+liblzma = { version = "0.4.8", features = ["static"] }
+
 [dev-dependencies]
 rustfs-tokio-tar = { workspace = true }
 futures = { workspace = true }

docs/README.md

@@ -21,6 +21,9 @@ For bounded local read-backend startup and cancellation, see
 
 For legacy protection assessment and bounded protected copies, see [Object integrity inventory, audit, and migration](operations/shard-integrity-audit.md).
 
+For encrypted listing compatibility and older stored fingerprints, see
+[Encrypted object ETags](operations/encrypted-etags.md).
+
 Operational runbooks live under [`operations/`](operations/). Replication
 operators should start with:
 

docs/architecture/README.md

@@ -32,6 +32,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
 
 | Document | Use this when |
 |---|---|
+| [capture-checkpoint-contract.md](capture-checkpoint-contract.md) | changing Capture checkpoint capture, provenance, installation, recovery, or log cleanup |
 | [local-descriptor-cache.md](local-descriptor-cache.md) | changing descriptor cache keys/open modes or invalidation after local mutations |
 | [erasure-coding.md](erasure-coding.md) | changing anything under `crates/ecstore/src/erasure/`, `crates/filemeta/`, `crates/ecstore/src/set_disk/`, storage-class or layout code, or any decode, quorum, or heal boundary (normative spec) |
 | [placement-repair-invariants.md](placement-repair-invariants.md) | changing anything that resolves an object to a pool, set, or disk, or that admits scanner or heal work |

docs/architecture/capture-checkpoint-contract.md

@@ -0,0 +1,211 @@
+**Use this when:** changing Capture checkpoint capture, provenance, installation, recovery, or log cleanup.
+
+# Capture checkpoint contract
+
+This document defines the requirements for a recoverable Capture checkpoint.
+It is a design contract, not evidence that production capture, installation,
+purge, quorum integration, or S3 behavior implements these requirements.
+
+## Existing evidence and the missing boundary
+
+In `crates/capture-authority/src/application.rs`, `apply_created` inserts one
+canonical command and result on the first decision for an operation ID. An
+identical command returns that stored result; a different command returns
+`OperationConflict` without replacing the canonical record. The first result
+remains authoritative even when later state would produce a different result.
+
+`CreatedOperation` in `crates/capture-authority/src/created.rs` contains command
+bytes and a result. Only `CreatedResult::Created` carries `decision_log_id`.
+Negative results carry no first-decision ID. Heads, versions, events, and
+operations therefore cannot reconstruct the source of every first decision.
+For example, a stored `HeadMismatch` records an observed head, but does not
+prove that the head existed at the original decision cut. Later retries do
+not supply the missing historical evidence.
+
+`validate_application` in `crates/capture-authority/src/application.rs` derives
+first operation IDs from retained log/application records and, for native
+schema 4, replays decisions from the beginning using in-memory maps and sets.
+That intact-prefix validator is evidence for existing behavior; it is not a
+checkpoint-baseline validator or a bounded recovery algorithm.
+
+The `Row` comment and `StagingSink` in
+`crates/capture-authority/src/snapshot.rs` reserve operation provenance for
+caller validation and require isolated staging. Rows remain opaque to the
+framing decoder. A framing digest detects byte corruption; it does not establish
+semantic consistency, historical truth, or authority to install a checkpoint.
+
+## Canonical operation provenance
+
+Checkpoint-capable storage must use an explicitly versioned operation envelope
+containing the full first-decision `LogId` and the existing canonical operation
+bytes, including their stored result. Preserve those bytes and result; do not
+recompute a negative result against checkpoint-time heads or binding state.
+The envelope version is a storage capability boundary, distinct from the
+snapshot framing version. The existing CSNP v1 framing and header must not be
+silently extended. Header capability metadata requires a declared framing
+revision or an authenticated, versioned manifest whose consistency with the
+captured header is validated. Distinct storage and framing versions do not
+permit silently changing the v1 format.
+
+The first insertion must atomically record the envelope, application response,
+applied position, and any successful head/version/event changes in the same
+write transaction. Its first-decision ID must identify that insertion's log
+entry. An identical-command retry returns the original result and leaves both
+bytes and first-decision ID unchanged. A conflicting-command retry records its
+own application response as `OperationConflict`; it cannot overwrite the
+canonical envelope or claim to be the first decision.
+
+```mermaid
+flowchart TD
+    A[Committed command and operation ID] --> B{Canonical operation exists?}
+    B -->|No| C[Evaluate once at this log ID]
+    C --> D[Atomically persist first ID, original bytes and result]
+    B -->|Yes| E{Same canonical command bytes?}
+    E -->|Yes| F[Return original result; preserve first ID]
+    E -->|No| G[Return OperationConflict; preserve canonical record]
+```
+
+## Compatibility and promotion
+
+The existing schema 3 to schema 4 migration remains unchanged. Adding provenance
+requires a separately declared storage capability with explicit readers, writers,
+upgrade, and downgrade rules. A schema 4 label alone must not imply provenance
+support. Unknown envelope versions and stores missing required provenance must
+be rejected by checkpoint installation and purge authorization.
+
+An old schema 4 store may backfill first IDs only from its complete, intact,
+validated prefix: replay and validate the original command/results, identify the
+first canonical insertion for each operation, then atomically persist the
+capability and envelopes. A retry's ID or a successful event's ID cannot stand
+in for missing negative provenance. If the prefix has already been removed or
+cannot be validated, refuse promotion and cleanup; do not invent IDs. Readers
+without the new capability must refuse the upgraded format rather than silently
+interpret it as the old operation encoding. Retaining all history forever is
+not the checkpoint design: validated promotion supplies a recoverable baseline
+that permits bounded suffix retention.
+
+## Capture cut and resource ownership
+
+Capture must read the header, heads, versions, events, and operations through
+one owned read transaction. The view must retain the owning store/database
+handle for the entire transaction, cursor, and output lifetime; an owned
+transaction type alone does not prove database lifetime safety. Keep the store
+`Arc` alive so dropping another handle cannot invalidate the view. These are
+ownership requirements, not a claim about unverified library behavior.
+
+The header must describe that same committed, applied cut with its full `LogId`,
+full effective membership including its decision position and node metadata,
+cluster identity, binding identity/revision, and supported storage/envelope
+capabilities. Membership and binding must come from the captured transaction,
+not from a later live read. Never capture beyond the durable committed position.
+A four-table view without this cut and metadata is not independently installable.
+
+Emit rows incrementally in deterministic section/key order. Bound row size,
+section and total bytes, counts, temporary disk consumption, and concurrent
+capture admission. Use checked arithmetic. Do not collect the whole snapshot
+in RAM. Cancellation, disk exhaustion, or limit failure must leave the live
+store unchanged and release task-owned staging resources.
+
+## Isolated validation and trust
+
+Decode only into isolated disk staging. Check framing, limits, section order,
+unique keys, complete header metadata, capability versions, cluster/binding
+compatibility, canonical command/result encoding, envelope IDs at or before the
+cut, and all head/version/event/operation relationships. A successful operation
+must agree with its version, event, and first decision. Negative operations must
+have no success event or side effect attributable to that decision.
+
+These checks still cannot prove the historical predicate behind a negative
+result. Neither a digest nor a first ID proves that a `HeadMismatch`,
+`BindingMismatch`, or `VersionConflict` was justified when first decided.
+Remote installation therefore requires an explicitly authenticated and
+authorized Raft authority for the claimed cluster, cut, and membership, or a
+complete historical proof validated from a trusted baseline. Reject input that
+has neither. Authentication and authorization must precede publication and
+purge; a caller's assertion of trust is insufficient. This document does not
+claim that this remote Raft authentication boundary is implemented.
+
+Replay validation must use disk-backed staging/indexes and bounded buffers,
+including duplicate-operation and version/head tracking. Putting the input on
+disk while retaining replay maps proportional to history does not meet this
+requirement. The existing `validate_application` RAM replay cannot simply be
+reused to claim bounded checkpoint installation.
+
+## Atomic publication and recovery
+
+After isolated validation, publication must have one durable atomic point that
+selects the complete checkpoint generation, its application state, cut,
+membership, binding, capability, and recovery metadata. A pointer switch is
+acceptable only when all referenced bytes are already durable and recovery
+cannot observe a partial generation. Serialize publication with application so
+no committed suffix write is lost between validation and the switch.
+
+Preserve local node identity and persisted vote/hard state. Remote checkpoint
+metadata cannot overwrite either. Validate cluster identity, binding, full cut
+identity, and membership before replacing state. Reject rollback behind the
+local durable application/checkpoint floor; an equal index with a different
+term/leader identity is a conflict, not an equivalent cut. Retain valid local
+suffix and commit state consistently with the installed baseline, rather than
+resetting them from remote metadata.
+
+On reopen, select only a fully published generation and verify its recoverable
+baseline before replay. The suffix starts exactly at cut.index + 1, with checked
+index arithmetic; the complete cut ID must agree wherever the boundary is
+retained. Validate contiguous suffix indices, entry identities, committed and
+applied bounds, typed application responses, canonical retry behavior, and
+membership transitions relative to checkpoint state. Reject gaps, conflicting
+boundary IDs, stale membership, or a suffix belonging to another cluster or
+binding. Recovery must begin from checkpoint heads and canonical operations,
+not from empty state or the assumption that the retained log starts at zero.
+
+Purge is authorized only after the published checkpoint can be reopened and
+validated as a recovery baseline. Delete only entries/results covered by that
+exact cut, retaining everything needed for the continuous suffix and local
+Raft durability. Re-check the selected generation and cut immediately before
+cleanup. Persist a recoverable purge boundary so interrupted cleanup is
+idempotent. Never delete the only evidence before provenance promotion or
+checkpoint durability succeeds.
+
+```mermaid
+flowchart TD
+    A[One owned transaction and store lifetime] --> B[Stream captured cut to isolated disk staging]
+    B --> C[Validate framing, semantics, provenance and authorized source]
+    C --> D[Make complete generation durable]
+    D --> E[Single atomic publication point]
+    E --> F[Reopen baseline and validate continuous suffix]
+    F --> G[Authorize purge only through that exact cut]
+    G --> H[Idempotent cleanup; preserve recoverable baseline and suffix]
+    B -->|Failure| I[Discard staging; live generation remains usable]
+    C -->|Failure| I
+    D -->|Before publication crash| I
+```
+
+A crash before publication leaves the previous generation recoverable. A crash
+after publication must recover the new complete baseline, including when purge
+has not begun or is only partly complete. Orphan staging is never selected as
+live state. Old generations may be reclaimed only after no active view needs
+them and the selected generation remains recoverable.
+
+## Behavioral verification requirements
+
+The following are required behavior checks, not reports of tests run or gates
+passed. Existing symbols above anchor the semantics; future checkpoint paths
+must demonstrate the additional requirements directly.
+
+| Behavior | Required observation |
+|---|---|
+| First negative result | Original command/result and first full ID survive capture, install, and reopen; later state does not re-evaluate it. |
+| Identical and conflicting retries | `apply_created` semantics hold across the checkpoint boundary: original result/ID retained; conflict cannot overwrite the canonical record. |
+| Old schema 4 promotion | An intact prefix validates and supplies first IDs; missing/corrupt prefix refuses promotion and purge; schema 3 to 4 behavior stays unchanged. |
+| Snapshot cut under concurrent writes | Header, membership, binding, and all rows describe one transaction cut; later writes remain in the suffix. |
+| View lifetime | Dropping other store handles during row emission cannot invalidate the retained transaction or database. |
+| Framing versus authority | Valid digest with forged negative history is insufficient; unauthenticated or unauthorized remote input is rejected. |
+| Cross-record corruption | Altered command, first ID, event, version, head, membership, or binding fails isolated validation before publication. |
+| Recovery boundary | Exact cut plus continuous suffix recovers; gaps, conflicting full IDs, cross-cluster input, and rollback are rejected. |
+| Local Raft state | Installation preserves local identity, vote/hard state, and compatible durable suffix/commit information. |
+| Resource exhaustion | Oversize rows, arithmetic overflow, disk exhaustion, cancellation, and large histories fail safely within declared memory/disk bounds. |
+| Crash publication | Crashes before and after the atomic point select a complete old or new generation, never a mixture. |
+| Interrupted purge | Reopen succeeds before cleanup is authorized; each interruption resumes without deleting the recoverable baseline or required suffix. |
+
+Documentation path and whitespace guards only validate document hygiene. They
+cannot establish these production behaviors or the remote trust boundary.

docs/architecture/global-state-crate-split-plan.md

@@ -33,7 +33,7 @@ The guard pins the production files allowed to reference `rustfs_ecstore::api::g
 
 - `rustfs/src/storage/storage_api.rs`
 
-That boundary keeps only bootstrap writes and lifecycle controls (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`). Read-only runtime getters are exported through `rustfs_ecstore::api::runtime` and consumed through the local storage facade. A new direct use either moves behind an existing owner-local boundary or updates this plan and the guard in the same reviewed change.
+That boundary keeps only bootstrap writes and lifecycle controls (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`, `wait_for_detached_mutations`). Read-only runtime getters are exported through `rustfs_ecstore::api::runtime` and consumed through the local storage facade. A new direct use either moves behind an existing owner-local boundary or updates this plan and the guard in the same reviewed change.
 
 ## Fallback Removal Plan
 

docs/architecture/global-state-inventory.md

@@ -27,7 +27,7 @@ Runtime ownership handles that exist today. Reads go through `rustfs_ecstore::ap
 | `global_tier_config_mgr` | Tier config manager | Reads and reloads stay behind this helper. |
 | `bucket_monitor` | Replication bandwidth monitor | Replication pool/stat handles are projected into RustFS wrapper types at the storage boundary. |
 | `global_lock_client`, `global_lock_clients` | `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS` (`crates/ecstore/src/runtime/global.rs`) | Preserve lock quorum and client selection; the process-level `GLOBAL_LOCK_MANAGER` stays separate. |
-| `boot_time`, `deployment_id`, `region`, `rustfs_port` | `GLOBAL_BOOT_TIME`, deployment id, region, and port state (`crates/ecstore/src/runtime/global.rs`) | Scalar writes remain behind the `api::global` setters (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`). |
+| `boot_time`, `deployment_id`, `region`, `rustfs_port` | `GLOBAL_BOOT_TIME`, deployment id, region, and port state (`crates/ecstore/src/runtime/global.rs`) | Scalar writes remain behind the `api::global` setters (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`, `wait_for_detached_mutations`). |
 
 Owner-helper handles outside the runtime-source list stay inside their owner and are reached through owner functions: `GLOBAL_EVENT_NOTIFIER` (`crates/ecstore/src/runtime/global.rs`); `GLOBAL_NOTIFICATION_SYS`, `EVENT_DISPATCH_HOOK`, `GLOBAL_PROCESSORS`, `INTERNODE_DATA_TRANSPORT`, `GLOBAL_BUCKET_TARGET_SYS`, `GLOBAL_CONFIG_SYS`, `GLOBAL_STORAGE_CLASS`, `WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER` (ECStore owner modules); `GLOBAL_SERVER_CONFIG` (`crates/config/src/server_config.rs`); `GLOBAL_HEAL_RUNTIME`, `GLOBAL_AHM_SERVICES_CANCEL_TOKEN` (`crates/heal/src/lib.rs`); `GLOBAL_KMS_SERVICE_MANAGER` (`crates/kms/src/service_manager.rs`); `GLOBAL_CAPACITY_MANAGER` (`crates/object-capacity/src/capacity_manager.rs`); `APP_CONTEXT_SINGLETON` (`rustfs/src/app/context/global.rs`).