fix(heal): fence format repair during pool transitions

* fix(scanner): defer usage publication during pool recovery

* fix(scanner): preserve metrics when publication is deferred

* fix(scanner): route test types through storage boundary

* fix(scanner): keep cache floor deferred during movement

.github/workflows/package.yml

@@ -189,6 +189,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - arch: x86_64
@@ -510,15 +511,13 @@ jobs:
 
           CHECKSUM_DIR="$(mktemp -d)"
           gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
-            -D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
+            -D "$CHECKSUM_DIR" --clobber
 
           for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
             asset="${spec%%:*}"
             checksum_cmd="${spec##*:}"
             checksum_file="${CHECKSUM_DIR}/${asset}"
 
-            touch "$checksum_file"
-
             for f in "$DEB_FILE" "$RPM_FILE"; do
               if [[ -n "$f" && -f "$f" ]]; then
                 base="$(basename "$f")"
@@ -531,7 +530,8 @@ jobs:
                 grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
                 grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
                 mv "${checksum_file}.tmp2" "$checksum_file"
-                (cd "$(dirname "$f")" && "$checksum_cmd" -- "$github_base") >> "$checksum_file"
+                digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
+                printf '%s  %s\n' "$digest" "$github_base" >> "$checksum_file"
               fi
             done
 

AGENTS.md

@@ -127,8 +127,9 @@ the broadest gate. Inspect only the final task-owned diff, classify it by
 behavioral impact rather than line count or path alone, and run the smallest
 set of checks that provides meaningful coverage. Do not let unrelated
 worktree changes or a generic contributor checklist expand the scope.
-Non-exempt changes must also pass Adversarial Validation (next section) before
-the checks below count as completion.
+For non-exempt changes, complete the applicable multi-role adversarial review
+before running `make pre-pr` (or an equivalent full gate). Resolve or rebut
+every finding first, then run the gate against the reviewed final diff.
 
 ### Validation floor
 
@@ -166,8 +167,9 @@ the checks below count as completion.
    dependency set is identifiable, validate those packages and known
    dependents instead of the whole workspace. Use `make pre-commit` only when
    a repository-wide fast gate adds useful confidence beyond those checks.
-4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage
-   cannot bound the impact, including:
+4. **Broad or high-risk change:** After the applicable adversarial review has
+   completed, run `make pre-pr` only when targeted coverage cannot bound the
+   impact, including:
    - dependency, feature, build-script, procedural-macro, code-generation,
      toolchain, or CI changes that alter compilation or the test matrix;
    - cross-crate public APIs, shared foundational code, or broad refactors with
@@ -287,8 +289,9 @@ High risk: all seven roles.
 - Every testable behavior change has a focused regression check. Exceptions
   follow the validation floor and state why a check is impractical and what
   risk remains.
-- The Verification Before PR gates pass — adversarial review supplements
-  those gates, never replaces them.
+- After the applicable adversarial review has completed, the Verification
+  Before PR gates pass; adversarial review supplements those gates, never
+  replaces them.
 - High risk only: record a one-line verdict per role in the PR description.
 
 ## Git and PR Baseline

CONTRIBUTING.md

@@ -91,8 +91,9 @@ A green `make pre-commit` is not enough to open a pull request.
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
-tests). Run `make pre-pr` before opening or updating a pull request — this is
-what CI enforces.
+tests). Complete the applicable multi-role adversarial review described in
+`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
+updating a pull request. This is what CI enforces.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -150,8 +151,9 @@ Example output when formatting fails:
 2. **Format your code**: `make fmt` or `cargo fmt --all`
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
-5. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
-6. **Push to your branch**: `git push`
+5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
+6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration
 

Cargo.lock

@@ -4757,9 +4757,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.17"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f877e75f39e9827ec50a572dd592684ac28c029578726c85f1b2aa6ab807449"
+checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228"
 dependencies = [
  "atomic-waker",
  "bytes",

crates/common/src/metrics.rs

@@ -729,7 +729,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
         return 0;
     }
 
-    u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
+    u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
 }
 
 #[derive(Clone, Copy, Debug, Default)]
@@ -781,6 +781,19 @@ struct ScannerBucketDriveResultValue {
     last_seen: u64,
 }
 
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+struct ScannerActiveBucketDriveKey {
+    source: String,
+    bucket: String,
+    drive: String,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct ScannerActiveBucketDriveValue {
+    count: u64,
+    started_at: Timestamp,
+}
+
 // ---------------------------------------------------------------------------
 // Metrics
 // ---------------------------------------------------------------------------
@@ -813,6 +826,7 @@ pub struct Metrics {
     scanner_set_scans_active: AtomicU64,
     scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
     scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
+    scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
     scanner_bucket_drive_result_clock: AtomicU64,
     current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
     last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
@@ -1045,6 +1059,15 @@ pub struct ScannerBucketDriveResultSnapshot {
     pub count: u64,
 }
 
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScannerActiveBucketDriveSnapshot {
+    pub source: String,
+    pub bucket: String,
+    pub drive: String,
+    pub count: u64,
+    pub age_seconds: u64,
+}
+
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct ScannerReplicationRepairSnapshot {
     pub source: String,
@@ -1387,6 +1410,8 @@ pub struct ScannerRuntimeDetailsReport {
     pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
     #[serde(default)]
     pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
+    #[serde(default)]
+    pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
 }
 
 impl CurrentCycle {
@@ -1746,7 +1771,7 @@ pub fn emit_scan_cycle_deferred(duration: Duration) {
     metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
 }
 
-pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
+pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
     let result = if success { "success" } else { "error" };
     global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
     metrics::counter!(
@@ -1764,7 +1789,7 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
     .record(duration.as_secs_f64());
 }
 
-pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
+pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
     global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
     metrics::counter!(
         OTEL_SCANNER_BUCKETS_SCANNED,
@@ -1817,6 +1842,7 @@ impl Metrics {
             scanner_set_scans_active: AtomicU64::new(0),
             scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
             scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
+            scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
             scanner_bucket_drive_result_clock: AtomicU64::new(0),
             current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
             last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
@@ -2308,8 +2334,45 @@ impl Metrics {
         }
     }
 
-    pub fn record_scan_bucket_drive_start(&self) {
+    pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
         self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
+        if bucket.is_empty() || drive.is_empty() {
+            return;
+        }
+        let key = ScannerActiveBucketDriveKey {
+            source: source.as_str().to_string(),
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        };
+        let mut active = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        active
+            .entry(key)
+            .and_modify(|value| value.count = value.count.saturating_add(1))
+            .or_insert(ScannerActiveBucketDriveValue {
+                count: 1,
+                started_at: Timestamp::now(),
+            });
+    }
+
+    pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
+        let key = ScannerActiveBucketDriveKey {
+            source: source.as_str().to_string(),
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        };
+        let mut active = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(value) = active.get_mut(&key) {
+            value.count = value.count.saturating_sub(1);
+            if value.count == 0 {
+                active.remove(&key);
+            }
+        }
     }
 
     pub fn record_scan_bucket_drive_failure(&self) {
@@ -2782,6 +2845,26 @@ impl Metrics {
         } else {
             Vec::new()
         };
+        let now = Timestamp::now();
+        let mut active_bucket_drive_scans = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .iter()
+            .map(|(key, value)| ScannerActiveBucketDriveSnapshot {
+                source: key.source.clone(),
+                bucket: key.bucket.clone(),
+                drive: key.drive.clone(),
+                count: value.count,
+                age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
+            })
+            .collect::<Vec<_>>();
+        active_bucket_drive_scans.sort_by(|left, right| {
+            left.source
+                .cmp(&right.source)
+                .then_with(|| left.bucket.cmp(&right.bucket))
+                .then_with(|| left.drive.cmp(&right.drive))
+        });
         ScannerRuntimeDetailsReport {
             disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
             bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
@@ -2791,6 +2874,7 @@ impl Metrics {
                 .lock()
                 .unwrap_or_else(|poisoned| poisoned.into_inner())
                 .clone(),
+            active_bucket_drive_scans,
         }
     }
 
@@ -4371,7 +4455,7 @@ mod tests {
     #[tokio::test]
     async fn report_includes_bucket_drive_scan_starts() {
         let metrics = Metrics::new();
-        metrics.record_scan_bucket_drive_start();
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
         metrics.record_scan_bucket_drive_failure();
 
         let report = metrics.report().await;
@@ -4380,6 +4464,27 @@ mod tests {
         assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
     }
 
+    #[tokio::test]
+    async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
+        let metrics = Metrics::new();
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
+        assert_eq!(active.len(), 1);
+        assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
+        assert_eq!(active[0].bucket, "bucket-a");
+        assert_eq!(active[0].drive, "/mnt/data/1");
+        assert_eq!(active[0].count, 2);
+
+        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
+        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
+
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
+        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
+    }
+
     #[tokio::test]
     async fn report_includes_structured_bucket_drive_results() {
         let metrics = Metrics::new();

crates/config/README.md

@@ -115,6 +115,15 @@ Current guidance:
   - enables KMS readiness enforcement for `/health/ready`.
   - default is `false`.
 
+## Object lock admission environment variables
+
+- `RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS`
+  - experimental same-object PUT commit namespace-lock admission budget.
+  - default is `0`, which disables this override and keeps `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` behavior.
+  - when set, only `put_object_commit` write-lock acquisition is bounded by this millisecond budget; other namespace lock users keep the global object-lock timeout.
+  - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
+  - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
+
 ## Drive timeout environment variables
 
 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`

crates/config/src/constants/object.rs

@@ -427,6 +427,19 @@ pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TI
 /// Default lock acquisition timeout: 5 seconds.
 pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
 
+/// Environment variable for the experimental PUT commit namespace lock acquire timeout in milliseconds.
+///
+/// A value of `0` disables the experiment and keeps
+/// `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` as the timeout. This only bounds the
+/// `put_object_commit` namespace write-lock wait and is intended for #925
+/// tail-drain admission experiments.
+///
+/// Default: 0 milliseconds (disabled).
+pub const ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: &str = "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS";
+
+/// Default: PUT commit namespace lock acquire timeout override is disabled.
+pub const DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: u64 = 0;
+
 /// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
 ///
 /// This timeout bounds the internode RPC call itself. It is intentionally

crates/e2e_test/src/common.rs

@@ -53,7 +53,8 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
 const RUSTFS_FULL_FEATURE: &str = "full";
 const TEST_PORT_MIN: u16 = 20_000;
-const TEST_PORT_RANGE: u16 = 40_000;
+// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
+const TEST_PORT_RANGE: u16 = 10_000;
 const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
 const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
 const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -1028,20 +1028,6 @@ impl<'a> ReaderPathExpectation<'a> {
         }
     }
 
-    fn with_size_bucket(
-        object: ReaderObject<'a>,
-        expected_path: &'a str,
-        object_class: &'a str,
-        expected_size_bucket: &'a str,
-    ) -> Self {
-        Self {
-            object,
-            expected_path,
-            object_class,
-            expected_size_bucket: Some(expected_size_bucket),
-        }
-    }
-
     fn with_any_size_bucket(object: ReaderObject<'a>, expected_path: &'a str, object_class: &'a str) -> Self {
         Self {
             object,
@@ -1909,12 +1895,7 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
     assert_reader_path(
         &collector,
         &client,
-        ReaderPathExpectation::with_size_bucket(
-            ReaderObject::new(bucket, key, &body, put.e_tag(), None),
-            LEGACY_DUPLEX,
-            COMPRESSED,
-            size_bucket(4 * KIB),
-        ),
+        ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
     )
     .await?;
 
@@ -2274,6 +2255,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
     hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
     hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
     hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1");
+    hot.set_env("RUSTFS_TRANSITION_QUEUE_SEND_TIMEOUT_MS", "1");
     hot.start().await?;
 
     let hot_client = hot.create_s3_client(0)?;
@@ -2290,7 +2272,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
             .put_object()
             .bucket(&bucket)
             .key(key)
-            .body(ByteStream::from(payload(64 * KIB, index)))
+            .body(ByteStream::from(payload(1024 * KIB, index)))
             .send()
             .await?;
     }

crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs

@@ -39,6 +39,7 @@ use std::time::Duration;
 use tracing::info;
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+type S3OperationResult<T> = Result<T, Box<aws_sdk_s3::Error>>;
 
 const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
 const OTHER_KEY: &str = "kms-matrix-other-key";
@@ -130,7 +131,7 @@ fn policy_document(statements: Vec<serde_json::Value>) -> String {
     serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
 }
 
-async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
+async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> S3OperationResult<()> {
     client
         .put_object()
         .bucket(BUCKET)
@@ -141,16 +142,23 @@ async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(),
         .send()
         .await
         .map(|_| ())
-        .map_err(aws_sdk_s3::Error::from)
+        .map_err(|error| Box::new(aws_sdk_s3::Error::from(error)))
 }
 
 /// Assert the operation failed with `AccessDenied` rather than any other error.
 ///
 /// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
 /// would hide both a leak of key state and an outage masquerading as a denial.
-fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
+fn assert_access_denied<T: std::fmt::Debug, E: std::fmt::Debug + std::borrow::Borrow<aws_sdk_s3::Error>>(
+    result: Result<T, E>,
+    what: &str,
+) {
     let error = result.expect_err(&format!("{what} must be denied"));
-    assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
+    assert_eq!(
+        error.borrow().code(),
+        Some("AccessDenied"),
+        "{what} must fail with AccessDenied: {error:?}"
+    );
 }
 
 /// Retry an SSE-KMS write until the identity's policy has reached the request path.
@@ -296,7 +304,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
             .send()
             .await
             .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
+            .map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
         "SSE-KMS read by an identity holding no kms grant",
     );
 
@@ -310,7 +318,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
             .send()
             .await
             .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
+            .map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
         "SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
     );
 

crates/e2e_test/src/protocols/webdav_core.rs

@@ -233,6 +233,111 @@ pub async fn test_webdav_core_operations() -> Result<()> {
         );
         info!("PASS: PUT file '{}' successful", filename);
 
+        // Regression for #6260: a bucket-scoped policy must be able to discover its bucket at the
+        // WebDAV root without the unrelated global ListAllMyBuckets permission.
+        let scoped_bucket = "webdav-scoped-bucket";
+        let scoped_file = "visible.txt";
+        let scoped_user = "webdav-scoped-user";
+        let scoped_secret = "webdav-scoped-secret";
+        let scoped_policy_name = "webdav-scoped-policy";
+
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"MKCOL").unwrap(), format!("{}/{}", base_url, scoped_bucket))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 201, "scoped test bucket should be created");
+
+        let resp = client
+            .put(format!("{}/{}/{}", base_url, scoped_bucket, scoped_file))
+            .header("Authorization", &auth_header)
+            .body("visible to the scoped principal")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 201, "scoped test object should be created");
+
+        admin_create_user(&admin_base_url, scoped_user, scoped_secret).await?;
+        admin_add_canned_policy(
+            &admin_base_url,
+            scoped_policy_name,
+            &serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ]
+                    },
+                    {
+                        "Effect": "Deny",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ],
+                        "Condition": { "Bool": { "aws:SecureTransport": "true" } }
+                    },
+                    {
+                        "Effect": "Deny",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ],
+                        "Condition": { "StringEquals": { "s3:signatureversion": "AWS4-HMAC-SHA256" } }
+                    }
+                ]
+            }),
+        )
+        .await?;
+        admin_attach_policy_to_user(&admin_base_url, scoped_policy_name, scoped_user).await?;
+
+        let scoped_auth = basic_auth_header_for(scoped_user, scoped_secret);
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", &scoped_auth)
+            .header("Depth", "1")
+            .header("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 207, "bucket-scoped root PROPFIND should succeed");
+        let root_listing = resp.text().await?;
+        assert!(root_listing.contains(scoped_bucket), "the authorized bucket should be listed");
+        assert!(!root_listing.contains(bucket_name), "an unauthorized bucket must not be listed");
+
+        let resp = client
+            .request(
+                reqwest::Method::from_bytes(b"PROPFIND").unwrap(),
+                format!("{}/{}", base_url, scoped_bucket),
+            )
+            .header("Authorization", &scoped_auth)
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 207, "authorized bucket PROPFIND should succeed");
+        assert!(resp.text().await?.contains(scoped_file), "the authorized object should be listed");
+
+        let denied_user = "webdav-no-buckets-user";
+        let denied_secret = "webdav-no-buckets-secret";
+        admin_create_user(&admin_base_url, denied_user, denied_secret).await?;
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", basic_auth_header_for(denied_user, denied_secret))
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert_eq!(
+            resp.status().as_u16(),
+            207,
+            "PROPFIND keeps the root resource visible when the directory listing is forbidden"
+        );
+        let denied_body = resp.text().await?;
+        assert!(!denied_body.contains(scoped_bucket), "a denied response must not leak the scoped bucket");
+        assert!(!denied_body.contains(bucket_name), "a denied response must not leak the admin bucket");
+
         // Test GET (download file)
         info!("Testing WebDAV: GET (download file '{}')", filename);
         let resp = client

crates/e2e_test/src/quota_test.rs

@@ -169,6 +169,42 @@ impl QuotaTestEnv {
         bucket: &str,
         quota_bytes: u64,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.wait_for_quota_usage_for(bucket).await?;
+
+        let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
+        let quota_config = serde_json::json!({
+            "quota": quota_bytes,
+            "quota_type": "HARD"
+        })
+        .to_string();
+        let readiness = async {
+            loop {
+                let (status, response) = admin_request(
+                    &self.env.url,
+                    Method::PUT,
+                    &quota_path,
+                    Some(quota_config.clone()),
+                    &self.env.access_key,
+                    &self.env.secret_key,
+                )
+                .await?;
+                if status.is_success() {
+                    return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
+                }
+                if status != StatusCode::SERVICE_UNAVAILABLE {
+                    return Err(format!("failed to set quota for {bucket}: {status} {response}").into());
+                }
+
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        match timeout(Duration::from_secs(30), readiness).await {
+            Ok(result) => result,
+            Err(_) => Err(format!("quota readiness did not converge for {bucket} within 30 seconds").into()),
+        }
+    }
+
+    pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
         let readiness = async {
             loop {
@@ -181,28 +217,12 @@ impl QuotaTestEnv {
                 if status != StatusCode::SERVICE_UNAVAILABLE {
                     return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
                 }
-
                 sleep(Duration::from_secs(1)).await;
             }
         };
         match timeout(Duration::from_secs(30), readiness).await {
-            Ok(result) => result?,
-            Err(_) => {
-                return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
-            }
-        }
-
-        let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
-        let quota_config = serde_json::json!({
-            "quota": quota_bytes,
-            "quota_type": "HARD"
-        });
-
-        let response = awscurl_put(&url, &quota_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
-        if response.contains("error") {
-            Err(format!("Failed to set quota: {}", response).into())
-        } else {
-            Ok(())
+            Ok(result) => result,
+            Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
         }
     }
 
@@ -614,6 +634,7 @@ mod integration_tests {
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
+        env.wait_for_quota_usage_for(&env.bucket_name).await?;
 
         // Test 1: GET quota for bucket without quota config
         let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
@@ -621,12 +642,7 @@ mod integration_tests {
         assert!(response.contains("quota") && response.contains("null"));
 
         // Test 2: PUT quota - valid config
-        let quota_config = serde_json::json!({
-            "quota": 1048576,
-            "quota_type": "HARD"
-        });
-        let response = awscurl_put(&url, &quota_config.to_string(), &env.env.access_key, &env.env.secret_key).await?;
-        assert!(response.contains("success") || !response.contains("error"));
+        env.set_bucket_quota(1048576).await?;
 
         // Test 3: GET quota after setting
         let response = awscurl_get(&url, &env.env.access_key, &env.env.secret_key).await?;

crates/e2e_test/src/reliant/tiering.rs

@@ -110,6 +110,7 @@ const USER_META_KEY: &str = "ilm7-origin";
 const USER_META_VAL: &str = "hermetic-transition";
 const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
 const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
+const TIER_MUTATION_RECOVERY_CHANGED: &str = "Remote tier mutation recovery changed before publish";
 
 /// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
 /// internal part boundary sits at this offset.
@@ -183,19 +184,39 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
     })
     .to_string();
 
-    let (status, resp) = signed_admin_request(
-        &hot.url,
-        Method::PUT,
-        "/rustfs/admin/v3/tier",
-        Some(&body),
-        &hot.access_key,
-        &hot.secret_key,
-    )
-    .await?;
-    if !status.is_success() {
-        return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
+    let verify_path = format!("/rustfs/admin/v3/tier/{TIER_NAME}");
+    let deadline = Instant::now() + StdDuration::from_secs(30);
+    let mut recovery_changed = false;
+    loop {
+        if recovery_changed {
+            let (status, _) =
+                signed_admin_request(&hot.url, Method::GET, &verify_path, None, &hot.access_key, &hot.secret_key).await?;
+            if status.is_success() {
+                return Ok(());
+            }
+        }
+        let (status, resp) = signed_admin_request(
+            &hot.url,
+            Method::PUT,
+            "/rustfs/admin/v3/tier",
+            Some(&body),
+            &hot.access_key,
+            &hot.secret_key,
+        )
+        .await?;
+        if status.is_success() {
+            return Ok(());
+        }
+        if resp.contains(TIER_MUTATION_RECOVERY_CHANGED) {
+            recovery_changed = true;
+        } else if !recovery_changed || !resp.contains("TierNameAlreadyExist") {
+            return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
+        }
+        tokio::time::sleep(StdDuration::from_millis(100)).await;
     }
-    Ok(())
 }
 
 async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
@@ -207,10 +228,12 @@ async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
         if status.is_success() {
             return Ok(());
         }
-        if !resp.contains("TierNameBackendInUse") || Instant::now() >= deadline {
+        if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
+            || Instant::now() >= deadline
+        {
             return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
         }
-        // AddTier cleanup is asynchronous; wait until its committed mutation fence clears.
+        // Tier mutation cleanup and startup recovery are asynchronous.
         tokio::time::sleep(StdDuration::from_millis(100)).await;
     }
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -90,6 +90,12 @@ use uuid::Uuid;
 const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
 const REDACTED_CREDENTIAL: &str = "<redacted>";
 
+pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
+pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
+pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
+pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
+pub type DeleteObjectTaggingSdkError = Box<SdkError<DeleteObjectTaggingError>>;
+
 pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
 
 fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
@@ -1968,7 +1974,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    ) -> Result<HeadObjectOutput, HeadObjectSdkError> {
         // Announce the replication check so a RustFS target returns SSE-C
         // object metadata (etag/size) without the customer key the replication
         // worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
@@ -1981,8 +1987,7 @@ impl TargetClient {
         // object with an identical ETag, and the worker concludes the object
         // already converged — so it never actually replicates it.
         insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
-        match self
-            .client
+        self.client
             .head_object()
             .bucket(bucket)
             .key(object)
@@ -1999,10 +2004,7 @@ impl TargetClient {
             })
             .send()
             .await
-        {
-            Ok(res) => Ok(res),
-            Err(e) => Err(e),
-        }
+            .map_err(Box::new)
     }
 
     /// HEAD used by the read-proxy path (GET/HEAD of an object not yet
@@ -2023,7 +2025,7 @@ impl TargetClient {
         range: Option<String>,
         part_number: Option<i32>,
         extra_headers: HeaderMap,
-    ) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    ) -> Result<HeadObjectOutput, HeadObjectSdkError> {
         let headers = proxy_outbound_headers(extra_headers);
         self.client
             .head_object()
@@ -2036,6 +2038,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
@@ -2051,7 +2054,7 @@ impl TargetClient {
         range: Option<String>,
         part_number: Option<i32>,
         extra_headers: HeaderMap,
-    ) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
+    ) -> Result<GetObjectOutput, GetObjectSdkError> {
         let headers = proxy_outbound_headers(extra_headers);
         self.client
             .get_object()
@@ -2064,6 +2067,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// GetObjectTagging for the tagging read-proxy path
@@ -2073,7 +2077,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
+    ) -> Result<GetObjectTaggingOutput, GetObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .get_object_tagging()
@@ -2084,6 +2088,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// PutObjectTagging for the tagging proxy path
@@ -2094,7 +2099,7 @@ impl TargetClient {
         object: &str,
         version_id: Option<String>,
         tagging: SdkTagging,
-    ) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
+    ) -> Result<PutObjectTaggingOutput, PutObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .put_object_tagging()
@@ -2106,6 +2111,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// DeleteObjectTagging for the tagging proxy path
@@ -2115,7 +2121,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
+    ) -> Result<DeleteObjectTaggingOutput, DeleteObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .delete_object_tagging()
@@ -2126,6 +2132,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// On success returns the version id the target assigned (from

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -2180,7 +2180,7 @@ pub async fn recover_manual_transition_jobs_once(
     if limit == 0 {
         return Err(Error::other("manual transition job recovery limit must be greater than zero"));
     }
-    let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
+    let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
     let page = api
         .clone()
         .list_objects_v2(
@@ -2386,7 +2386,7 @@ async fn replay_manual_transition_pending_tasks(
             version_id: task.version_id,
             etag: task.etag,
             mod_time,
-            size: task.size.map_or(0, |size| size),
+            size: task.size.unwrap_or(0),
             is_latest: task.is_latest.unwrap_or(false),
             ..Default::default()
         };

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -1016,7 +1016,7 @@ pub async fn recover_transition_transaction_records(
         return Err(Error::other("transition transaction recovery limit must be greater than zero"));
     }
 
-    let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
+    let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
     let list = api
         .clone()
         .list_objects_v2(

crates/ecstore/src/bucket/migration.rs

@@ -41,6 +41,7 @@ const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
 const IAM_USERS_PREFIX: &str = "config/iam/users/";
 const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
 const IAM_STS_PREFIX: &str = "config/iam/sts/";
+const MINIO_GO_ZERO_TIME: OffsetDateTime = time::macros::datetime!(0001-01-01 00:00 UTC);
 const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
 const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
 const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
@@ -120,6 +121,15 @@ fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Opt
     if is_identity_path(path) {
         let mut identity: UserIdentity =
             serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
+        if (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX))
+            && identity
+                .credentials
+                .expiration
+                .as_ref()
+                .is_some_and(|expiration| *expiration == MINIO_GO_ZERO_TIME || *expiration == OffsetDateTime::UNIX_EPOCH)
+        {
+            identity.credentials.expiration = None;
+        }
         if identity.update_at.is_none() {
             identity.update_at = Some(OffsetDateTime::now_utc());
         }
@@ -441,7 +451,10 @@ mod tests {
     use crate::bucket::replication::{
         BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
     };
+    use rustfs_policy::auth::UserIdentity;
     use std::collections::HashMap;
+    use time::OffsetDateTime;
+    use time::format_description::well_known::Rfc3339;
 
     #[test]
     fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
@@ -493,6 +506,54 @@ mod tests {
         assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
     }
 
+    #[test]
+    fn test_normalize_minio_permanent_credential_expiration() {
+        let cases = [
+            ("config/iam/users/alice/identity.json", "0001-01-01T00:00:00Z", true),
+            ("config/iam/users/alice/identity.json", "1970-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "0001-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00.000000001Z", false),
+            ("config/iam/sts/temp/identity.json", "0001-01-01T00:00:00Z", false),
+            ("config/iam/sts/temp/identity.json", "1970-01-01T00:00:00Z", false),
+            ("config/iam/users/alice/identity.json", "1969-12-31T23:59:59Z", false),
+            ("config/iam/users/alice/identity.json", "1970-01-01T00:00:00.000000001Z", false),
+            ("config/iam/users/alice/identity.json", "0001-01-01T00:00:00.000000001Z", false),
+            ("config/iam/users/alice/identity.json", "2030-01-01T00:00:00Z", false),
+        ];
+
+        for (path, expiration, should_clear) in cases {
+            let input = serde_json::json!({
+                "version": 1,
+                "credentials": {
+                    "accessKey": "test-access",
+                    "secretKey": "test-secret",
+                    "sessionToken": "test-session-token",
+                    "parentUser": "test-parent",
+                    "expiration": expiration,
+                }
+            });
+            let output = normalize_iam_config_blob(path, &serde_json::to_vec(&input).expect("serialize identity fixture"))
+                .expect("normalize should succeed")
+                .expect("identity path should be supported");
+            let identity: UserIdentity = serde_json::from_slice(&output).expect("deserialize normalized identity");
+
+            assert_eq!(identity.credentials.access_key, "test-access");
+            assert_eq!(identity.credentials.secret_key, "test-secret");
+            assert_eq!(identity.credentials.session_token, "test-session-token");
+            assert_eq!(identity.credentials.parent_user, "test-parent");
+            if should_clear {
+                assert_eq!(identity.credentials.expiration, None, "path: {path}, expiration: {expiration}");
+            } else {
+                assert_eq!(
+                    identity.credentials.expiration,
+                    Some(OffsetDateTime::parse(expiration, &Rfc3339).expect("parse expected expiration")),
+                    "path: {path}, expiration: {expiration}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn test_normalize_bucket_meta_blob_resync_reencode() {
         let path = ".buckets/test/.replication/resync.bin";

crates/ecstore/src/bucket/quota/checker.rs

@@ -76,7 +76,12 @@ impl QuotaChecker {
 
         let current_usage = self.get_real_time_usage(bucket).await?;
 
-        let admission_size = if uses_durable_reservations { 0 } else { operation_size };
+        // The reporting path projects this operation; storage mutations reserve it at commit.
+        let admission_size = if uses_durable_reservations && !force_usage_calculation {
+            0
+        } else {
+            operation_size
+        };
         let expected_usage = match operation {
             QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
                 current_usage.saturating_add(admission_size)

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -52,8 +52,8 @@ use super::replication_storage_boundary::{
     ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
 };
 use super::replication_target_boundary::{
-    ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore,
-    SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
+    ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
+    ReplicationTargetStore, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
     replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
     replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
     replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
@@ -214,7 +214,7 @@ async fn head_object_for_worker(
     target_bucket: &str,
     object: &str,
     version_id: Option<String>,
-) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+) -> std::result::Result<HeadObjectOutput, HeadObjectSdkError> {
     target_client.head_object(target_bucket, object, version_id).await
 }
 
@@ -233,7 +233,7 @@ async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetCli
 async fn head_object_fallback(
     tgt_client: &TargetClient,
     object: &str,
-) -> std::result::Result<Option<HeadObjectOutput>, SdkError<HeadObjectError>> {
+) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
     match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
         Ok(oi) => Ok(Some(oi)),
         Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
@@ -1152,11 +1152,11 @@ fn spawn_resync_walk_task<S: ReplicationStorage>(
 /// updating the per-object status counters and returning the accounted size
 /// together with any verification error.
 async fn verify_resync_head_result(
-    head_result: std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>>,
+    head_result: std::result::Result<HeadObjectOutput, HeadObjectSdkError>,
     roi: &ReplicateObjectInfo,
     st: &mut TargetReplicationResyncStatus,
     target_client: &Arc<TargetClient>,
-) -> (i64, Option<SdkError<HeadObjectError>>) {
+) -> (i64, Option<HeadObjectSdkError>) {
     match head_result {
         Ok(_) => {
             st.replicated_count += 1;
@@ -1275,7 +1275,7 @@ async fn resync_worker_process_object<S: ReplicationStorage>(
             "Processed resync object"
         );
     }
-    st.error = err.as_ref().and_then(resync_target_error_detail);
+    st.error = err.as_ref().and_then(|err| resync_target_error_detail(err.as_ref()));
 
     st
 }
@@ -2467,7 +2467,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             Ok(_) => {}
             Err(e) => {
                 let non_retryable = matches!(
-                    &e,
+                    e.as_ref(),
                     SdkError::ServiceError(service_err)
                         if is_retryable_delete_replication_head_error(
                             service_err.err().is_not_found(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -36,7 +36,8 @@ use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 
 pub(crate) use crate::bucket::bucket_target_sys::{
-    AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
+    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
+    resolve_read_api_version_id,
 };
 #[cfg(test)]
 pub(crate) use crate::bucket::target::BucketTarget;

crates/ecstore/src/core/pools.rs

@@ -94,6 +94,9 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
 const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
 const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
 const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
+/// Background decommission walks must tolerate slow object migrations; the
+/// stall timeout is the drive-health bound, not the total listing duration.
+const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
 
 pub const POOL_META_NAME: &str = "pool.bin";
 pub const POOL_META_FORMAT: u16 = 1;
@@ -1672,7 +1675,7 @@ impl PoolMeta {
         self.load_no_lock(pool).await
     }
 
-    async fn load_no_lock<S>(&mut self, pool: Arc<S>) -> Result<()>
+    pub(crate) async fn load_no_lock<S>(&mut self, pool: Arc<S>) -> Result<()>
     where
         S: EcstoreObjectIO,
     {
@@ -5047,6 +5050,8 @@ impl SetDisks {
                 path: bucket_info.prefix.clone(),
                 recursive: true,
                 min_disks: listing_quorum,
+                skip_walkdir_total_timeout: true,
+                walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
                 agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
                     let resolver = resolver.clone();

crates/ecstore/src/disk/os.rs

@@ -23,7 +23,7 @@ use std::{
     io,
     path::{Component, Path, PathBuf},
     sync::{Arc, LazyLock, Weak},
-    time::Instant,
+    time::{Duration, Instant},
 };
 use tokio::fs;
 use tokio::sync::{
@@ -328,6 +328,9 @@ const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR
 const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
 const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
 const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
+const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
+const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
+const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
 #[cfg(not(test))]
 const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
 #[cfg(test)]
@@ -354,6 +357,16 @@ static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
 static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
     rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
 });
+fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
+    Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
+}
+
+static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
+    file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
+        ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
+        DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
+    ))
+});
 
 #[cfg(test)]
 mod dst_dir_fsync_group_commit_override {
@@ -402,6 +415,7 @@ mod file_fdatasync_group_commit_override {
     use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
 
     static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
+    static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
     static SERIAL: Mutex<()> = Mutex::new(());
 
     pub(crate) fn get() -> Option<bool> {
@@ -415,6 +429,7 @@ mod file_fdatasync_group_commit_override {
     impl Drop for OverrideGuard {
         fn drop(&mut self) {
             *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
+            *WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
         }
     }
 
@@ -423,6 +438,14 @@ mod file_fdatasync_group_commit_override {
         *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
         OverrideGuard { _serial: serial }
     }
+
+    pub(crate) fn set_wait_micros(wait_micros: u64) {
+        *WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = Some(wait_micros);
+    }
+
+    pub(crate) fn wait_micros() -> Option<u64> {
+        *WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
+    }
 }
 
 #[cfg(test)]
@@ -430,6 +453,11 @@ pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fd
     file_fdatasync_group_commit_override::set(enabled)
 }
 
+#[cfg(test)]
+fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
+    file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
+}
+
 fn file_fdatasync_group_commit_enabled() -> bool {
     #[cfg(test)]
     if let Some(enabled) = file_fdatasync_group_commit_override::get() {
@@ -439,6 +467,15 @@ fn file_fdatasync_group_commit_enabled() -> bool {
     *FILE_FDATASYNC_GROUP_COMMIT_ENABLED
 }
 
+fn file_fdatasync_group_commit_wait() -> Duration {
+    #[cfg(test)]
+    if let Some(wait_micros) = file_fdatasync_group_commit_override::wait_micros() {
+        return file_fdatasync_group_commit_wait_duration(wait_micros);
+    }
+
+    *FILE_FDATASYNC_GROUP_COMMIT_WAIT
+}
+
 #[derive(Clone, Eq, Hash, PartialEq)]
 struct DstDirFsyncGroupKey {
     canonical_path: PathBuf,
@@ -934,6 +971,10 @@ async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
         #[cfg(test)]
         file_sync_probe::run_before_group_batch();
         tokio::task::yield_now().await;
+        let wait = file_fdatasync_group_commit_wait();
+        if !wait.is_zero() {
+            tokio::time::sleep(wait).await;
+        }
         let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
             let mut group_state = group.inner.lock();
             let batch_file_count = group_state.pending_files;
@@ -6075,6 +6116,7 @@ mod tests {
         use std::sync::mpsc;
 
         let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        set_file_fdatasync_group_commit_wait_for_test(0);
         clear_file_fdatasync_group_commit_for_test();
         let temp_dir = tempdir().expect("create temp dir");
         let first_dir = temp_dir.path().join("first");
@@ -6141,12 +6183,105 @@ mod tests {
         assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
     }
 
+    #[test]
+    fn file_fdatasync_group_commit_wait_duration_uses_default_and_cap() {
+        assert_eq!(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS, 0);
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
+            Duration::ZERO
+        );
+        assert_eq!(file_fdatasync_group_commit_wait_duration(250), Duration::from_micros(250));
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
+            Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
+        );
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(u64::MAX),
+            Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial(file_sync_probe)]
+    async fn file_fdatasync_group_commit_wait_budget_batches_late_follower() {
+        use std::sync::mpsc;
+
+        let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        let wait_budget_micros = 1_000;
+        let wait_budget = file_fdatasync_group_commit_wait_duration(wait_budget_micros);
+        set_file_fdatasync_group_commit_wait_for_test(wait_budget_micros);
+        clear_file_fdatasync_group_commit_for_test();
+        let temp_dir = tempdir().expect("create temp dir");
+        let first_dir = temp_dir.path().join("first");
+        let second_dir = temp_dir.path().join("second");
+        std::fs::create_dir(&first_dir).expect("create first dir");
+        std::fs::create_dir(&second_dir).expect("create second dir");
+        std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
+        std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
+        let _probe = file_sync_probe::set_blocking(temp_dir.path());
+        let (entered_tx, entered_rx) = mpsc::channel();
+        file_sync_probe::set_before_group_batch(move || {
+            entered_tx.send(()).expect("signal first file fdatasync group worker");
+        });
+
+        let limiter = file_sync_limiter();
+        let first_limiter = limiter.clone();
+        let first_path = first_dir.clone();
+        let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
+        tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
+            .await
+            .expect("group worker hook waiter should run")
+            .expect("first file fdatasync group worker should start");
+
+        let second_limiter = limiter.clone();
+        let second_path = second_dir.clone();
+        let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                if file_fdatasync_group_commit_counts_for_test().1 == 2 {
+                    return;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("second waiter should enqueue during the configured wait budget");
+        tokio::time::advance(wait_budget).await;
+        tokio::task::yield_now().await;
+        file_sync_probe::wait_for_active(1).await;
+
+        assert_eq!(
+            file_sync_probe::group_batches(),
+            vec![2],
+            "configured wait budget should let a follower join the leader's batch"
+        );
+        file_sync_probe::release();
+        first
+            .await
+            .expect("join first wait-budget file sync")
+            .expect("first wait-budget file sync must succeed");
+        second
+            .await
+            .expect("join second wait-budget file sync")
+            .expect("second wait-budget file sync must succeed");
+        assert!(
+            fsync_dir_recorder::was_fsynced(&first_dir),
+            "first source directory must still be fsynced"
+        );
+        assert!(
+            fsync_dir_recorder::was_fsynced(&second_dir),
+            "second source directory must still be fsynced"
+        );
+        assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial_test::serial(file_sync_probe)]
     async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
         use std::sync::mpsc;
 
         let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        set_file_fdatasync_group_commit_wait_for_test(0);
         clear_file_fdatasync_group_commit_for_test();
         let temp_dir = tempdir().expect("create temp dir");
         let first_dir = temp_dir.path().join("first");

crates/ecstore/src/object_api/types.rs

@@ -719,14 +719,23 @@ impl ObjectInfo {
     }
 
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
-        let name = decode_dir_object(object);
-
         let mut version_id = fi.version_id;
 
         if versioned && version_id.is_none() {
             version_id = Some(Uuid::nil())
         }
 
+        Self::from_file_info_with_version_id(fi, bucket, object, version_id)
+    }
+
+    pub(crate) fn from_file_info_with_version_id(
+        fi: &FileInfo,
+        bucket: &str,
+        object: &str,
+        version_id: Option<Uuid>,
+    ) -> ObjectInfo {
+        let name = decode_dir_object(object);
+
         // etag
         let (content_type, content_encoding, etag) = {
             let content_type = fi.metadata.get("content-type").cloned();
@@ -1640,6 +1649,18 @@ mod tests {
         assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
     }
 
+    #[test]
+    fn from_file_info_with_version_id_keeps_normalized_absent_version() {
+        let fi = FileInfo {
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        let info = ObjectInfo::from_file_info_with_version_id(&fi, "bucket", "object", None);
+
+        assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
+    }
+
     #[test]
     fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
         for legacy_label in [

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -657,7 +657,7 @@ where
             prefix,
             marker,
             None,
-            i32::try_from(limit).map_or(i32::MAX, |value| value),
+            i32::try_from(limit).unwrap_or(i32::MAX),
             false,
             None,
             false,

crates/ecstore/src/set_disk/mod.rs

@@ -1501,6 +1501,102 @@ pub fn get_lock_acquire_timeout() -> Duration {
     }
 }
 
+fn get_put_object_commit_lock_acquire_timeout_override_ms() -> u64 {
+    #[cfg(test)]
+    {
+        rustfs_utils::get_env_u64(
+            rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+            rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+        )
+    }
+    #[cfg(not(test))]
+    {
+        static CACHED: OnceLock<u64> = OnceLock::new();
+        *CACHED.get_or_init(|| {
+            rustfs_utils::get_env_u64(
+                rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+                rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+            )
+        })
+    }
+}
+
+fn get_put_object_commit_lock_acquire_timeout(op: &'static str) -> Duration {
+    let default_timeout = get_lock_acquire_timeout();
+    if op != "put_object_commit" {
+        return default_timeout;
+    }
+
+    let timeout_ms = get_put_object_commit_lock_acquire_timeout_override_ms();
+    if timeout_ms == 0 {
+        default_timeout
+    } else {
+        Duration::from_millis(timeout_ms)
+    }
+}
+
+fn put_object_commit_lock_timeout_override_enabled(op: &'static str) -> bool {
+    op == "put_object_commit" && get_put_object_commit_lock_acquire_timeout_override_ms() != 0
+}
+
+fn put_object_commit_lock_admission_budget_label() -> &'static str {
+    match get_put_object_commit_lock_acquire_timeout_override_ms() {
+        0 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+        1..=250 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+        251..=500 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+        501..=1000 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
+        _ => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
+    }
+}
+
+fn record_put_object_commit_lock_admission(op: &'static str, outcome: &'static str) {
+    if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
+        return;
+    }
+    rustfs_io_metrics::record_put_object_commit_lock_admission(put_object_commit_lock_admission_budget_label(), outcome);
+}
+
+fn put_object_commit_lock_acquire_error_outcome(op: &'static str, err: &rustfs_lock::error::LockError) -> &'static str {
+    if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
+        rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
+    } else {
+        rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+    }
+}
+
+fn resolve_put_object_commit_lock_acquire_result(
+    set: &SetDisks,
+    op: &'static str,
+    bucket: &str,
+    object: &str,
+    result: std::result::Result<rustfs_lock::namespace::NamespaceLockGuard, rustfs_lock::error::LockError>,
+) -> Result<rustfs_lock::namespace::NamespaceLockGuard> {
+    match result {
+        Ok(guard) => {
+            record_put_object_commit_lock_admission(op, rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED);
+            Ok(guard)
+        }
+        Err(err) => {
+            record_put_object_commit_lock_admission(op, put_object_commit_lock_acquire_error_outcome(op, &err));
+            Err(map_put_object_commit_lock_acquire_error(set, op, bucket, object, err))
+        }
+    }
+}
+
+fn map_put_object_commit_lock_acquire_error(
+    set: &SetDisks,
+    op: &'static str,
+    bucket: &str,
+    object: &str,
+    err: rustfs_lock::error::LockError,
+) -> StorageError {
+    if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
+        StorageError::SlowDown
+    } else {
+        set.map_namespace_lock_error(bucket, object, "write", err)
+    }
+}
+
 pub fn is_object_lock_diag_enabled() -> bool {
     *OBJECT_LOCK_DIAG_ENABLED.get_or_init(|| {
         let enabled = rustfs_utils::get_env_bool(
@@ -3302,10 +3398,14 @@ impl SetDisks {
         let diag_enabled = is_object_lock_diag_enabled();
         let ns_lock = self.new_ns_lock(bucket, object).await?;
         let acquire_start = Instant::now();
-        let guard = ns_lock
-            .get_write_lock(get_lock_acquire_timeout())
-            .await
-            .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
+        let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
+        let guard = resolve_put_object_commit_lock_acquire_result(
+            self,
+            op,
+            bucket,
+            object,
+            ns_lock.get_write_lock(acquire_timeout).await,
+        )?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
         let owner = diag_enabled.then(|| ns_lock.owner().to_string());
         self.log_object_lock_acquire_if_slow(
@@ -3340,20 +3440,26 @@ impl SetDisks {
         let diag_enabled = is_object_lock_diag_enabled();
         let ns_lock = self.new_ns_lock(bucket, object).await?;
         let acquire_start = Instant::now();
-        let acquire = ns_lock.get_write_lock(get_lock_acquire_timeout());
+        let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
+        let acquire = ns_lock.get_write_lock(acquire_timeout);
         tokio::pin!(acquire);
         let mut on_pending = Some(on_pending);
-        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
-            std::task::Poll::Pending => {
-                if let Some(on_pending) = on_pending.take() {
-                    on_pending();
+        let guard = resolve_put_object_commit_lock_acquire_result(
+            self,
+            op,
+            bucket,
+            object,
+            futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+                std::task::Poll::Pending => {
+                    if let Some(on_pending) = on_pending.take() {
+                        on_pending();
+                    }
+                    std::task::Poll::Pending
                 }
-                std::task::Poll::Pending
-            }
-            std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
-        })
-        .await
-        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
+                std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
+            })
+            .await,
+        )?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
         let owner = diag_enabled.then(|| ns_lock.owner().to_string());
         self.log_object_lock_acquire_if_slow(
@@ -5717,8 +5823,8 @@ mod tests {
             .filter(|(composite, _, _, _)| {
                 composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
                     && composite.key().labels().any(|label| {
-                        label.key().to_string() == "stage"
-                            && label.value().to_string() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
+                        label.key() == "stage"
+                            && label.value() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
                     })
             })
             .map(|(_, _, _, value)| match value {
@@ -5728,6 +5834,81 @@ mod tests {
             .sum()
     }
 
+    fn put_object_commit_lock_admission_count(
+        rows: &[(
+            metrics_util::CompositeKey,
+            Option<metrics::Unit>,
+            Option<metrics::SharedString>,
+            DebugValue,
+        )],
+        budget: &'static str,
+        outcome: &'static str,
+    ) -> u64 {
+        rows.iter()
+            .filter(|(composite, _, _, _)| {
+                composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "budget" && label.value() == budget)
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "outcome" && label.value() == outcome)
+            })
+            .map(|(_, _, _, value)| match value {
+                DebugValue::Counter(count) => *count,
+                _ => 0,
+            })
+            .sum()
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_budget_labels_are_bounded() {
+        let cases = [
+            ("0", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED),
+            ("250", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS),
+            ("251", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
+            ("500", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
+            ("501", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
+            ("1000", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
+            ("1001", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS),
+        ];
+        for (timeout_ms, expected) in cases {
+            temp_env::with_vars(
+                [(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some(timeout_ms))],
+                || {
+                    assert_eq!(put_object_commit_lock_admission_budget_label(), expected);
+                },
+            );
+        }
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_error_outcomes_are_bounded() {
+        let timeout = LockError::timeout("bucket/object", Duration::from_millis(1));
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("put_object_commit", &timeout),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
+            );
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("complete_multipart_upload_commit", &timeout),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+            );
+        });
+
+        let internal = LockError::internal("simulated lock manager error");
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("put_object_commit", &internal),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+            );
+        });
+    }
+
     #[test]
     #[serial]
     fn put_object_commit_namespace_lock_wait_metric_is_wired_to_both_write_lock_paths() {
@@ -5793,6 +5974,289 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_timeout_override_only_applies_to_put_commit() {
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("17"))], || {
+            assert_eq!(get_put_object_commit_lock_acquire_timeout("put_object_commit"), Duration::from_millis(17));
+            assert_eq!(
+                get_put_object_commit_lock_acquire_timeout("complete_multipart_upload_commit"),
+                get_lock_acquire_timeout()
+            );
+        });
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
+            assert_eq!(
+                get_put_object_commit_lock_acquire_timeout("put_object_commit"),
+                get_lock_acquire_timeout()
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_timeout_override_bounds_contention_wait() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            runtime.block_on(async {
+                let ctx = Arc::new(InstanceContext::new());
+                ctx.update_erasure_type(SetupType::Erasure).await;
+                let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                let bucket = "bucket";
+                let object = "object";
+
+                let held_guard = set
+                    .acquire_write_lock_diag("put_object_commit", bucket, object)
+                    .await
+                    .expect("holder acquire should succeed");
+                let started = Instant::now();
+                let err = match set.acquire_write_lock_diag("put_object_commit", bucket, object).await {
+                    Ok(_) => panic!("contended PUT commit lock should honor the short timeout"),
+                    Err(err) => err,
+                };
+                assert!(
+                    started.elapsed() < Duration::from_secs(1),
+                    "short PUT commit lock timeout should not wait for the global timeout"
+                );
+                assert!(matches!(err, StorageError::SlowDown));
+
+                drop(held_guard);
+                set.acquire_write_lock_diag("put_object_commit", bucket, object)
+                    .await
+                    .expect("permit should not leak after timeout");
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_acquired_and_timeout() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let held_guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("holder acquire should succeed");
+                    let err = match set.acquire_write_lock_diag("put_object_commit", "bucket", "object").await {
+                        Ok(_) => panic!("contended PUT commit acquire should return SlowDown"),
+                        Err(err) => err,
+                    };
+                    assert!(matches!(err, StorageError::SlowDown));
+                    drop(held_guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                1
+            );
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+                ),
+                1
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_disabled_budget_acquired() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("PUT commit acquire should succeed with default timeout");
+                    drop(guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                1
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_skips_non_put_commit_ops() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let guard = set
+                        .acquire_write_lock_diag("complete_multipart_upload_commit", "bucket", "object")
+                        .await
+                        .expect("non-PUT commit acquire should succeed");
+                    drop(guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                rows.iter()
+                    .filter(|(composite, _, _, _)| {
+                        composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+                    })
+                    .count(),
+                0
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_lock_error() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let healthy: Arc<dyn LockClient> =
+                        Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+                    let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::DistErasure).await;
+                    let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
+                    assert!(
+                        set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                            .await
+                            .is_err(),
+                        "one healthy locker must not satisfy the PUT commit write quorum"
+                    );
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
+                ),
+                1
+            );
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+                ),
+                0
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_pending_hook_acquired() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("500"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let held_guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("holder acquire should succeed");
+                    let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
+                    let pending_acquire =
+                        set.acquire_write_lock_diag_with_pending_hook("put_object_commit", "bucket", "object", move || {
+                            let _ = pending_tx.send(());
+                        });
+                    let release_holder = async {
+                        pending_rx.await.expect("pending hook should fire");
+                        drop(held_guard);
+                    };
+                    let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
+                    drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                2
+            );
+        });
+    }
+
     #[tokio::test]
     async fn new_ns_lock_shares_clients_without_changing_quorum() {
         let healthy: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));

crates/ecstore/src/set_disk/ops/heal_walk.rs

@@ -124,14 +124,7 @@ impl HealWalkCollector {
         for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
             let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
             let lifecycle_object_info = if self.include_lifecycle_object_info {
-                let mut lifecycle_fi = fi.clone();
-                lifecycle_fi.version_id = version_uuid;
-                Some(ObjectInfo::from_file_info(
-                    &lifecycle_fi,
-                    &self.bucket,
-                    &entry.name,
-                    version_uuid.is_some(),
-                ))
+                Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
             } else {
                 None
             };
@@ -198,14 +191,7 @@ impl HealWalkCollector {
                 let vid = version_uuid.map(|u| u.to_string());
                 if seen.insert(vid.clone()) {
                     let lifecycle_object_info = if self.include_lifecycle_object_info {
-                        let mut lifecycle_fi = fi.clone();
-                        lifecycle_fi.version_id = version_uuid;
-                        Some(ObjectInfo::from_file_info(
-                            &lifecycle_fi,
-                            &self.bucket,
-                            &entry.name,
-                            version_uuid.is_some(),
-                        ))
+                        Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
                     } else {
                         None
                     };

crates/ecstore/src/set_disk/ops/object.rs

@@ -1322,7 +1322,12 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let object_info = prepared_object_info
             .unwrap_or_else(|| build_get_object_info(fi, bucket, object, opts.versioned || opts.version_suspended));
         let object_class = classify_get_codec_streaming_object_class(&range, &object_info, fi);
-        let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size);
+        let metrics_size = if stage_metrics_enabled {
+            object_info.get_actual_size().unwrap_or(object_info.size)
+        } else {
+            object_info.size
+        };
+        let size_bucket = rustfs_io_metrics::get_object_size_bucket(metrics_size);
         record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start);
         let metadata_elapsed = metadata_stage_start.elapsed().as_secs_f64();
         rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_elapsed);
@@ -3766,7 +3771,7 @@ pub(crate) async fn complete_transition_upload<Remote, Producer>(
     producer: Producer,
     expected_size: u64,
     consumed: Arc<AtomicU64>,
-) -> std::result::Result<TransitionUploadCompletion, TransitionUploadFailure>
+) -> std::result::Result<TransitionUploadCompletion, Box<TransitionUploadFailure>>
 where
     Remote: Future<Output = std::result::Result<String, std::io::Error>>,
     Producer: Future<Output = Result<u64>>,
@@ -3784,23 +3789,23 @@ where
                 Err(_) => StorageError::Unexpected,
                 Ok(Ok(_)) => StorageError::Io(remote_error),
             };
-            return Err(TransitionUploadFailure { error, candidate: None });
+            return Err(Box::new(TransitionUploadFailure { error, candidate: None }));
         }
     };
     let candidate = TransitionUploadCandidate::from_put_response(remote_version);
     let produced = match producer_result {
         Ok(Ok(produced)) => produced,
         Ok(Err(error)) => {
-            return Err(TransitionUploadFailure {
+            return Err(Box::new(TransitionUploadFailure {
                 error,
                 candidate: Some(candidate),
-            });
+            }));
         }
         Err(_) => {
-            return Err(TransitionUploadFailure {
+            return Err(Box::new(TransitionUploadFailure {
                 error: StorageError::Unexpected,
                 candidate: Some(candidate),
-            });
+            }));
         }
     };
     let consumed = consumed.load(Ordering::Acquire);
@@ -3810,10 +3815,10 @@ where
         } else {
             StorageError::MoreData
         };
-        return Err(TransitionUploadFailure {
+        return Err(Box::new(TransitionUploadFailure {
             error,
             candidate: Some(candidate),
-        });
+        }));
     }
     Ok(TransitionUploadCompletion {
         candidate,
@@ -7284,7 +7289,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
             let gr = gr?;
             let reader = BufReader::new(gr.stream);
-            let hash_reader = HashReader::from_stream(reader, gr.object_info.size, gr.object_info.size, None, None, false)?;
+            let hash_reader = HashReader::from_stream(reader, gr.object_info.size, oi.get_actual_size()?, None, None, false)?;
             let mut p_reader = PutObjReader::new(hash_reader);
             return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
                 Ok(restored_info) => {
@@ -8826,7 +8831,7 @@ mod transition_commit_failure_tests {
     use s3s::dto::RestoreRequest;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-    fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
+    pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
         let mut metadata = HashMap::new();
         rustfs_utils::http::metadata_compat::insert_str(
             &mut metadata,
@@ -8836,7 +8841,7 @@ mod transition_commit_failure_tests {
         metadata
     }
 
-    fn restore_metadata(operation_id: Uuid, ongoing: bool) -> HashMap<String, String> {
+    pub(super) fn restore_metadata(operation_id: Uuid, ongoing: bool) -> HashMap<String, String> {
         let mut metadata = restore_operation_id_metadata(operation_id);
         metadata.insert(s3s::header::X_AMZ_RESTORE.as_str().to_string(), format!("ongoing-request=\"{ongoing}\""));
         metadata
@@ -10097,6 +10102,51 @@ mod transition_commit_failure_tests {
             .await
             .expect("operation B should replace operation A before final commit");
 
+        let mismatch = set_disks
+            .finalize_restore_metadata(
+                bucket,
+                object,
+                &set_disks
+                    .get_object_info(bucket, object, &ObjectOptions::default())
+                    .await
+                    .expect("operation B metadata should be readable"),
+                &ObjectOptions {
+                    user_defined: restore_operation_id_metadata(operation_a),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("operation A must not finalize operation B metadata");
+        assert!(matches!(
+            mismatch,
+            Error::Io(ref error)
+                if error.kind() == std::io::ErrorKind::Other
+                    && error.to_string() == "restore operation id changed before metadata finalization"
+        ));
+        let current = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("operation B metadata should remain after mismatched finalization");
+        assert_eq!(
+            rustfs_utils::http::metadata_compat::get_consistent_str(
+                current.user_defined.as_ref(),
+                rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
+            ),
+            Some(operation_b.to_string().as_str()),
+            "mismatched finalization must not remove operation B"
+        );
+        assert!(
+            parse_restore_obj_status(
+                current
+                    .user_defined
+                    .get(s3s::header::X_AMZ_RESTORE.as_str())
+                    .expect("operation B restore header should remain pending"),
+            )
+            .expect("operation B restore header should parse")
+            .on_going(),
+            "mismatched finalization must not publish restore completion"
+        );
+
         let mut stale_restore_reader = PutObjReader::from_vec(b"stale A restored body".repeat(1024));
         let result = set_disks
             .put_object(
@@ -10126,18 +10176,37 @@ mod transition_commit_failure_tests {
 
         let mut matching_restore_reader = PutObjReader::from_vec(b"matching B restored body".repeat(1024));
         let operation_b_restore_metadata = restore_metadata(operation_b, false);
-        set_disks
+        let restored = set_disks
             .put_object(
                 bucket,
                 object,
                 &mut matching_restore_reader,
                 &ObjectOptions {
-                    user_defined: operation_b_restore_metadata,
+                    user_defined: operation_b_restore_metadata.clone(),
                     ..Default::default()
                 },
             )
             .await
             .expect("matching operation B should be allowed to commit");
+        set_disks
+            .finalize_restore_metadata(
+                bucket,
+                object,
+                &restored,
+                &ObjectOptions {
+                    user_defined: restore_operation_id_metadata(operation_b),
+                    transition: TransitionOptions {
+                        restore_request: RestoreRequest {
+                            days: Some(1),
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("matching operation B should finalize after its commit consumes the operation id");
         let restored = set_disks
             .get_object_info(bucket, object, &ObjectOptions::default())
             .await
@@ -10503,13 +10572,16 @@ mod transition_commit_failure_tests {
 #[cfg(all(test, feature = "test-util"))]
 mod transition_upload_integrity_tests {
     use super::hermetic_set_disks_support::{hermetic_set_disks, hermetic_set_disks_with_lockers};
+    use super::transition_commit_failure_tests::{restore_metadata, restore_operation_id_metadata};
     use super::*;
     use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
     use crate::disk::DiskAPI as _;
     use crate::layout::endpoints::SetupType;
     use crate::services::tier::test_util::register_mock_tier;
+    use crate::set_disk::replication::RestoreFinalizeBarrier;
     use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
     use http::HeaderMap;
+    use rustfs_filemeta::RestoreStatusOps as _;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats};
     use std::collections::HashSet;
@@ -10655,6 +10727,162 @@ mod transition_upload_integrity_tests {
         }
     }
 
+    async fn write_committed_restore(
+        set_disks: &Arc<SetDisks>,
+        disk_stores: &[DiskStore],
+        bucket: &str,
+        object: &str,
+        operation_id: Uuid,
+    ) -> ObjectInfo {
+        for disk in disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+        let mut source = PutObjReader::from_vec(b"restore source body".repeat(1024));
+        set_disks
+            .put_object(bucket, object, &mut source, &ObjectOptions::default())
+            .await
+            .expect("source object should be written");
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    eval_metadata: Some(restore_metadata(operation_id, true)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("pending restore metadata should be installed");
+
+        let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(1024));
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut restored_reader,
+                &ObjectOptions {
+                    user_defined: restore_metadata(operation_id, true),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("matching restore commit should consume its operation id")
+    }
+
+    fn restore_finalize_options(operation_id: Uuid) -> ObjectOptions {
+        ObjectOptions {
+            user_defined: restore_operation_id_metadata(operation_id),
+            transition: TransitionOptions {
+                restore_request: s3s::dto::RestoreRequest {
+                    days: Some(1),
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
+        let current = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("pending restore metadata should remain readable");
+        assert!(
+            restore_operation_id_from_metadata(current.user_defined.as_ref())
+                .expect("operation id metadata should parse")
+                .is_none(),
+            "successful restore commit must have consumed the operation id"
+        );
+        assert!(
+            rustfs_filemeta::parse_restore_obj_status(
+                current
+                    .user_defined
+                    .get(s3s::header::X_AMZ_RESTORE.as_str())
+                    .expect("pending restore header should remain"),
+            )
+            .expect("restore header should parse")
+            .on_going(),
+            "failed finalization must not publish completion metadata"
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial]
+    async fn restore_finalize_rejects_acquired_lock_loss_after_commit() {
+        let refresh_calls = Arc::new(AtomicUsize::new(0));
+        let lockers: Vec<Arc<dyn LockClient>> = (0..4)
+            .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
+            .collect();
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+        let bucket = "restore-finalize-acquired-lock-lost-bucket";
+        let object = "object.bin";
+        let operation_id = Uuid::new_v4();
+        let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+        let barrier = RestoreFinalizeBarrier::install(bucket, object);
+        let finalize_set = Arc::clone(&set_disks);
+        let finalize = tokio::spawn(async move {
+            finalize_set
+                .finalize_restore_metadata(bucket, object, &restored, &restore_finalize_options(operation_id))
+                .await
+        });
+        barrier.wait_until_paused().await;
+        tokio::time::advance(Duration::from_secs(11)).await;
+        tokio::task::yield_now().await;
+        assert!(refresh_calls.load(Ordering::SeqCst) > 0, "restore finalization lock must attempt renewal");
+        barrier.release();
+
+        let error = finalize
+            .await
+            .expect("restore finalization task should join")
+            .expect_err("lost acquired lock must reject restore finalization");
+        assert!(matches!(
+            error,
+            Error::Io(ref error)
+                if error.kind() == std::io::ErrorKind::Other
+                    && error.to_string() == "restore finalization lock lost before metadata update"
+        ));
+        assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn restore_finalize_rejects_outer_fence_loss_after_metadata_read() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "restore-finalize-outer-fence-lost-bucket";
+        let object = "object.bin";
+        let operation_id = Uuid::new_v4();
+        let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
+        let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
+        let barrier = RestoreFinalizeBarrier::install(bucket, object);
+        let finalize_set = Arc::clone(&set_disks);
+        let finalize = tokio::spawn(async move {
+            let mut opts = restore_finalize_options(operation_id);
+            opts.no_lock = true;
+            opts.namespace_lock_fence = Some(fence);
+            finalize_set.finalize_restore_metadata(bucket, object, &restored, &opts).await
+        });
+        barrier.wait_until_paused().await;
+        loss_handle.store(true, std::sync::atomic::Ordering::Release);
+        barrier.release();
+
+        let error = finalize
+            .await
+            .expect("restore finalization task should join")
+            .expect_err("lost outer fence must reject restore finalization");
+        assert!(matches!(
+            error,
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "restore_finalize_metadata",
+                required: 1,
+                achieved: 0,
+                ..
+            }
+        ));
+        assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
+    }
+
     async fn assert_local_source_intact(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, payload: &[u8]) {
         let mut restored = Vec::new();
         set_disks

crates/ecstore/src/set_disk/replication.rs

@@ -18,6 +18,78 @@ use rustfs_filemeta::RestoreStatusOps;
 use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
 use s3s::dto::{RestoreStatus, Timestamp};
 
+#[cfg(all(test, feature = "test-util"))]
+struct RestoreFinalizeBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+static RESTORE_FINALIZE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<RestoreFinalizeBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(all(test, feature = "test-util"))]
+pub(in crate::set_disk) struct RestoreFinalizeBarrier {
+    state: Arc<RestoreFinalizeBarrierState>,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl RestoreFinalizeBarrier {
+    pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(RestoreFinalizeBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        assert!(slot.is_none(), "restore finalize barrier must be installed by one test at a time");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(in crate::set_disk) async fn wait_until_paused(&self) {
+        self.state.arrived.notified().await;
+    }
+
+    pub(in crate::set_disk) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl Drop for RestoreFinalizeBarrier {
+    fn drop(&mut self) {
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
+    let barrier = RESTORE_FINALIZE_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("restore finalize barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 struct RestoreCleanupIdentity {
     version_id: Option<Uuid>,
@@ -80,7 +152,7 @@ impl SetDisks {
             .clone()
             .unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
         let version_id = expected.version_id.map(|v| v.to_string());
-        let _lock_guard = if !opts.no_lock {
+        let lock_guard = if !opts.no_lock {
             Some(
                 self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
                     .await?,
@@ -99,13 +171,16 @@ impl SetDisks {
             .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
             .await?
             .into_owned();
-        if let Some(expected_operation_id) = expected_operation_id {
-            require_restore_operation_id(&fi.metadata, expected_operation_id)?;
+        if let Some(expected_operation_id) = expected_operation_id
+            && restore_operation_id_from_metadata(&fi.metadata)?.is_some_and(|actual| actual != expected_operation_id)
+        {
+            return Err(Error::other("restore operation id changed before metadata finalization"));
         }
         if !expected.matches_file_info(&fi, &expected_etag) {
             return Err(Error::other("restored object changed before restore metadata finalization"));
         }
-        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        #[cfg(all(test, feature = "test-util"))]
+        maybe_pause_restore_finalize(bucket, object).await;
         let restore_expiry =
             lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
         fi.metadata.insert(
@@ -117,6 +192,10 @@ impl SetDisks {
             .to_string(),
         );
         self.invalidate_get_object_metadata_cache(bucket, object).await;
+        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(Error::other("restore finalization lock lost before metadata update"));
+        }
         self.update_object_meta_with_opts(
             bucket,
             object,

crates/ecstore/src/store/heal.rs

@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use super::*;
+use crate::core::pools::POOL_META_NAME;
+use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::heal::HealOperations as _;
+use crate::storage_api_contracts::namespace::NamespaceLocking as _;
+use rustfs_lock::NamespaceLockGuard;
 use tracing::trace;
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+#[derive(Debug, Clone, Copy)]
+enum HealFormatPoolSkip {
+    Completed,
+    Retryable,
+}
+
+fn classify_heal_format_pool(
+    pool_idx: usize,
+    pool_cmd_line: &str,
+    pool_meta: &PoolMeta,
+    rebalance_meta: Option<&RebalanceMeta>,
+) -> Option<HealFormatPoolSkip> {
+    let Some(pool) = pool_meta.pools.get(pool_idx) else {
+        return Some(HealFormatPoolSkip::Retryable);
+    };
+
+    if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
+        return Some(HealFormatPoolSkip::Retryable);
+    }
+
+    if let Some(decommission) = pool.decommission.as_ref() {
+        if decommission.complete {
+            return Some(HealFormatPoolSkip::Completed);
+        }
+        if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    if let Some(meta) = rebalance_meta {
+        let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
+            return Some(HealFormatPoolSkip::Retryable);
+        };
+        if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    None
+}
+
+fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
+    match skip {
+        HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
+        HealFormatPoolSkip::Retryable => StorageError::SlowDown,
+    }
+}
+
+fn heal_format_fence_lost_error() -> Error {
+    StorageError::SlowDown
+}
+
 impl ECStore {
+    async fn acquire_heal_format_fence(
+        &self,
+    ) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option<RebalanceMeta>)> {
+        let metadata_pool = self
+            .pools
+            .first()
+            .cloned()
+            .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
+
+        // Metadata fence order is part of the decommission/rebalance protocol:
+        // pool.bin must always be acquired before rebalance.bin.
+        let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
+        let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
+        let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut pool_meta = PoolMeta::default();
+        pool_meta.load_no_lock(metadata_pool.clone()).await?;
+        if pool_meta.pools.len() != self.pools.len()
+            || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
+                pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
+            })
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut rebalance_meta = RebalanceMeta::new();
+        let rebalance_meta = match rebalance_meta
+            .load_with_opts(
+                metadata_pool,
+                ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+        {
+            Ok(()) => Some(rebalance_meta),
+            Err(Error::ConfigNotFound) => None,
+            Err(err) => return Err(err),
+        };
+
+        if rebalance_meta
+            .as_ref()
+            .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta))
+    }
+
     fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
         match opts.pool {
             Some(pool_idx) => Ok(vec![
@@ -52,8 +169,24 @@ impl ECStore {
         };
 
         let mut count_no_heal = 0;
+        let mut count_completed = 0;
         let mut first_error = None;
-        for pool in self.pools.iter() {
+        for (pool_idx, pool) in self.pools.iter().enumerate() {
+            let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
+            if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
+            {
+                if matches!(skip, HealFormatPoolSkip::Completed) {
+                    count_completed += 1;
+                } else {
+                    first_error.get_or_insert(heal_format_pool_skip_error(skip));
+                }
+                continue;
+            }
+
             let (mut result, err) = pool.heal_format(dry_run).await?;
             if let Some(err) = err {
                 match err {
@@ -69,11 +202,18 @@ impl ECStore {
             r.set_count += result.set_count;
             r.before.drives.append(&mut result.before.drives);
             r.after.drives.append(&mut result.after.drives);
+
+            // Sets::heal_format cannot observe this guard before each disk write;
+            // fail closed after the call if the lease was lost during format IO.
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
         }
         if let Some(err) = first_error {
             return Ok((r, Some(err)));
         }
-        if count_no_heal == self.pools.len() {
+        if count_no_heal + count_completed == self.pools.len() {
             info!(
                 event = EVENT_HEAL_FORMAT_COMPLETED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -300,6 +440,7 @@ mod tests {
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
     use crate::disk::{DiskOption, format::FormatV3, new_disk};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints};
+    use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
     use crate::store::init_format::{load_format_erasure, save_format_file};
 
     async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
@@ -347,6 +488,164 @@ mod tests {
         }
     }
 
+    fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
+        PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(info),
+            }],
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn heal_format_pool_state_barriers_are_classified() {
+        let active = pool_meta_with_decommission(PoolDecommissionInfo {
+            start_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &active, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        for info in [
+            PoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            },
+            PoolDecommissionInfo {
+                canceled: true,
+                ..Default::default()
+            },
+        ] {
+            assert!(matches!(
+                classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
+                Some(HealFormatPoolSkip::Retryable)
+            ));
+        }
+
+        let completed = pool_meta_with_decommission(PoolDecommissionInfo {
+            complete: true,
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &completed, None),
+            Some(HealFormatPoolSkip::Completed)
+        ));
+    }
+
+    #[test]
+    fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
+        let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        let rebalance = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopping = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                info: RebalanceInfo {
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity_without_decommission = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "", &identity_meta, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopped = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Stopped,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
+
+        let stopping_after_stop = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+    }
+
+    #[test]
+    fn skipped_heal_format_pool_is_never_reported_as_success() {
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
+            StorageError::SlowDown
+        ));
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
+            StorageError::NoHealRequired
+        ));
+    }
+
     #[tokio::test]
     async fn heal_object_pool_scope_selects_only_requested_pool() {
         let store = minimal_heal_store().await;
@@ -615,6 +914,18 @@ mod tests {
             bucket_fence_registry: std::sync::Arc::default(),
         };
 
+        let err = store
+            .handle_heal_format(false)
+            .await
+            .expect_err("missing pool metadata must fail closed before format writes");
+        assert!(matches!(err, StorageError::SlowDown));
+
+        let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        pool_meta
+            .save(store.pools.clone())
+            .await
+            .expect("pool metadata should be persisted before format heal");
+
         let (result, err) = store
             .handle_heal_format(false)
             .await
@@ -628,5 +939,22 @@ mod tests {
             .await
             .expect("the later pool should be healed despite the first pool error");
         assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
+
+        let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        for status in &mut completed_meta.pools {
+            status.decommission = Some(PoolDecommissionInfo {
+                complete: true,
+                ..Default::default()
+            });
+        }
+        completed_meta
+            .save(store.pools.clone())
+            .await
+            .expect("completed pool metadata should be persisted");
+        let (_, err) = store
+            .handle_heal_format(false)
+            .await
+            .expect("completed pools should be reported as a no-op");
+        assert!(matches!(err, Some(StorageError::NoHealRequired)));
     }
 }

crates/ecstore/src/store/mod.rs

@@ -343,6 +343,23 @@ impl ECStore {
         let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
         decommission || rebalance
     }
+
+    /// Returns whether scanner metadata may still be hidden by a local
+    /// data-movement state. Terminal failed/canceled decommission entries
+    /// remain suspended until an operator clears or retries them, so they are
+    /// a publication barrier even after the worker has stopped.
+    pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
+        if self.scanner_data_movement_active().await {
+            return true;
+        }
+
+        let pool_meta = self.pool_meta.read().await;
+        pool_meta.pools.iter().any(|pool| {
+            pool.decommission
+                .as_ref()
+                .is_some_and(|info| !info.queued && (info.failed || info.canceled))
+        })
+    }
 }
 
 // impl Clone for ECStore {
@@ -875,6 +892,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
     use crate::runtime::global::reset_local_disk_test_state;
     use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
@@ -911,6 +929,72 @@ mod tests {
         })
     }
 
+    #[tokio::test]
+    async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let cases = [
+            (
+                "active",
+                PoolDecommissionInfo {
+                    start_time: Some(OffsetDateTime::now_utc()),
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "failed",
+                PoolDecommissionInfo {
+                    failed: true,
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "canceled",
+                PoolDecommissionInfo {
+                    canceled: true,
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "queued_failed",
+                PoolDecommissionInfo {
+                    failed: true,
+                    queued: true,
+                    ..Default::default()
+                },
+                false,
+            ),
+            (
+                "complete",
+                PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                },
+                false,
+            ),
+            ("idle", PoolDecommissionInfo::default(), false),
+        ];
+
+        for (name, decommission, expected) in cases {
+            *store.pool_meta.write().await = PoolMeta {
+                pools: vec![PoolStatus {
+                    id: 0,
+                    cmd_line: format!("scanner-publication-{name}"),
+                    last_update: OffsetDateTime::now_utc(),
+                    decommission: Some(decommission),
+                }],
+                ..Default::default()
+            };
+            assert_eq!(
+                store.scanner_data_usage_publication_blocked().await,
+                expected,
+                "unexpected scanner publication barrier state for {name}"
+            );
+        }
+    }
+
     // The object graph is the isolation carrier: two ECStore instances holding
     // distinct contexts report independent erasure state through their real
     // `&self` accessors — no cross-contamination.

crates/heal/src/heal/manager/scheduler.rs

@@ -567,35 +567,17 @@ impl HealManager {
 pub(super) fn heal_request_set_key(request: &HealRequest) -> Option<String> {
     match &request.heal_type {
         HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
-        HealType::Object { .. } => heal_options_set_key(&request.options),
-        _ => None,
-    }
-}
-
-pub(super) fn heal_options_set_key(options: &HealOptions) -> Option<String> {
-    match (options.pool_index, options.set_index) {
-        (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
+        HealType::Object { .. } => request.options.set_key(),
         _ => None,
     }
 }
 
 pub(super) fn heal_request_type_label(request: &HealRequest) -> &'static str {
-    match &request.heal_type {
-        HealType::Cluster => "cluster",
-        HealType::Object { .. } => "object",
-        HealType::Bucket { .. } => "bucket",
-        HealType::Prefix { .. } => "prefix",
-        HealType::ErasureSet { .. } => "erasure_set",
-        HealType::Metadata { .. } => "metadata",
-        HealType::ECDecode { .. } => "ec_decode",
-    }
+    request.heal_type.kind_label()
 }
 
 pub(super) fn heal_request_set_metric_label(request: &HealRequest) -> String {
-    heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) {
-        (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
-        _ => "global".to_string(),
-    })
+    heal_request_set_key(request).unwrap_or_else(|| request.options.set_metric_label())
 }
 
 pub(super) fn record_scheduler_skip(set_label: &str) {
@@ -673,7 +655,7 @@ fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
 pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
     match &task.heal_type {
         HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
-        HealType::Object { .. } => heal_options_set_key(&task.options),
+        HealType::Object { .. } => task.options.set_key(),
         _ => None,
     }
 }

crates/heal/src/heal/manager/tests.rs

@@ -744,10 +744,8 @@ fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() {
     let mut running = HashMap::new();
     running.insert("pool_0_set_1".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
-        |request| can_schedule_request(request, &running, 1),
-        |request| heal_request_set_key(request),
-    );
+    let (popped, skipped_sets) =
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
     let popped = popped.expect("should find runnable request");
 
     assert_eq!(skipped_sets, vec!["pool_0_set_1".to_string()]);
@@ -788,10 +786,8 @@ fn test_priority_queue_pop_runnable_restores_all_blocked_items() {
     running.insert("pool_0_set_2".to_string(), 1);
     running.insert("pool_0_set_3".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
-        |request| can_schedule_request(request, &running, 1),
-        |request| heal_request_set_key(request),
-    );
+    let (popped, skipped_sets) =
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
 
     assert!(popped.is_none());
     assert_eq!(
@@ -843,10 +839,8 @@ fn test_priority_queue_pop_runnable_restores_deferred_with_tail() {
     running.insert("pool_0_set_1".to_string(), 1);
     running.insert("pool_0_set_2".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
-        |request| can_schedule_request(request, &running, 1),
-        |request| heal_request_set_key(request),
-    );
+    let (popped, skipped_sets) =
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
 
     assert_eq!(skipped_sets, vec!["pool_0_set_1".to_string(), "pool_0_set_2".to_string()]);
     assert!(matches!(
@@ -904,6 +898,31 @@ fn test_can_schedule_scoped_object_request_respects_per_set_limit() {
     assert!(can_schedule_request(&request, &running, 2));
 }
 
+#[test]
+fn test_heal_request_and_task_metric_labels_match() {
+    let request = HealRequest::new(
+        HealType::Object {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+        },
+        HealOptions {
+            pool_index: Some(0),
+            set_index: Some(1),
+            ..Default::default()
+        },
+        HealPriority::Normal,
+    );
+
+    assert_eq!(heal_request_type_label(&request), "object");
+    assert_eq!(heal_request_set_key(&request), Some("pool_0_set_1".to_string()));
+    assert_eq!(heal_request_set_metric_label(&request), "pool_0_set_1");
+
+    let task = HealTask::from_request(request, Arc::new(MockStorage));
+    assert_eq!(task.metric_type_label(), "object");
+    assert_eq!(task.metric_set_label(), "pool_0_set_1");
+}
+
 #[tokio::test]
 async fn test_submit_heal_request_returns_merged_for_duplicate() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/heal/src/heal/progress.rs

@@ -218,15 +218,6 @@ impl HealStatistics {
         self.total_bytes_healed += bytes;
         self.last_update_time = SystemTime::now();
     }
-
-    pub fn get_success_rate(&self) -> f64 {
-        let total = self.successful_tasks + self.failed_tasks;
-        if total > 0 {
-            (self.successful_tasks as f64 / total as f64) * 100.0
-        } else {
-            0.0
-        }
-    }
 }
 
 #[cfg(test)]
@@ -539,38 +530,4 @@ mod tests {
         assert_eq!(stats.total_objects_healed, 8);
         assert_eq!(stats.total_bytes_healed, 8192);
     }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 8;
-        stats.failed_tasks = 2;
-
-        // success_rate = 8 / (8 + 2) * 100 = 80%
-        assert!((stats.get_success_rate() - 80.0).abs() < 0.001);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_zero_total() {
-        let stats = HealStatistics::new();
-        assert_eq!(stats.get_success_rate(), 0.0);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_all_success() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 10;
-        stats.failed_tasks = 0;
-
-        assert!((stats.get_success_rate() - 100.0).abs() < 0.001);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_all_failure() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 0;
-        stats.failed_tasks = 5;
-
-        assert_eq!(stats.get_success_rate(), 0.0);
-    }
 }

crates/heal/src/heal/storage.rs

@@ -1202,13 +1202,22 @@ impl HealStorageAPI for ECStoreHealStorage {
                 let version_id = obj.version_id.map(|u| u.to_string());
                 let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
                 let is_delete_marker = obj.delete_marker;
-                let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
-                HealListItem {
-                    name: obj.name,
-                    version_id,
-                    mod_time_unix_nanos,
-                    lifecycle_object_info,
-                    is_delete_marker,
+                if include_lifecycle_object_info {
+                    HealListItem {
+                        name: obj.name.clone(),
+                        version_id,
+                        mod_time_unix_nanos,
+                        lifecycle_object_info: Some(obj),
+                        is_delete_marker,
+                    }
+                } else {
+                    HealListItem {
+                        name: obj.name,
+                        version_id,
+                        mod_time_unix_nanos,
+                        lifecycle_object_info: None,
+                        is_delete_marker,
+                    }
                 }
             })
             .collect();

crates/heal/src/heal/task.rs

@@ -109,7 +109,7 @@ pub enum HealType {
 }
 
 impl HealType {
-    fn log_kind(&self) -> &'static str {
+    pub(crate) fn kind_label(&self) -> &'static str {
         match self {
             Self::Cluster => "cluster",
             Self::Object { .. } => "object",
@@ -227,6 +227,19 @@ impl Default for HealOptions {
     }
 }
 
+impl HealOptions {
+    pub(crate) fn set_key(&self) -> Option<String> {
+        match (self.pool_index, self.set_index) {
+            (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
+            _ => None,
+        }
+    }
+
+    pub(crate) fn set_metric_label(&self) -> String {
+        self.set_key().unwrap_or_else(|| "global".to_string())
+    }
+}
+
 /// Heal task status
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum HealTaskStatus {
@@ -491,15 +504,7 @@ impl HealTask {
     }
 
     pub fn metric_type_label(&self) -> &'static str {
-        match &self.heal_type {
-            HealType::Cluster => "cluster",
-            HealType::Object { .. } => "object",
-            HealType::Bucket { .. } => "bucket",
-            HealType::Prefix { .. } => "prefix",
-            HealType::ErasureSet { .. } => "erasure_set",
-            HealType::Metadata { .. } => "metadata",
-            HealType::ECDecode { .. } => "ec_decode",
-        }
+        self.heal_type.kind_label()
     }
 
     pub(crate) fn has_batch_failure(&self) -> bool {
@@ -520,10 +525,7 @@ impl HealTask {
     pub fn metric_set_label(&self) -> String {
         match &self.heal_type {
             HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
-            _ => match (self.options.pool_index, self.options.set_index) {
-                (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
-                _ => "global".to_string(),
-            },
+            _ => self.options.set_metric_label(),
         }
     }
 
@@ -532,7 +534,7 @@ impl HealTask {
             let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
                 .with_duration(duration)
                 .with_attr("task_id", self.id.as_str())
-                .with_attr("heal_type", self.heal_type.log_kind())
+                .with_attr("heal_type", self.heal_type.kind_label())
                 .with_attr("state", state)
                 .with_attr("source", self.source.as_str())
                 .with_attr("priority", self.priority.as_str())
@@ -795,7 +797,7 @@ impl HealTask {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_TASK,
             task_id = %self.id,
-            heal_type = self.heal_type.log_kind(),
+            heal_type = self.heal_type.kind_label(),
             state = "started",
             queue_delay = ?queue_delay,
             "Heal task started"
@@ -836,7 +838,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "completed",
                     "Heal task completed"
                 });
@@ -850,7 +852,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "cancelled",
                     "Heal task cancelled"
                 );
@@ -863,7 +865,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "timed_out",
                     "Heal task timed out"
                 });
@@ -880,7 +882,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "failed",
                     error = %e,
                     "Heal task failed"
@@ -909,7 +911,7 @@ impl HealTask {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_TASK,
             task_id = %self.id,
-            heal_type = self.heal_type.log_kind(),
+            heal_type = self.heal_type.kind_label(),
             state = "cancelled",
             source = "manual",
             "Heal task cancellation requested"

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -231,6 +231,10 @@ impl HealTask {
                             "Heal erasure set format repair skipped because no format heal was required"
                         );
                     } else {
+                        let error = e;
+                        if error.is_recoverable_heal() {
+                            return Err(error);
+                        }
                         error!(
                             target: "rustfs::heal::task",
                             event = EVENT_HEAL_ERASURE_SET_RESULT,
@@ -239,7 +243,7 @@ impl HealTask {
                             task_id = %self.id,
                             set_disk_id,
                             result = "format_failed",
-                            error = %e,
+                            error = %error,
                             "Heal erasure set failed"
                         );
                         {
@@ -247,7 +251,7 @@ impl HealTask {
                             progress.update_progress(4, 4, 0, 0);
                         }
                         return Err(Error::TaskExecutionFailed {
-                            message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
+                            message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
                         });
                     }
                 } else {
@@ -284,6 +288,9 @@ impl HealTask {
             Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
             Err(e) => {
+                if e.is_recoverable_heal() {
+                    return Err(e);
+                }
                 error!(
                     target: "rustfs::heal::task",
                     event = EVENT_HEAL_ERASURE_SET_RESULT,

crates/heal/src/heal/task/tests.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk};
+use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
 use super::*;
 use crate::heal::storage::{HealListItem, HealObjectInfo};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
@@ -547,6 +547,7 @@ struct MockStorage {
     heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
     heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
     format_no_heal_required: Mutex<bool>,
+    format_error: Mutex<Option<Error>>,
     global_format_calls: Mutex<u32>,
     replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
     replacement_targets_ready: Mutex<bool>,
@@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage {
 
     async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         *self.global_format_calls.lock().unwrap() += 1;
+        if let Some(error) = self.format_error.lock().unwrap().take() {
+            return Err(error);
+        }
         let no_heal_required = *self.format_no_heal_required.lock().unwrap();
         if no_heal_required {
             Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
@@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() {
     );
 }
 
+#[tokio::test]
+async fn erasure_set_format_slowdown_is_propagated() {
+    let storage = Arc::new(MockStorage {
+        format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::ErasureSet {
+            buckets: Vec::new(),
+            set_disk_id: "pool_0_set_0".to_string(),
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage);
+
+    let error = task
+        .execute()
+        .await
+        .expect_err("format SlowDown must remain recoverable for the task manager");
+
+    assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
+}
+
 #[tokio::test]
 async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/iam/tests/ecstore_test_compat/mod.rs

@@ -17,11 +17,11 @@
 //! All direct `rustfs_ecstore` facade imports used by tests in this crate
 //! must go through this module (architecture migration rule:
 //! `check_architecture_migration_rules.sh`). Keep the surface minimal —
-//! only what the tests actually need to build a temp-disk ECStore fixture
-//! and to flip the erasure setup type for lock-quorum fault injection.
+//! only what the tests actually need to run storage-backed IAM scenarios.
 
 #[allow(unused_imports)]
 pub(crate) mod fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
     pub(crate) use rustfs_ecstore::api::layout::SetupType;
 
     // `update_erasure_type` is a write-side global facade entry. Its use is

crates/iam/tests/minio_iam_migration_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod ecstore_test_compat;
+
+use ecstore_test_compat::fixture::try_migrate_iam_config;
+use rustfs_credentials::{get_global_action_cred, init_global_action_credentials};
+use rustfs_iam::store::object::{
+    IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX, IAM_CONFIG_POLICY_DB_USERS_PREFIX, IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX,
+    IAM_CONFIG_USERS_PREFIX, ObjectStore,
+};
+use rustfs_iam::store::{Store, UserType};
+use rustfs_iam::utils::generate_jwt;
+use rustfs_policy::auth::UserIdentity;
+use serde_json::{Value, json};
+use std::collections::HashMap;
+
+const LEGACY_META_BUCKET: &str = ".minio.sys";
+const REGULAR_USER: &str = "minio-user";
+const SERVICE_ACCOUNT: &str = "minio-service-account";
+
+async fn seed_legacy_iam_object(env: &rustfs_test_utils::TestECStoreEnv, path: &str, value: &Value) {
+    env.put_object_bytes(
+        LEGACY_META_BUCKET,
+        path,
+        serde_json::to_vec(value).expect("legacy IAM object must serialize"),
+    )
+    .await;
+}
+
+fn assert_identity_fields(actual: &UserIdentity, expected: &Value) {
+    assert_eq!(
+        serde_json::to_value(actual).expect("loaded identity must serialize"),
+        *expected,
+        "migration must preserve every credential field except expiration",
+    );
+}
+
+async fn assert_identity_survives(
+    store: &ObjectStore,
+    identity_path: &str,
+    name: &str,
+    user_type: UserType,
+    source: &Value,
+    expected_policy: &Value,
+) {
+    let mut expected = source.clone();
+    expected["credentials"]["expiration"] = Value::Null;
+
+    let persisted: UserIdentity = store
+        .load_iam_config(identity_path)
+        .await
+        .expect("migrated identity must be persisted");
+    assert_identity_fields(&persisted, &expected);
+
+    for _ in 0..2 {
+        let actual = store
+            .load_user_identity(name, user_type)
+            .await
+            .expect("migrated permanent identity must remain loadable");
+        assert_identity_fields(&actual, &expected);
+    }
+
+    let mut mappings = HashMap::new();
+    store
+        .load_mapped_policy(name, user_type, false, &mut mappings)
+        .await
+        .expect("loading the identity must not delete its policy mapping");
+    let actual_policy = mappings.get(name).expect("migrated policy mapping must exist");
+    assert_eq!(
+        serde_json::to_value(actual_policy).expect("loaded policy mapping must serialize"),
+        *expected_policy,
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
+    if get_global_action_cred().is_none() {
+        init_global_action_credentials(Some("MINIOMIGRATIONROOT".to_string()), Some("minio-migration-root-secret".to_string()))
+            .expect("root credentials must initialize for JWT validation");
+    }
+
+    let temp_dir = tempfile::TempDir::with_prefix("rustfs_minio_iam_migration_").expect("temp directory must be created");
+    let env = rustfs_test_utils::TestECStoreEnv::builder()
+        .base_dir(temp_dir.path())
+        .init_bucket_metadata(false)
+        .build()
+        .await;
+    for disk_path in &env.disk_paths {
+        tokio::fs::create_dir_all(disk_path.join(LEGACY_META_BUCKET))
+            .await
+            .expect("legacy metadata volume must be created");
+    }
+
+    let regular_source = json!({
+        "version": 1,
+        "credentials": {
+            "accessKey": REGULAR_USER,
+            "secretKey": "regular-user-secret",
+            "sessionToken": "",
+            "expiration": "0001-01-01T00:00:00Z",
+            "status": "on",
+            "parentUser": "regular-parent",
+            "groups": ["engineering", "operations"],
+            "claims": {"tenant": "alpha"},
+            "name": "MinIO regular user",
+            "description": "migrated regular identity"
+        },
+        "updatedAt": "2025-03-07T12:00:00Z"
+    });
+    let service_claims = json!({"sa-policy": "inherited-policy", "tenant": "alpha"});
+    let service_secret = "service-account-secret";
+    let service_source = json!({
+        "version": 1,
+        "credentials": {
+            "accessKey": SERVICE_ACCOUNT,
+            "secretKey": service_secret,
+            "sessionToken": generate_jwt(&service_claims, service_secret).expect("service-account JWT must be generated"),
+            "expiration": "1970-01-01T00:00:00Z",
+            "status": "on",
+            "parentUser": REGULAR_USER,
+            "groups": ["service-accounts"],
+            "claims": service_claims,
+            "name": "MinIO service account",
+            "description": "migrated service identity"
+        },
+        "updatedAt": "2025-03-07T12:00:00Z"
+    });
+    let regular_policy_source = json!({"version": 1, "policy": "readwrite", "updatedAt": "2025-03-07T12:00:00Z"});
+    let service_policy_source = json!({"version": 1, "policy": "readonly", "updatedAt": "2025-03-07T12:00:00Z"});
+
+    let regular_identity_path = format!("{}{REGULAR_USER}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
+    let service_identity_path = format!("{}{SERVICE_ACCOUNT}/identity.json", IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX.as_str());
+
+    seed_legacy_iam_object(&env, &regular_identity_path, &regular_source).await;
+    seed_legacy_iam_object(&env, &service_identity_path, &service_source).await;
+    seed_legacy_iam_object(
+        &env,
+        &format!("{}{REGULAR_USER}.json", IAM_CONFIG_POLICY_DB_USERS_PREFIX.as_str()),
+        &regular_policy_source,
+    )
+    .await;
+    seed_legacy_iam_object(
+        &env,
+        &format!("{}{SERVICE_ACCOUNT}.json", IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX.as_str()),
+        &service_policy_source,
+    )
+    .await;
+
+    try_migrate_iam_config(env.ecstore.clone(), None).await;
+
+    let store = ObjectStore::new(env.ecstore);
+    assert_identity_survives(
+        &store,
+        &regular_identity_path,
+        REGULAR_USER,
+        UserType::Reg,
+        &regular_source,
+        &regular_policy_source,
+    )
+    .await;
+    assert_identity_survives(
+        &store,
+        &service_identity_path,
+        SERVICE_ACCOUNT,
+        UserType::Svc,
+        &service_source,
+        &service_policy_source,
+    )
+    .await;
+}

crates/io-metrics/src/internode_metrics.rs

@@ -211,6 +211,146 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
 
 static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
 
+#[cfg(not(test))]
+struct InternodeServerMetricHandles {
+    sent_bytes: metrics::Counter,
+    recv_bytes: metrics::Counter,
+    outgoing_requests: metrics::Counter,
+    incoming_requests: metrics::Counter,
+    errors: metrics::Counter,
+}
+
+#[cfg(not(test))]
+impl InternodeServerMetricHandles {
+    fn new(server: &'static str) -> Self {
+        Self {
+            sent_bytes: counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => server),
+            recv_bytes: counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => server),
+            outgoing_requests: counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => server),
+            incoming_requests: counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => server),
+            errors: counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => server),
+        }
+    }
+}
+
+#[cfg(not(test))]
+static INTERNODE_SERVER_METRIC_HANDLES: LazyLock<InternodeServerMetricHandles> =
+    LazyLock::new(|| InternodeServerMetricHandles::new(current_server_label()));
+
+#[cfg(not(test))]
+struct GrpcReadVersionMetricHandles {
+    sent_bytes: metrics::Counter,
+    recv_bytes: metrics::Counter,
+    outgoing_requests: metrics::Counter,
+    incoming_requests: metrics::Counter,
+    errors: metrics::Counter,
+    duration: metrics::Histogram,
+    request_encode: metrics::Histogram,
+    request_decode: metrics::Histogram,
+    disk_read: metrics::Histogram,
+    response_json_encode: metrics::Histogram,
+    response_msgpack_encode: metrics::Histogram,
+    rpc_roundtrip: metrics::Histogram,
+    response_decode: metrics::Histogram,
+}
+
+#[cfg(not(test))]
+impl GrpcReadVersionMetricHandles {
+    fn new(server: &'static str) -> Self {
+        Self {
+            sent_bytes: counter!(
+                INTERNODE_OPERATION_SENT_BYTES_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            recv_bytes: counter!(
+                INTERNODE_OPERATION_RECV_BYTES_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            outgoing_requests: counter!(
+                INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            incoming_requests: counter!(
+                INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            errors: counter!(
+                INTERNODE_OPERATION_ERRORS_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            duration: metrics::histogram!(
+                INTERNODE_OPERATION_DURATION_MS,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            request_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE),
+            request_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE),
+            disk_read: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_DISK_READ),
+            response_json_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE),
+            response_msgpack_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE),
+            rpc_roundtrip: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
+            response_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE),
+        }
+    }
+
+    fn stage_duration(server: &'static str, stage: &'static str) -> metrics::Histogram {
+        metrics::histogram!(
+            INTERNODE_OPERATION_STAGE_DURATION_MS,
+            SERVER_LABEL => server,
+            OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+            BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC,
+            STAGE_LABEL => stage
+        )
+    }
+
+    fn stage_duration_for(&self, stage: &'static str) -> Option<&metrics::Histogram> {
+        match stage {
+            INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE => Some(&self.request_encode),
+            INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE => Some(&self.request_decode),
+            INTERNODE_STAGE_READ_VERSION_DISK_READ => Some(&self.disk_read),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE => Some(&self.response_json_encode),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE => Some(&self.response_msgpack_encode),
+            INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP => Some(&self.rpc_roundtrip),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE => Some(&self.response_decode),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(not(test))]
+static GRPC_READ_VERSION_METRIC_HANDLES: LazyLock<GrpcReadVersionMetricHandles> =
+    LazyLock::new(|| GrpcReadVersionMetricHandles::new(current_server_label()));
+
+#[cfg(not(test))]
+fn server_metric_handles_if_ready() -> Option<&'static InternodeServerMetricHandles> {
+    STABLE_SERVER_LABEL.get()?;
+    Some(&INTERNODE_SERVER_METRIC_HANDLES)
+}
+
+#[cfg(not(test))]
+fn grpc_read_version_metric_handles_if_ready(
+    operation: &'static str,
+    backend: &'static str,
+) -> Option<&'static GrpcReadVersionMetricHandles> {
+    STABLE_SERVER_LABEL.get()?;
+    if operation == INTERNODE_OPERATION_GRPC_READ_VERSION && backend == INTERNODE_TRANSPORT_BACKEND_GRPC {
+        Some(&GRPC_READ_VERSION_METRIC_HANDLES)
+    } else {
+        None
+    }
+}
+
 /// Injects the stable server label (node name or address) stamped on
 /// internode metrics. The runtime calls this when the local node name is
 /// published (see ecstore's `set_local_node_name`); the first write wins.
@@ -284,6 +424,11 @@ impl InternodeMetrics {
             return;
         }
         self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.sent_bytes.increment(bytes);
+            return;
+        }
         counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
     }
 
@@ -298,6 +443,11 @@ impl InternodeMetrics {
         if bytes == 0 {
             return;
         }
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.sent_bytes.increment(bytes);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_SENT_BYTES_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -313,6 +463,11 @@ impl InternodeMetrics {
             return;
         }
         self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.recv_bytes.increment(bytes);
+            return;
+        }
         counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
     }
 
@@ -327,6 +482,11 @@ impl InternodeMetrics {
         if bytes == 0 {
             return;
         }
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.recv_bytes.increment(bytes);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_RECV_BYTES_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -338,6 +498,11 @@ impl InternodeMetrics {
 
     pub fn record_outgoing_request(&self) {
         self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.outgoing_requests.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -347,6 +512,11 @@ impl InternodeMetrics {
 
     pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_outgoing_request();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.outgoing_requests.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -358,6 +528,11 @@ impl InternodeMetrics {
 
     pub fn record_incoming_request(&self) {
         self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.incoming_requests.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -367,6 +542,11 @@ impl InternodeMetrics {
 
     pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_incoming_request();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.incoming_requests.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -378,6 +558,11 @@ impl InternodeMetrics {
 
     pub fn record_error(&self) {
         self.errors_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.errors.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -387,6 +572,11 @@ impl InternodeMetrics {
 
     pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_error();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.errors.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_ERRORS_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -398,6 +588,11 @@ impl InternodeMetrics {
 
     pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
         let duration_ms = duration.as_secs_f64() * 1000.0;
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.duration.record(duration_ms);
+            return;
+        }
         metrics::histogram!(
             INTERNODE_OPERATION_DURATION_MS,
             SERVER_LABEL => current_server_label(),
@@ -415,6 +610,13 @@ impl InternodeMetrics {
         duration: Duration,
     ) {
         let duration_ms = duration.as_secs_f64() * 1000.0;
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend)
+            && let Some(histogram) = handles.stage_duration_for(stage)
+        {
+            histogram.record(duration_ms);
+            return;
+        }
         metrics::histogram!(
             INTERNODE_OPERATION_STAGE_DURATION_MS,
             SERVER_LABEL => current_server_label(),

crates/io-metrics/src/lib.rs

@@ -121,6 +121,16 @@ pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_ba
 pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync";
 pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall";
 
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED: &str = "disabled";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS: &str = "le_250ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS: &str = "le_500ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS: &str = "le_1000ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS: &str = "gt_1000ms";
+
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED: &str = "acquired";
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN: &str = "timeout_slowdown";
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR: &str = "lock_error";
+
 pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial";
 pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel";
 pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER: &str = "leader";
@@ -2060,6 +2070,14 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
     }
 }
 
+#[inline(always)]
+pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
+    if !put_stage_metrics_enabled() {
+        return;
+    }
+    counter!("rustfs_s3_put_object_commit_namespace_lock_admission_total", "budget" => budget, "outcome" => outcome).increment(1);
+}
+
 #[inline(always)]
 fn put_stage_count_value(value: usize) -> f64 {
     match u32::try_from(value) {
@@ -3204,6 +3222,83 @@ mod tests {
         assert!(stages.iter().all(|stage| recorded.contains(*stage)));
     }
 
+    #[test]
+    fn put_commit_lock_admission_labels_are_static_and_gated() {
+        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        let budgets = [
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
+        ];
+        let outcomes = [
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
+        ];
+        assert_eq!(budgets.iter().copied().collect::<HashSet<_>>().len(), budgets.len());
+        assert_eq!(outcomes.iter().copied().collect::<HashSet<_>>().len(), outcomes.len());
+        assert!(budgets.iter().chain(outcomes.iter()).all(|label| {
+            !label.contains('/')
+                && !label.contains('{')
+                && !label.contains('}')
+                && !label.contains(' ')
+                && label
+                    .chars()
+                    .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_')
+        }));
+
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            set_put_stage_metrics_enabled(false);
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            );
+
+            set_put_stage_metrics_enabled(true);
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            );
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+            );
+            set_put_stage_metrics_enabled(false);
+        });
+
+        let rows = snapshotter.snapshot().into_vec();
+        assert_eq!(
+            counter_total(&rows, "rustfs_s3_put_object_commit_namespace_lock_admission_total"),
+            Some(2)
+        );
+        let label_sets = rows
+            .iter()
+            .filter(|(composite, _, _, _)| {
+                composite.kind() == MetricKind::Counter
+                    && composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+            })
+            .map(|(composite, _, _, _)| {
+                composite
+                    .key()
+                    .labels()
+                    .map(|label| (label.key().to_string(), label.value().to_string()))
+                    .collect::<HashSet<_>>()
+            })
+            .collect::<Vec<_>>();
+        assert!(label_sets.contains(&HashSet::from([
+            ("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS.to_string()),
+            ("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN.to_string(),),
+        ])));
+        assert!(label_sets.contains(&HashSet::from([
+            ("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS.to_string()),
+            ("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED.to_string()),
+        ])));
+    }
+
     #[test]
     fn put_rename_code_level_metrics_are_static_and_gated() {
         let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());

crates/io-metrics/tests/internode_cached_handles.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::with_local_recorder;
+use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+    INTERNODE_TRANSPORT_BACKEND_GRPC, InternodeMetrics, set_internode_server_label,
+};
+use std::time::Duration;
+
+type MetricRow = (
+    metrics_util::CompositeKey,
+    Option<metrics::Unit>,
+    Option<metrics::SharedString>,
+    DebugValue,
+);
+
+const SERVER_LABEL: &str = "server";
+const OPERATION_LABEL: &str = "operation";
+const BACKEND_LABEL: &str = "backend";
+const STAGE_LABEL: &str = "stage";
+const SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_sent_bytes_total";
+const RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_recv_bytes_total";
+const REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_requests_outgoing_total";
+const REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_requests_incoming_total";
+const ERRORS_TOTAL: &str = "rustfs_system_network_internode_errors_total";
+const OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
+const OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
+const OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
+const OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
+const OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
+const OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
+const OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
+
+#[test]
+fn cached_grpc_read_version_metric_handles_preserve_labels_and_values() {
+    set_internode_server_label("cached-grpc-read-version-test");
+
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    let metrics = InternodeMetrics::default();
+
+    with_local_recorder(&recorder, || {
+        metrics.record_sent_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            17,
+        );
+        metrics.record_recv_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            23,
+        );
+        metrics.record_outgoing_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_incoming_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+        metrics.record_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            Duration::from_micros(250),
+        );
+        metrics.record_stage_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+            Duration::from_micros(125),
+        );
+        metrics.record_stage_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            INTERNODE_STAGE_READ_VERSION_DISK_READ,
+            Duration::from_micros(75),
+        );
+    });
+
+    let rows = snapshotter.snapshot().into_vec();
+    assert_counter(&rows, SENT_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 17);
+    assert_counter(&rows, RECV_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 23);
+    assert_counter(&rows, REQUESTS_OUTGOING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(&rows, REQUESTS_INCOMING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(&rows, ERRORS_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(
+        &rows,
+        OPERATION_SENT_BYTES_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        17,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_RECV_BYTES_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        23,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_REQUESTS_OUTGOING_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_REQUESTS_INCOMING_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_ERRORS_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        &[0.25],
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_STAGE_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+            (STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
+        ],
+        &[0.125],
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_STAGE_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+            (STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_DISK_READ),
+        ],
+        &[0.075],
+    );
+}
+
+fn assert_counter(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: u64) {
+    match metric_value(rows, name, labels) {
+        DebugValue::Counter(value) => assert_eq!(*value, expected),
+        other => panic!("{name} should be a counter, got {other:?}"),
+    }
+}
+
+fn assert_histogram(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: &[f64]) {
+    match metric_value(rows, name, labels) {
+        DebugValue::Histogram(samples) => {
+            let actual: Vec<_> = samples.iter().map(|sample| sample.0).collect();
+            assert_eq!(actual, expected);
+        }
+        other => panic!("{name} should be a histogram, got {other:?}"),
+    }
+}
+
+fn metric_value<'a>(rows: &'a [MetricRow], name: &str, labels: &[(&str, &str)]) -> &'a DebugValue {
+    let mut matches = rows.iter().filter(|(composite, _, _, _)| {
+        composite.key().name() == name
+            && labels.iter().all(|(key, value)| {
+                composite
+                    .key()
+                    .labels()
+                    .any(|label| label.key() == *key && label.value() == *value)
+            })
+    });
+    let Some((_, _, _, value)) = matches.next() else {
+        panic!("{name} with labels {labels:?} was not recorded; rows={rows:?}");
+    };
+    assert!(matches.next().is_none(), "{name} with labels {labels:?} must be unique; rows={rows:?}");
+    value
+}

crates/lifecycle/src/evaluator.rs

@@ -66,9 +66,10 @@ impl Evaluator {
     }
 
     /// IsObjectLocked checks if it is appropriate to remove an
-    /// object according to its persisted object-lock metadata.
+    /// object according to its persisted object-lock metadata and the bucket
+    /// default retention.
     pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
-        object_lock::is_object_locked_by_metadata(&obj.user_defined, obj.delete_marker)
+        object_lock::is_object_locked(&obj.user_defined, obj.delete_marker, self.lock_retention.as_deref(), obj.mod_time)
     }
 
     /// eval will return a lifecycle event for each object in objs for a given time.
@@ -198,8 +199,9 @@ mod tests {
 
     use rustfs_common::metrics::IlmAction;
     use s3s::dto::{
-        BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, ObjectLockConfiguration,
-        ObjectLockEnabled, Transition, TransitionStorageClass,
+        BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
+        NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
+        Transition, TransitionStorageClass,
     };
     use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
     use time::OffsetDateTime;
@@ -300,6 +302,40 @@ mod tests {
         })
     }
 
+    fn lock_enabled_with_default_retention(days: i32) -> Arc<ObjectLockConfiguration> {
+        Arc::new(ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: Some(days),
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: None,
+                }),
+            }),
+        })
+    }
+
+    fn noncurrent_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
+        Arc::new(BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("expire-noncurrent".to_string()),
+                noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: None,
+                }),
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        })
+    }
+
     fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
         ObjectOpts {
             name: "logs/object".to_string(),
@@ -459,6 +495,52 @@ mod tests {
         assert_eq!(events[0].action, IlmAction::NoneAction);
     }
 
+    #[tokio::test]
+    async fn evaluator_skips_noncurrent_expiration_during_default_retention() {
+        let evaluator =
+            Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(30)));
+        let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
+        let noncurrent = ObjectOpts {
+            name: "logs/object".to_string(),
+            mod_time: Some(successor_time - time::Duration::days(1)),
+            successor_mod_time: Some(successor_time),
+            version_id: Some(Uuid::new_v4()),
+            is_latest: false,
+            num_versions: 1,
+            ..Default::default()
+        };
+
+        let events = evaluator
+            .eval(&[noncurrent])
+            .await
+            .expect("lifecycle evaluation should succeed");
+
+        assert_eq!(events[0].action, IlmAction::NoneAction);
+    }
+
+    #[tokio::test]
+    async fn evaluator_allows_noncurrent_expiration_after_default_retention() {
+        let evaluator =
+            Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(1)));
+        let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
+        let noncurrent = ObjectOpts {
+            name: "logs/object".to_string(),
+            mod_time: Some(successor_time - time::Duration::days(1)),
+            successor_mod_time: Some(successor_time),
+            version_id: Some(Uuid::new_v4()),
+            is_latest: false,
+            num_versions: 1,
+            ..Default::default()
+        };
+
+        let events = evaluator
+            .eval(&[noncurrent])
+            .await
+            .expect("lifecycle evaluation should succeed");
+
+        assert_eq!(events[0].action, IlmAction::DeleteVersionAction);
+    }
+
     #[tokio::test]
     async fn evaluator_skips_transition_while_replication_pending() {
         let evaluator = Evaluator::new(latest_transition_lifecycle());

crates/lifecycle/src/object_lock.rs

@@ -14,7 +14,7 @@
 
 use std::collections::HashMap;
 
-use s3s::dto::ObjectLockRetentionMode;
+use s3s::dto::{ObjectLockConfiguration, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use time::{OffsetDateTime, format_description};
 
@@ -43,6 +43,90 @@ pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_d
         .is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
 }
 
+/// Check persisted object-lock metadata and the bucket default retention.
+///
+/// A configured default retention with missing or malformed input is treated
+/// as locked so a lifecycle worker cannot turn incomplete metadata into an
+/// unsafe delete.
+pub fn is_object_locked(
+    user_defined: &HashMap<String, String>,
+    is_delete_marker: bool,
+    config: Option<&ObjectLockConfiguration>,
+    mod_time: Option<OffsetDateTime>,
+) -> bool {
+    if is_delete_marker {
+        return false;
+    }
+    if is_object_locked_by_metadata(user_defined, false) {
+        return true;
+    }
+    if has_explicit_lock_metadata(user_defined) {
+        return !explicit_lock_metadata_is_well_formed(user_defined);
+    }
+
+    let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
+        return false;
+    };
+    let Some(mode) = default_retention.mode.as_ref() else {
+        return true;
+    };
+    if !is_retention_mode(mode.as_str()) {
+        return true;
+    }
+
+    let Some(mod_time) = mod_time else {
+        return true;
+    };
+    let Some(retain_until) = default_retention_until(mod_time, default_retention) else {
+        return true;
+    };
+
+    retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp()
+}
+
+fn has_explicit_lock_metadata(user_defined: &HashMap<String, String>) -> bool {
+    user_defined.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
+        || user_defined.contains_key(X_AMZ_OBJECT_LOCK_MODE.as_str())
+        || user_defined.contains_key(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
+}
+
+fn explicit_lock_metadata_is_well_formed(user_defined: &HashMap<String, String>) -> bool {
+    if user_defined
+        .get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
+        .is_some_and(|value| !value.eq_ignore_ascii_case("ON") && !value.eq_ignore_ascii_case("OFF"))
+    {
+        return false;
+    }
+
+    match (
+        user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
+        user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
+    ) {
+        (None, None) => true,
+        (Some(mode), Some(retain_until)) => {
+            is_retention_mode(mode)
+                && OffsetDateTime::parse(retain_until, &format_description::well_known::Iso8601::DEFAULT).is_ok()
+        }
+        _ => false,
+    }
+}
+
+fn default_retention_until(mod_time: OffsetDateTime, retention: &s3s::dto::DefaultRetention) -> Option<OffsetDateTime> {
+    match (retention.days, retention.years) {
+        (Some(days), None) if days > 0 => Some(mod_time.saturating_add(time::Duration::days(i64::from(days)))),
+        (None, Some(years)) if years > 0 => add_years(mod_time, years),
+        _ => None,
+    }
+}
+
+fn add_years(mod_time: OffsetDateTime, years: i32) -> Option<OffsetDateTime> {
+    let target_year = mod_time.year().checked_add(years)?;
+    mod_time
+        .replace_year(target_year)
+        .or_else(|_| mod_time.replace_day(28).and_then(|date| date.replace_year(target_year)))
+        .ok()
+}
+
 fn is_retention_mode(mode: &str) -> bool {
     mode.eq_ignore_ascii_case(ObjectLockRetentionMode::COMPLIANCE)
         || mode.eq_ignore_ascii_case(ObjectLockRetentionMode::GOVERNANCE)
@@ -52,6 +136,9 @@ fn is_retention_mode(mode: &str) -> bool {
 mod tests {
     use super::*;
 
+    use s3s::dto::{DefaultRetention, ObjectLockEnabled, ObjectLockRule};
+    use time::Duration;
+
     #[test]
     fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
         let mut user_defined = HashMap::new();
@@ -60,4 +147,120 @@ mod tests {
         assert!(is_object_locked_by_metadata(&user_defined, false));
         assert!(!is_object_locked_by_metadata(&user_defined, true));
     }
+
+    fn default_retention_config(days: i32) -> ObjectLockConfiguration {
+        ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: Some(days),
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: None,
+                }),
+            }),
+        }
+    }
+
+    #[test]
+    fn default_retention_blocks_lifecycle_delete_until_expired() {
+        let config = default_retention_config(30);
+        let created = OffsetDateTime::now_utc() - Duration::days(1);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn expired_default_retention_allows_lifecycle_delete() {
+        let config = default_retention_config(1);
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(!is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn missing_mod_time_blocks_default_retention_delete() {
+        let config = default_retention_config(30);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), None));
+    }
+
+    #[test]
+    fn zero_default_retention_days_fail_closed() {
+        let config = default_retention_config(0);
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn zero_default_retention_years_fail_closed() {
+        let config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: None,
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: Some(0),
+                }),
+            }),
+        };
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn default_retention_years_block_lifecycle_delete_until_expired() {
+        let config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: None,
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
+                    years: Some(1),
+                }),
+            }),
+        };
+        let created = OffsetDateTime::now_utc() - Duration::days(1);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn expired_explicit_retention_does_not_reapply_default_retention() {
+        let config = default_retention_config(30);
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            (OffsetDateTime::now_utc() - Duration::days(1))
+                .format(&format_description::well_known::Iso8601::DEFAULT)
+                .expect("expired retention date should format"),
+        );
+
+        assert!(!is_object_locked(&user_defined, false, Some(&config), Some(OffsetDateTime::now_utc())));
+    }
+
+    #[test]
+    fn malformed_explicit_retention_fails_closed() {
+        let config = default_retention_config(1);
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&user_defined, false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn delete_markers_are_not_locked_by_default_retention() {
+        let config = default_retention_config(30);
+
+        assert!(!is_object_locked(&HashMap::new(), true, Some(&config), None));
+    }
 }

crates/obs/src/metrics/collectors/ilm.rs

@@ -54,12 +54,37 @@ pub(crate) struct IlmActionTaskStats {
     pub(crate) value: u64,
 }
 
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmQueueTaskStats {
+    pub(crate) action: String,
+    pub(crate) state: String,
+    pub(crate) value: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmTaskEventStats {
+    pub(crate) action: String,
+    pub(crate) result: String,
+    pub(crate) value: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmBackpressureStats {
+    pub(crate) action: String,
+    pub(crate) reason: String,
+    pub(crate) value: u64,
+}
+
 /// ILM statistics with runtime-local node identity and bounded action/state details.
 #[derive(Debug, Clone, Default)]
 pub(crate) struct IlmRuntimeStats {
     pub(crate) server: String,
     pub(crate) stats: IlmStats,
     pub(crate) action_tasks: Vec<IlmActionTaskStats>,
+    pub(crate) queue_tasks: Vec<IlmQueueTaskStats>,
+    pub(crate) task_events: Vec<IlmTaskEventStats>,
+    pub(crate) backpressure: Vec<IlmBackpressureStats>,
+    pub(crate) versions_scanned: u64,
 }
 
 fn is_live_action_task_state(state: &str) -> bool {
@@ -112,6 +137,30 @@ pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<Promet
             }),
     );
 
+    metrics.extend(stats.queue_tasks.iter().map(|task| {
+        PrometheusMetric::from_descriptor(&ILM_TASKS_MD, task.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, task.action.clone())
+            .with_label_owned(QUEUE_STATE_LABEL, task.state.clone())
+    }));
+    metrics.extend(stats.task_events.iter().map(|event| {
+        PrometheusMetric::from_descriptor(&ILM_TASK_EVENTS_MD, event.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, event.action.clone())
+            .with_label_owned(RESULT_LABEL, event.result.clone())
+    }));
+    metrics.extend(stats.backpressure.iter().map(|event| {
+        PrometheusMetric::from_descriptor(&ILM_QUEUE_BACKPRESSURE_MD, event.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, event.action.clone())
+            .with_label_owned(REASON_LABEL, event.reason.clone())
+    }));
+    metrics.push(
+        PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_BY_SERVER_MD, stats.versions_scanned as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(SOURCE_LABEL, "lifecycle".to_string()),
+    );
+
     metrics
 }
 
@@ -135,6 +184,22 @@ mod tests {
         let runtime_stats = IlmRuntimeStats {
             server: "node1:9000".to_string(),
             stats,
+            queue_tasks: vec![IlmQueueTaskStats {
+                action: "transition".to_string(),
+                state: "pending".to_string(),
+                value: 8,
+            }],
+            task_events: vec![IlmTaskEventStats {
+                action: "transition".to_string(),
+                result: "completed".to_string(),
+                value: 7,
+            }],
+            backpressure: vec![IlmBackpressureStats {
+                action: "transition".to_string(),
+                reason: "queue_full".to_string(),
+                value: 2,
+            }],
+            versions_scanned: 1000000,
             action_tasks: vec![
                 IlmActionTaskStats {
                     action: "expiry".to_string(),
@@ -156,7 +221,7 @@ mod tests {
 
         let metrics = collect_ilm_runtime_metrics(&runtime_stats);
 
-        assert_eq!(metrics.len(), 11);
+        assert_eq!(metrics.len(), 15);
 
         let pending = metrics.iter().find(|m| m.value == 100.0);
         assert!(pending.is_some());
@@ -178,6 +243,44 @@ mod tests {
         });
         assert!(transition_timeout.is_none());
 
+        let transition_queue = metrics.iter().find(|m| {
+            m.name == ILM_TASKS_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == QUEUE_STATE_LABEL && value.as_ref() == "pending")
+        });
+        assert_eq!(transition_queue.map(|metric| metric.value), Some(8.0));
+
+        let completed = metrics.iter().find(|m| {
+            m.name == ILM_TASK_EVENTS_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == RESULT_LABEL && value.as_ref() == "completed")
+        });
+        assert_eq!(completed.map(|metric| metric.value), Some(7.0));
+
+        let backpressure = metrics.iter().find(|m| {
+            m.name == ILM_QUEUE_BACKPRESSURE_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == REASON_LABEL && value.as_ref() == "queue_full")
+        });
+        assert_eq!(backpressure.map(|metric| metric.value), Some(2.0));
+
+        let version_detail = metrics.iter().find(|m| {
+            m.name == ILM_VERSIONS_SCANNED_BY_SERVER_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
+        });
+        assert_eq!(version_detail.map(|metric| metric.value), Some(1000000.0));
+
         let transition_active = metrics.iter().find(|m| {
             m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
                 && m.labels

crates/obs/src/metrics/collectors/mod.rs

@@ -59,9 +59,12 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
 pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
 pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
 pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
-pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
+pub(crate) use ilm::{
+    IlmActionTaskStats, IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmTaskEventStats, collect_ilm_runtime_metrics,
+};
 pub use ilm::{IlmStats, collect_ilm_metrics};
 pub use node::{DiskStats, collect_node_metrics};
+pub(crate) use notification::collect_notification_runtime_metrics;
 pub use notification::{NotificationStats, collect_notification_metrics};
 pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
 pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};

crates/obs/src/metrics/collectors/notification.rs

@@ -19,9 +19,12 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::cluster_notification::{
-    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
-    NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
+    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD, NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD,
+    NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD,
+    NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
+    NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, SERVER,
 };
+use std::borrow::Cow;
 
 /// Notification statistics.
 #[derive(Debug, Clone, Default)]
@@ -49,6 +52,30 @@ pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<Prometheus
     ]
 }
 
+/// Collects the legacy aggregate metrics and node-local runtime siblings.
+pub(crate) fn collect_notification_runtime_metrics(stats: &NotificationStats, server: &str) -> Vec<PrometheusMetric> {
+    let mut metrics = collect_notification_metrics(stats);
+    if server.is_empty() {
+        return metrics;
+    }
+
+    let server_label: Cow<'static, str> = Cow::Owned(server.to_string());
+    metrics.extend([
+        PrometheusMetric::from_descriptor(
+            &NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD,
+            stats.current_send_in_progress as f64,
+        )
+        .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD, stats.events_errors_total as f64)
+            .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD, stats.events_sent_total as f64)
+            .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD, stats.events_skipped_total as f64)
+            .with_label(SERVER, server_label),
+    ]);
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -86,4 +113,32 @@ mod tests {
             assert!(metric.labels.is_empty());
         }
     }
+
+    #[test]
+    fn runtime_metrics_keep_aggregate_and_add_server_siblings() {
+        let stats = NotificationStats {
+            current_send_in_progress: 5,
+            events_errors_total: 10,
+            events_sent_total: 100,
+            events_skipped_total: 2,
+        };
+
+        let metrics = collect_notification_runtime_metrics(&stats, "node1:9000");
+        assert_eq!(metrics.len(), 8);
+        assert_eq!(metrics.iter().filter(|metric| metric.labels.is_empty()).count(), 4);
+        assert_eq!(metrics.iter().filter(|metric| metric.labels.len() == 1).count(), 4);
+        assert!(metrics.iter().filter(|metric| metric.labels.len() == 1).all(|metric| {
+            metric
+                .labels
+                .iter()
+                .any(|(name, value)| *name == SERVER && value == "node1:9000")
+        }));
+    }
+
+    #[test]
+    fn runtime_metrics_do_not_publish_empty_server_series() {
+        let metrics = collect_notification_runtime_metrics(&NotificationStats::default(), "");
+        assert_eq!(metrics.len(), 4);
+        assert!(metrics.iter().all(|metric| metric.labels.is_empty()));
+    }
 }

crates/obs/src/metrics/collectors/scanner.rs

@@ -184,6 +184,15 @@ pub struct ScannerBucketDriveResultStats {
     pub count: u64,
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct ScannerActiveBucketDriveStats {
+    pub source: String,
+    pub bucket: String,
+    pub drive: String,
+    pub count: u64,
+    pub age_seconds: u64,
+}
+
 /// Scanner statistics with runtime-local node identity and bounded source/result details.
 #[derive(Debug, Clone, Default)]
 pub(crate) struct ScannerRuntimeStats {
@@ -195,6 +204,7 @@ pub(crate) struct ScannerRuntimeStats {
     pub(crate) bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
     pub(crate) current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
     pub(crate) last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
+    pub(crate) active_bucket_drive_scans: Vec<ScannerActiveBucketDriveStats>,
 }
 
 /// Collects scanner metrics from the given stats.
@@ -452,6 +462,23 @@ fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&S
             &runtime.last_cycle_bucket_drive_results,
             Some("last"),
         );
+        for active in &runtime.active_bucket_drive_scans {
+            let labels = |metric: PrometheusMetric| {
+                metric
+                    .with_label_owned(SERVER_LABEL, runtime.server.clone())
+                    .with_label_owned(SOURCE_LABEL, active.source.clone())
+                    .with_label_owned(BUCKET_LABEL, active.bucket.clone())
+                    .with_label_owned(DRIVE_LABEL, active.drive.clone())
+            };
+            metrics.push(labels(PrometheusMetric::from_descriptor(
+                &SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD,
+                active.count as f64,
+            )));
+            metrics.push(labels(PrometheusMetric::from_descriptor(
+                &SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD,
+                active.age_seconds as f64,
+            )));
+        }
     }
 
     metrics
@@ -566,6 +593,13 @@ mod tests {
                 result: "error".to_string(),
                 count: 2,
             }],
+            active_bucket_drive_scans: vec![ScannerActiveBucketDriveStats {
+                source: "usage".to_string(),
+                bucket: "photos".to_string(),
+                drive: "/data1".to_string(),
+                count: 2,
+                age_seconds: 7,
+            }],
             stats: ScannerStats {
                 bucket_scans_finished: 100,
                 bucket_scans_started: 100,
@@ -642,7 +676,7 @@ mod tests {
         let metrics = collect_scanner_runtime_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 90);
+        assert_eq!(metrics.len(), 92);
 
         let objects = metrics.iter().find(|m| m.value == 1000000.0);
         assert!(objects.is_some());
@@ -656,6 +690,35 @@ mod tests {
         assert_eq!(active_paths.map(|m| m.value), Some(4.0));
         assert_eq!(active_paths.map(|m| m.labels.len()), Some(0));
 
+        let active_bucket_drive = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name())
+            .expect("active bucket-drive metric");
+        assert_eq!(active_bucket_drive.value, 2.0);
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == SOURCE_LABEL && value == "usage")
+        );
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == BUCKET_LABEL && value == "photos")
+        );
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == DRIVE_LABEL && value == "/data1")
+        );
+        let active_age = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD.get_full_metric_name())
+            .expect("active bucket-drive age metric");
+        assert_eq!(active_age.value, 7.0);
+
         let bucket_drive_result = metrics
             .iter()
             .find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name());

crates/obs/src/metrics/collectors/system_drive.rs

@@ -60,6 +60,10 @@ pub struct DriveDetailedStats {
     pub api_latency_micros: Option<u64>,
     /// Health status (1=healthy, 0=unhealthy)
     pub health: u8,
+    /// Total successful write operations when backed by a real disk metric.
+    pub writes_total: Option<u64>,
+    /// Total successful delete operations when backed by a real disk metric.
+    pub deletes_total: Option<u64>,
     /// Reads per second when backed by a real iostat sample
     pub reads_per_sec: Option<f64>,
     /// Kilobytes read per second when backed by a real iostat sample
@@ -282,6 +286,12 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail
         if let Some(value) = stat.stats.perc_util {
             push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, value, server_label, drive_label);
         }
+        if let Some(value) = stat.stats.writes_total {
+            push_drive_metric(&mut metrics, &DRIVE_WRITES_TOTAL_MD, value as f64, server_label, drive_label);
+        }
+        if let Some(value) = stat.stats.deletes_total {
+            push_drive_metric(&mut metrics, &DRIVE_DELETES_TOTAL_MD, value as f64, server_label, drive_label);
+        }
         if let Some(labels) = &topology_labels {
             if let Some(disk_id) = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty()) {
                 metrics.push(
@@ -449,6 +459,8 @@ mod tests {
                 waiting_io: Some(3),
                 api_latency_micros: Some(1500),
                 health: 1,
+                writes_total: Some(11),
+                deletes_total: Some(4),
                 reads_per_sec: Some(100.0),
                 reads_kb_per_sec: Some(1024.0),
                 reads_await: Some(5.5),
@@ -462,7 +474,7 @@ mod tests {
         let metrics = collect_drive_runtime_detailed_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 34);
+        assert_eq!(metrics.len(), 36);
 
         // Verify total bytes metric
         let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
@@ -503,6 +515,8 @@ mod tests {
                 API_LABEL,
             ],
         );
+        assert_metric_label_keys(&metrics, &DRIVE_WRITES_TOTAL_MD, 11.0, &[SERVER_LABEL, DRIVE_LABEL]);
+        assert_metric_label_keys(&metrics, &DRIVE_DELETES_TOTAL_MD, 4.0, &[SERVER_LABEL, DRIVE_LABEL]);
     }
 
     #[test]
@@ -524,6 +538,8 @@ mod tests {
             waiting_io: None,
             api_latency_micros: None,
             health: 1,
+            writes_total: None,
+            deletes_total: None,
             reads_per_sec: None,
             reads_kb_per_sec: None,
             reads_await: None,

crates/obs/src/metrics/scheduler.rs

@@ -62,7 +62,7 @@ use crate::metrics::collectors::{
     collect_memory_metrics,
     collect_network_metrics,
     collect_node_metrics,
-    collect_notification_metrics,
+    collect_notification_runtime_metrics,
     collect_notification_target_runtime_metrics,
     collect_process_attributes,
     collect_process_cpu_metrics,
@@ -120,12 +120,13 @@ use crate::metrics::schema::notification_target::{
 };
 use crate::metrics::schema::scanner::{
     BUCKET_LABEL as SCANNER_BUCKET_LABEL, CYCLE_SCOPE_LABEL as SCANNER_CYCLE_SCOPE_LABEL, DRIVE_LABEL as SCANNER_DRIVE_LABEL,
-    RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
+    RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD, SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD,
+    SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, SOURCE_LABEL as SCANNER_SOURCE_LABEL,
 };
 use crate::metrics::schema::system_drive::{
-    API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_HEALING_MD,
-    DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD, DRIVE_SCANNING_MD,
-    POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL,
+    API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_DELETES_TOTAL_MD,
+    DRIVE_HEALING_MD, DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD,
+    DRIVE_SCANNING_MD, DRIVE_WRITES_TOTAL_MD, POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL,
 };
 use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
 use crate::metrics::stats_collector::{
@@ -303,15 +304,33 @@ type AuditTargetKey = (String, String); // (server, target_id)
 type NotificationLegacyTargetKey = (String, String); // (target_id, target_type)
 type NotificationTargetKey = (String, String, String); // (server, target_id, target_type)
 type DriveTopologyKey = (String, String, String, String, String); // (server, drive, pool, set, drive_index)
+type DriveBasicKey = (String, String); // (server, drive)
 type DriveTopologyApiKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, api)
 type DriveInfoKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, disk_id)
 type ScannerCycleBucketDriveResultKey = (String, String, String, String, String); // (server, cycle_scope, bucket, drive, result)
 type ScannerBucketDriveResultKey = (String, String, String, String); // (server, bucket, drive, result)
+type ScannerActiveBucketDriveKey = (String, String, String, String); // (server, source, bucket, drive)
 
 fn drive_info_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveInfoKey> {
     stats.iter().filter_map(drive_info_key).collect()
 }
 
+fn drive_basic_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveBasicKey> {
+    stats
+        .iter()
+        .map(|stat| (stat.stats.server.clone(), stat.stats.drive.clone()))
+        .collect()
+}
+
+fn retire_drive_basic_metric_series(key: &DriveBasicKey) -> usize {
+    let labels = [
+        (SERVER_LABEL, Cow::Owned(key.0.clone())),
+        (DRIVE_LABEL, Cow::Owned(key.1.clone())),
+    ];
+    retire_metric_series(&DRIVE_WRITES_TOTAL_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&DRIVE_DELETES_TOTAL_MD.get_full_metric_name(), &labels)
+}
+
 fn drive_topology_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveTopologyKey> {
     stats.iter().filter_map(drive_topology_key).collect()
 }
@@ -469,6 +488,25 @@ fn retire_scanner_bucket_drive_result_metric_series(key: &ScannerBucketDriveResu
     retire_metric_series(&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name(), &labels)
 }
 
+fn scanner_active_bucket_drive_live_keys(stats: &ScannerRuntimeStats) -> HashSet<ScannerActiveBucketDriveKey> {
+    stats
+        .active_bucket_drive_scans
+        .iter()
+        .map(|active| (stats.server.clone(), active.source.clone(), active.bucket.clone(), active.drive.clone()))
+        .collect()
+}
+
+fn retire_scanner_active_bucket_drive_metric_series(key: &ScannerActiveBucketDriveKey) -> usize {
+    let labels = [
+        (SERVER_LABEL, Cow::Owned(key.0.clone())),
+        (SCANNER_SOURCE_LABEL, Cow::Owned(key.1.clone())),
+        (SCANNER_BUCKET_LABEL, Cow::Owned(key.2.clone())),
+        (SCANNER_DRIVE_LABEL, Cow::Owned(key.3.clone())),
+    ];
+    retire_metric_series(&SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD.get_full_metric_name(), &labels)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
 pub struct MetricsRuntimeCollectorHealthSnapshot {
     pub healthy_collectors: u8,
@@ -1841,6 +1879,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
     let token_clone = token.clone();
     tokio::spawn(async move {
         let mut interval = metrics_interval(node_interval, Duration::ZERO);
+        let mut prev_drive_basic_keys: HashSet<DriveBasicKey> = HashSet::new();
         let mut prev_drive_info_keys: HashSet<DriveInfoKey> = HashSet::new();
         let mut prev_drive_topology_keys: HashSet<DriveTopologyKey> = HashSet::new();
         let mut prev_drive_topology_api_keys: HashSet<DriveTopologyApiKey> = HashSet::new();
@@ -1851,6 +1890,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                     run_metrics_collector_tick(health, MetricsCollectorTaskId::NodeDiskStats, "node_disk_stats", async {
                         let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_runtime_stats().await;
                         let current_drive_info_keys = drive_info_live_keys(&drive_stats);
+                        let current_drive_basic_keys = drive_basic_live_keys(&drive_stats);
                         let current_drive_topology_keys = drive_topology_live_keys(&drive_stats);
                         let current_drive_topology_api_keys = drive_topology_api_live_keys(&drive_stats);
                         let retire_drive_info_keys = if has_seen_drive_info_snapshot {
@@ -1858,6 +1898,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                         } else {
                             Vec::new()
                         };
+                        let retire_drive_basic_keys = if has_seen_drive_info_snapshot {
+                            prev_drive_basic_keys.difference(&current_drive_basic_keys).cloned().collect::<Vec<_>>()
+                        } else {
+                            Vec::new()
+                        };
                         let retire_drive_topology_keys = if has_seen_drive_info_snapshot {
                             prev_drive_topology_keys.difference(&current_drive_topology_keys).cloned().collect::<Vec<_>>()
                         } else {
@@ -1872,6 +1917,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             Vec::new()
                         };
                         prev_drive_info_keys = current_drive_info_keys;
+                        prev_drive_basic_keys = current_drive_basic_keys;
                         prev_drive_topology_keys = current_drive_topology_keys;
                         prev_drive_topology_api_keys = current_drive_topology_api_keys;
                         has_seen_drive_info_snapshot = true;
@@ -1882,6 +1928,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                         for key in retire_drive_info_keys {
                             let _ = retire_drive_info_metric_series(&key);
                         }
+                        for key in retire_drive_basic_keys {
+                            let _ = retire_drive_basic_metric_series(&key);
+                        }
                         for key in retire_drive_topology_keys {
                             let _ = retire_drive_topology_metric_series(&key);
                         }
@@ -2106,14 +2155,14 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                 _ = interval.tick() => {
                     run_metrics_collector_tick(health, MetricsCollectorTaskId::NotificationStats, "notification_stats", async {
                         let snapshot = notification_metrics_snapshot();
-                        let mut metrics = collect_notification_metrics(&NotificationStats {
+                        let server = current_local_node_identity();
+                        let mut metrics = collect_notification_runtime_metrics(&NotificationStats {
                             current_send_in_progress: snapshot.current_send_in_progress,
                             events_errors_total: snapshot.events_errors_total,
                             events_sent_total: snapshot.events_sent_total,
                             events_skipped_total: snapshot.events_skipped_total,
-                        });
+                        }, &server);
 
-                        let server = current_local_node_identity();
                         let target_stats = notification_target_metrics().await
                             .into_iter()
                             .map(|snapshot| NotificationTargetRuntimeStats {
@@ -2173,6 +2222,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         let mut has_seen_scanner_snapshot = false;
         let mut prev_scanner_cycle_bucket_drive_result_keys: HashSet<ScannerCycleBucketDriveResultKey> = HashSet::new();
         let mut prev_scanner_bucket_drive_result_keys: HashSet<ScannerBucketDriveResultKey> = HashSet::new();
+        let mut prev_scanner_active_bucket_drive_keys: HashSet<ScannerActiveBucketDriveKey> = HashSet::new();
         loop {
             tokio::select! {
                 _ = interval.tick() => {
@@ -2189,9 +2239,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
 
                             let mut retire_scanner_cycle_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_bucket_drive_result_keys = Vec::new();
+                            let mut retire_scanner_active_bucket_drive_keys = Vec::new();
                             if let Some(stats) = collect_scanner_runtime_metric_stats().await {
                                 let current_cycle_keys = scanner_cycle_bucket_drive_result_live_keys(&stats);
                                 let current_keys = scanner_bucket_drive_result_live_keys(&stats);
+                                let current_active_keys = scanner_active_bucket_drive_live_keys(&stats);
                                 if has_seen_scanner_snapshot {
                                     retire_scanner_cycle_bucket_drive_result_keys = prev_scanner_cycle_bucket_drive_result_keys
                                         .difference(&current_cycle_keys)
@@ -2201,9 +2253,14 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                                         .difference(&current_keys)
                                         .cloned()
                                         .collect();
+                                    retire_scanner_active_bucket_drive_keys = prev_scanner_active_bucket_drive_keys
+                                        .difference(&current_active_keys)
+                                        .cloned()
+                                        .collect();
                                 }
                                 prev_scanner_cycle_bucket_drive_result_keys = current_cycle_keys;
                                 prev_scanner_bucket_drive_result_keys = current_keys;
+                                prev_scanner_active_bucket_drive_keys = current_active_keys;
                                 has_seen_scanner_snapshot = true;
                                 metrics.extend(collect_scanner_runtime_metrics(&stats));
                             }
@@ -2217,6 +2274,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             for key in retire_scanner_bucket_drive_result_keys {
                                 let _ = retire_scanner_bucket_drive_result_metric_series(&key);
                             }
+                            for key in retire_scanner_active_bucket_drive_keys {
+                                let _ = retire_scanner_active_bucket_drive_metric_series(&key);
+                            }
                         },
                     ).await;
                 }
@@ -2495,6 +2555,7 @@ fn collect_system_monitoring_metrics(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::metrics::collectors::scanner::ScannerActiveBucketDriveStats;
     use std::collections::{HashMap, HashSet};
     use std::time::Duration;
     use tokio::time::Instant;
@@ -2723,6 +2784,30 @@ mod tests {
         assert!(current.contains(&("server-a".to_string(), "logs".to_string(), "/data1".to_string(), "success".to_string(),)));
     }
 
+    #[test]
+    fn scanner_active_bucket_drive_keys_detect_completed_scans() {
+        let previous = scanner_active_bucket_drive_live_keys(&ScannerRuntimeStats {
+            server: "server-a".to_string(),
+            active_bucket_drive_scans: vec![ScannerActiveBucketDriveStats {
+                source: "usage".to_string(),
+                bucket: "photos".to_string(),
+                drive: "/data1".to_string(),
+                count: 1,
+                age_seconds: 3,
+            }],
+            ..Default::default()
+        });
+        let current = scanner_active_bucket_drive_live_keys(&ScannerRuntimeStats {
+            server: "server-a".to_string(),
+            ..Default::default()
+        });
+        assert!(
+            previous
+                .difference(&current)
+                .any(|key| key == &("server-a".to_string(), "usage".to_string(), "photos".to_string(), "/data1".to_string()))
+        );
+    }
+
     #[test]
     fn replication_proxy_bucket_keys_detect_removed_buckets() {
         let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {

crates/obs/src/metrics/schema/cluster_notification.rs

@@ -15,6 +15,10 @@
 use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
 use std::sync::LazyLock;
 
+pub const SERVER: &str = "server";
+
+const SERVER_LABELS: [&str; 1] = [SERVER];
+
 pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::NotificationCurrentSendInProgress,
@@ -24,6 +28,15 @@ pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor>
     )
 });
 
+pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("current_send_in_progress_by_server".to_string()),
+        "Number of concurrent async Send calls active to all targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsErrorsTotal,
@@ -33,6 +46,15 @@ pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = Laz
     )
 });
 
+pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_errors_total_by_server".to_string()),
+        "Events that failed to be sent to the targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsSentTotal,
@@ -42,6 +64,15 @@ pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyL
     )
 });
 
+pub static NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_sent_total_by_server".to_string()),
+        "Total number of events sent to the targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsSkippedTotal,
@@ -50,3 +81,12 @@ pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = La
         subsystems::NOTIFICATION,
     )
 });
+
+pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_skipped_total_by_server".to_string()),
+        "Notification dispatch attempts skipped before delivery by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -371,6 +371,8 @@ pub enum MetricName {
     DriveWaitingIO,
     DriveAPILatencyMicros,
     DriveHealth,
+    DriveWritesTotal,
+    DriveDeletesTotal,
 
     DriveOfflineCount,
     DriveOnlineCount,
@@ -780,6 +782,8 @@ impl MetricName {
             Self::DriveWaitingIO => "waiting_io".to_string(),
             Self::DriveAPILatencyMicros => "api_latency_micros".to_string(),
             Self::DriveHealth => "health".to_string(),
+            Self::DriveWritesTotal => "writes_total".to_string(),
+            Self::DriveDeletesTotal => "deletes_total".to_string(),
 
             Self::DriveOfflineCount => "offline_count".to_string(),
             Self::DriveOnlineCount => "online_count".to_string(),

crates/obs/src/metrics/schema/ilm.rs

@@ -18,6 +18,10 @@ use std::sync::LazyLock;
 pub const SERVER_LABEL: &str = "server";
 pub const ACTION_LABEL: &str = "action";
 pub const STATE_LABEL: &str = "state";
+pub const QUEUE_STATE_LABEL: &str = "queue_state";
+pub const RESULT_LABEL: &str = "result";
+pub const REASON_LABEL: &str = "reason";
+pub const SOURCE_LABEL: &str = "source";
 
 pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
@@ -28,6 +32,33 @@ pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     )
 });
 
+pub static ILM_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("tasks".to_string()),
+        "Current ILM task counts by server, action, and queue state",
+        &[SERVER_LABEL, ACTION_LABEL, QUEUE_STATE_LABEL],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TASK_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("task_events_total".to_string()),
+        "ILM task events by server, action, and result",
+        &[SERVER_LABEL, ACTION_LABEL, RESULT_LABEL],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_QUEUE_BACKPRESSURE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("queue_backpressure_total".to_string()),
+        "ILM queue backpressure events by server, action, and reason",
+        &[SERVER_LABEL, ACTION_LABEL, REASON_LABEL],
+        subsystems::ILM,
+    )
+});
+
 pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::IlmExpiryPendingTasks,
@@ -108,3 +139,12 @@ pub static ILM_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
         subsystems::ILM,
     )
 });
+
+pub static ILM_VERSIONS_SCANNED_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("versions_scanned_by_server".to_string()),
+        "ILM lifecycle-checked object versions by server and source",
+        &[SERVER_LABEL, SOURCE_LABEL],
+        subsystems::ILM,
+    )
+});

crates/obs/src/metrics/schema/scanner.rs

@@ -59,6 +59,24 @@ pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock<MetricDescriptor> = La
     )
 });
 
+pub static SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("active_bucket_drive_scans".to_string()),
+        "Current active scanner bucket-drive scans by server, source, bucket, and drive",
+        &[SERVER_LABEL, SOURCE_LABEL, BUCKET_LABEL, DRIVE_LABEL],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("active_bucket_drive_scan_age_seconds".to_string()),
+        "Age of the oldest active scanner bucket-drive scan by server, source, bucket, and drive",
+        &[SERVER_LABEL, SOURCE_LABEL, BUCKET_LABEL, DRIVE_LABEL],
+        subsystems::SCANNER,
+    )
+});
+
 pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::ScannerBucketScansFinished,

crates/obs/src/metrics/schema/system_drive.rs

@@ -259,6 +259,24 @@ pub static DRIVE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     )
 });
 
+pub static DRIVE_WRITES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveWritesTotal,
+        "Total successful write operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_DELETES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveDeletesTotal,
+        "Total successful delete operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
 pub static DRIVE_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> =
     LazyLock::new(|| new_gauge_md(MetricName::DriveOfflineCount, "Count of offline drives", &[], subsystems::SYSTEM_DRIVE));
 

crates/obs/src/metrics/stats_collector.rs

@@ -18,15 +18,15 @@
 //! RustFS internal sources (storage layer, bucket monitor, system info)
 //! and convert them to the Stats structs used by collectors.
 
-use crate::metrics::collectors::scanner::{ScannerBucketDriveResultStats, ScannerSourceWorkStats};
+use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
 use crate::metrics::collectors::{
     ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
     BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
     BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
     ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
-    DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmRuntimeStats, IlmStats,
-    MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerRuntimeStats,
-    ScannerStats,
+    DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
+    IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
+    ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
@@ -38,7 +38,10 @@ use crate::metrics::{
 use crate::node_identity::current_local_node_identity;
 use jiff::Timestamp;
 use rustfs_common::heal_channel::HealScanMode;
-use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics};
+use rustfs_common::metrics::{
+    ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot,
+    global_metrics,
+};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
 use rustfs_io_metrics::{
     ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
@@ -334,7 +337,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
         return 0;
     }
 
-    u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
+    u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
 }
 
 fn scanner_scan_mode_code(scan_mode: &str) -> u64 {
@@ -835,6 +838,8 @@ pub(crate) async fn collect_disk_and_system_drive_runtime_stats()
                         drive_api_latency_micros(metrics.last_minute.values().map(|action| (action.count, action.acc_time)))
                     }),
                     health: if is_online { 1 } else { 0 },
+                    writes_total: disk.metrics.as_ref().map(|metrics| metrics.total_writes),
+                    deletes_total: disk.metrics.as_ref().map(|metrics| metrics.total_deletes),
                     reads_per_sec: None,
                     reads_kb_per_sec: None,
                     reads_await: None,
@@ -1275,6 +1280,110 @@ fn ilm_action_task_stats(ilm: &ObsIlmRuntimeSnapshot) -> Vec<IlmActionTaskStats>
     ]
 }
 
+fn ilm_queue_task_stats(metrics: &ScannerMetricsReport) -> Vec<IlmQueueTaskStats> {
+    let expiry = &metrics.lifecycle_expiry;
+    let transition = &metrics.lifecycle_transition;
+    vec![
+        IlmQueueTaskStats {
+            action: "expiry".to_string(),
+            state: "pending".to_string(),
+            value: expiry.current_queued,
+        },
+        IlmQueueTaskStats {
+            action: "expiry".to_string(),
+            state: "active".to_string(),
+            value: expiry.current_active,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "pending".to_string(),
+            value: transition.current_queued,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "active".to_string(),
+            value: transition.current_active,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "compensation_running".to_string(),
+            value: transition.compensation_running,
+        },
+    ]
+}
+
+fn ilm_task_event_stats(metrics: &ScannerMetricsReport) -> Vec<IlmTaskEventStats> {
+    let expiry = &metrics.lifecycle_expiry;
+    let transition = &metrics.lifecycle_transition;
+    vec![
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "queued".to_string(),
+            value: expiry.scanner_queued,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "missed".to_string(),
+            value: expiry.scanner_missed,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "blocked".to_string(),
+            value: expiry.scanner_blocked,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "not_enqueued".to_string(),
+            value: expiry.scanner_not_enqueued,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "failed".to_string(),
+            value: expiry.delete_failed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "queued".to_string(),
+            value: transition.scanner_queued,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "missed".to_string(),
+            value: transition.scanner_missed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "completed".to_string(),
+            value: transition.completed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "failed".to_string(),
+            value: transition.failed,
+        },
+    ]
+}
+
+fn ilm_backpressure_stats(metrics: &ScannerMetricsReport) -> Vec<IlmBackpressureStats> {
+    vec![
+        IlmBackpressureStats {
+            action: "expiry".to_string(),
+            reason: "queue_missed".to_string(),
+            value: metrics.lifecycle_expiry.queue_missed,
+        },
+        IlmBackpressureStats {
+            action: "transition".to_string(),
+            reason: "queue_full".to_string(),
+            value: metrics.lifecycle_transition.queue_full,
+        },
+        IlmBackpressureStats {
+            action: "transition".to_string(),
+            reason: "send_timeout".to_string(),
+            value: metrics.lifecycle_transition.queue_send_timeout,
+        },
+    ]
+}
+
 /// Collect ILM metrics from the current lifecycle runtime state.
 pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
     collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)
@@ -1288,6 +1397,10 @@ pub(crate) async fn collect_ilm_runtime_metric_stats() -> Option<IlmRuntimeStats
     Some(IlmRuntimeStats {
         server: current_local_node_identity(),
         action_tasks: ilm_action_task_stats(&ilm),
+        queue_tasks: ilm_queue_task_stats(&metrics),
+        task_events: ilm_task_event_stats(&metrics),
+        backpressure: ilm_backpressure_stats(&metrics),
+        versions_scanned,
         stats: IlmStats {
             expiry_pending_tasks: ilm.expiry_pending_tasks,
             transition_active_tasks: ilm.transition_active_tasks,
@@ -1377,6 +1490,27 @@ fn scanner_bucket_drive_result_stats(results: &[ScannerBucketDriveResultSnapshot
     stats
 }
 
+fn scanner_active_bucket_drive_stats(results: &[ScannerActiveBucketDriveSnapshot]) -> Vec<ScannerActiveBucketDriveStats> {
+    let mut stats = results
+        .iter()
+        .filter(|result| !result.source.is_empty() && !result.bucket.is_empty() && !result.drive.is_empty() && result.count > 0)
+        .map(|result| ScannerActiveBucketDriveStats {
+            source: result.source.clone(),
+            bucket: result.bucket.clone(),
+            drive: result.drive.clone(),
+            count: result.count,
+            age_seconds: result.age_seconds,
+        })
+        .collect::<Vec<_>>();
+    stats.sort_by(|left, right| {
+        left.source
+            .cmp(&right.source)
+            .then_with(|| left.bucket.cmp(&right.bucket))
+            .then_with(|| left.drive.cmp(&right.drive))
+    });
+    stats
+}
+
 pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     collect_scanner_runtime_metric_stats().await.map(|stats| stats.stats)
 }
@@ -1418,6 +1552,7 @@ pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option<ScannerRunt
             &runtime_details.current_cycle_bucket_drive_results,
         ),
         last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results),
+        active_bucket_drive_scans: scanner_active_bucket_drive_stats(&runtime_details.active_bucket_drive_scans),
         stats: ScannerStats {
             bucket_scans_finished,
             bucket_scans_started,
@@ -1984,6 +2119,72 @@ mod tests {
         assert_eq!(scanner_lifecycle_checked_versions(&report), 37);
     }
 
+    #[test]
+    fn ilm_detail_stats_keep_expiry_and_transition_results_separate() {
+        let report = ScannerMetricsReport {
+            lifecycle_expiry: rustfs_common::metrics::ScannerLifecycleExpirySnapshot {
+                current_queued: 2,
+                current_active: 1,
+                scanner_queued: 10,
+                scanner_missed: 3,
+                delete_failed: 4,
+                ..Default::default()
+            },
+            lifecycle_transition: rustfs_common::metrics::ScannerLifecycleTransitionSnapshot {
+                current_queued: 5,
+                current_active: 6,
+                queue_full: 7,
+                queue_send_timeout: 8,
+                scanner_queued: 11,
+                completed: 12,
+                failed: 13,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let queues = ilm_queue_task_stats(&report);
+        assert!(
+            queues
+                .iter()
+                .any(|task| task.action == "expiry" && task.state == "pending" && task.value == 2)
+        );
+        assert!(
+            queues
+                .iter()
+                .any(|task| task.action == "transition" && task.state == "active" && task.value == 6)
+        );
+
+        let events = ilm_task_event_stats(&report);
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "expiry" && event.result == "failed" && event.value == 4)
+        );
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "transition" && event.result == "completed" && event.value == 12)
+        );
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "transition" && event.result == "failed" && event.value == 13)
+        );
+
+        let backpressure = ilm_backpressure_stats(&report);
+        assert!(
+            backpressure
+                .iter()
+                .any(|event| event.action == "transition" && event.reason == "queue_full" && event.value == 7)
+        );
+        assert!(
+            backpressure
+                .iter()
+                .any(|event| event.action == "transition" && event.reason == "send_timeout" && event.value == 8)
+        );
+    }
+
     #[test]
     fn scanner_source_work_stats_sorts_and_skips_empty_source() {
         let stats = scanner_source_work_stats(&[

crates/policy/src/policy/function/date.rs

@@ -31,7 +31,7 @@ impl DateFunc {
                 return false;
             };
 
-            if !op(&inner.values.0, &rv) {
+            if !op(&rv, &inner.values.0) {
                 return false;
             }
         }
@@ -95,6 +95,7 @@ mod tests {
         key_name::KeyName::{self, *},
         key_name::S3KeyName::*,
     };
+    use std::collections::HashMap;
     use test_case::test_case;
     use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 
@@ -122,4 +123,16 @@ mod tests {
         assert_eq!(v, expect);
         Ok(())
     }
+
+    #[test]
+    fn evaluate_compares_request_date_to_policy_date() {
+        let function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
+        let later = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2099-01-01T00:00:00Z".to_string()])]);
+        let earlier = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2029-01-01T00:00:00Z".to_string()])]);
+
+        assert!(function.evaluate(OffsetDateTime::gt, &later));
+        assert!(!function.evaluate(OffsetDateTime::gt, &earlier));
+        assert!(function.evaluate(OffsetDateTime::lt, &earlier));
+        assert!(!function.evaluate(OffsetDateTime::lt, &later));
+    }
 }

crates/protocols/Cargo.toml

@@ -106,7 +106,7 @@ swift = [
     "dep:base64",
     "dep:async-compression",
 ]
-webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
+webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
 sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
 
 [dependencies]

crates/protocols/src/common/client/s3.rs

@@ -15,6 +15,9 @@
 use async_trait::async_trait;
 use s3s::dto::*;
 
+#[cfg(feature = "webdav")]
+use crate::common::session::SessionContext;
+
 #[async_trait]
 pub trait StorageBackend: Send + Sync {
     /// Error type for this storage backend
@@ -65,8 +68,24 @@ pub trait StorageBackend: Send + Sync {
         access_key: &str,
         secret_key: &str,
     ) -> Result<ListObjectsV2Output, Self::Error>;
-    /// List all buckets (requires authentication)
+    /// List all buckets (requires authentication).
     async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error>;
+    /// List buckets visible to the authenticated session.
+    ///
+    /// Backends that implement this must apply per-bucket authorization. The default denies the
+    /// request so existing backends cannot expose unfiltered bucket names.
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        _session_context: &SessionContext,
+        _request_headers: &http::HeaderMap,
+        _secure_transport: bool,
+    ) -> s3s::S3Result<ListBucketsOutput> {
+        Err(s3s::S3Error::with_message(
+            s3s::S3ErrorCode::AccessDenied,
+            "Session-aware bucket listing is not supported",
+        ))
+    }
     /// Create a new bucket
     async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error>;
     /// Delete a bucket (must be empty)

crates/protocols/src/common/dummy_storage.rs

@@ -30,6 +30,8 @@
 //! SessionContext type in common::session.
 
 use crate::common::client::s3::StorageBackend;
+#[cfg(feature = "webdav")]
+use crate::common::session::SessionContext;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures_util::stream::{self, StreamExt};
@@ -140,6 +142,8 @@ struct Inner {
     head_bucket: VecDeque<Result<HeadBucketOutput, DummyError>>,
     list_objects_v2: VecDeque<Result<ListObjectsV2Output, DummyError>>,
     list_buckets: VecDeque<Result<ListBucketsOutput, DummyError>>,
+    session_list_buckets: VecDeque<s3s::S3Result<ListBucketsOutput>>,
+    last_session_list_context: Option<(http::HeaderMap, bool)>,
     create_bucket: VecDeque<Result<CreateBucketOutput, DummyError>>,
     delete_bucket: VecDeque<Result<DeleteBucketOutput, DummyError>>,
     copy_object: VecDeque<Result<CopyObjectOutput, DummyError>>,
@@ -193,6 +197,8 @@ impl Inner {
             head_bucket: VecDeque::new(),
             list_objects_v2: VecDeque::new(),
             list_buckets: VecDeque::new(),
+            session_list_buckets: VecDeque::new(),
+            last_session_list_context: None,
             create_bucket: VecDeque::new(),
             delete_bucket: VecDeque::new(),
             copy_object: VecDeque::new(),
@@ -301,6 +307,31 @@ impl DummyBackend {
             .push_back(Ok(CreateBucketOutput::default()));
     }
 
+    /// Queue a legacy list_buckets response.
+    pub fn queue_list_buckets_ok(&self, output: ListBucketsOutput) {
+        self.inner.lock().expect("lock").list_buckets.push_back(Ok(output));
+    }
+
+    /// Queue a legacy list_buckets error.
+    pub fn queue_list_buckets_err(&self, error: DummyError) {
+        self.inner.lock().expect("lock").list_buckets.push_back(Err(error));
+    }
+
+    /// Queue a session-aware list_buckets response.
+    pub fn queue_session_list_buckets_ok(&self, output: ListBucketsOutput) {
+        self.inner.lock().expect("lock").session_list_buckets.push_back(Ok(output));
+    }
+
+    /// Queue a session-aware list_buckets error.
+    pub fn queue_session_list_buckets_err(&self, error: s3s::S3Error) {
+        self.inner.lock().expect("lock").session_list_buckets.push_back(Err(error));
+    }
+
+    /// Return the context from the last session-aware list_buckets request.
+    pub fn last_session_list_context(&self) -> Option<(http::HeaderMap, bool)> {
+        self.inner.lock().expect("lock").last_session_list_context.clone()
+    }
+
     /// Queue a put_object error. Used by the commit_write retry tests
     /// to script SlowDown / AccessDenied sequences against the
     /// rustfs_utils::retry::is_s3code_in_message_retryable predicate.
@@ -690,13 +721,29 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn list_buckets(&self, _ak: &str, _sk: &str) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
         match self.inner.lock().expect("lock").list_buckets.pop_front() {
             Some(r) => r,
             None => Ok(ListBucketsOutput::default()),
         }
     }
 
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        session_context: &SessionContext,
+        request_headers: &http::HeaderMap,
+        secure_transport: bool,
+    ) -> s3s::S3Result<ListBucketsOutput> {
+        let _ = session_context;
+        let mut inner = self.inner.lock().expect("lock");
+        inner.last_session_list_context = Some((request_headers.clone(), secure_transport));
+        inner
+            .session_list_buckets
+            .pop_front()
+            .unwrap_or_else(|| Ok(ListBucketsOutput::default()))
+    }
+
     async fn create_bucket(&self, _bucket: &str, _ak: &str, _sk: &str) -> Result<CreateBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").create_bucket.pop_front() {
             Some(r) => r,

crates/protocols/src/webdav/driver.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::common::client::s3::StorageBackend as S3StorageBackend;
-use crate::common::gateway::{S3Action, authorize_operation};
+use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation};
 use crate::common::session::SessionContext;
 use bytes::Bytes;
 use dav_server::davpath::DavPath;
@@ -24,6 +24,7 @@ use futures_util::{FutureExt, StreamExt, stream};
 use percent_encoding::percent_decode_str;
 use rustfs_utils::MaskedAccessKey;
 use rustfs_utils::path;
+use s3s::S3ErrorCode;
 use s3s::dto::*;
 use std::fmt::Debug;
 use std::io::SeekFrom;
@@ -457,6 +458,10 @@ where
     storage: S,
     /// Session context for authorization
     session_context: Arc<SessionContext>,
+    /// Policy-safe WebDAV request headers used by IAM conditions.
+    request_headers: Option<http::HeaderMap>,
+    /// Whether the WebDAV connection uses TLS.
+    secure_transport: bool,
 }
 
 enum ResolvedPath {
@@ -490,6 +495,8 @@ where
         Self {
             storage: self.storage.clone(),
             session_context: self.session_context.clone(),
+            request_headers: self.request_headers.clone(),
+            secure_transport: self.secure_transport,
         }
     }
 }
@@ -503,9 +510,18 @@ where
         Self {
             storage,
             session_context,
+            request_headers: None,
+            secure_transport: false,
         }
     }
 
+    /// Attach the request context used by IAM policy conditions.
+    pub fn with_request_context(mut self, request_headers: http::HeaderMap, secure_transport: bool) -> Self {
+        self.request_headers = Some(request_headers);
+        self.secure_transport = secure_transport;
+        self
+    }
+
     fn credentials(&self) -> (&str, &str) {
         (
             &self.session_context.principal.user_identity.credentials.access_key,
@@ -799,50 +815,41 @@ where
     /// List all buckets (for root path)
     async fn list_buckets(&self) -> FsResult<Vec<WebDavDirEntry>> {
         match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await {
-            Ok(_) => {}
-            Err(_e) => {
-                return Err(FsError::Forbidden);
+            Ok(()) => {
+                let (access_key, secret_key) = self.credentials();
+                return match self.storage.list_buckets(access_key, secret_key).await {
+                    Ok(output) => Ok(Self::bucket_entries(output)),
+                    Err(error) => {
+                        error!(
+                            event = EVENT_WEBDAV_BUCKET_LIST_FAILED,
+                            component = LOG_COMPONENT_PROTOCOLS,
+                            subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                            error = %error,
+                            access_key = %MaskedAccessKey(access_key),
+                            "webdav bucket list failed"
+                        );
+                        Err(FsError::GeneralFailure)
+                    }
+                };
             }
+            Err(AuthorizationError::AccessDenied) => {}
+            Err(AuthorizationError::IamUnavailable) => return Err(FsError::GeneralFailure),
         }
 
-        match self
+        let Some(request_headers) = self.request_headers.as_ref() else {
+            return Err(FsError::Forbidden);
+        };
+        let result = self
             .storage
-            .list_buckets(
-                &self.session_context.principal.user_identity.credentials.access_key,
-                &self.session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
-            Ok(output) => {
-                let mut entries = Vec::new();
-                if let Some(buckets) = output.buckets {
-                    for bucket in buckets {
-                        if let Some(ref bucket_name) = bucket.name {
-                            let modified = bucket
-                                .creation_date
-                                .map(|dt| {
-                                    let offset_dt: time::OffsetDateTime = dt.into();
-                                    SystemTime::from(offset_dt)
-                                })
-                                .unwrap_or_else(SystemTime::now);
+            .list_buckets_for_session(&self.session_context, request_headers, self.secure_transport)
+            .await;
 
-                            entries.push(WebDavDirEntry {
-                                name: bucket_name.clone(),
-                                metadata: WebDavMetaData {
-                                    size: 0,
-                                    modified,
-                                    created: modified,
-                                    is_dir: true,
-                                    etag: None,
-                                    content_type: None,
-                                },
-                            });
-                        }
-                    }
-                }
-                Ok(entries)
-            }
+        match result {
+            Ok(output) => Ok(Self::bucket_entries(output)),
             Err(e) => {
+                if matches!(e.code(), S3ErrorCode::AccessDenied) {
+                    return Err(FsError::Forbidden);
+                }
                 error!(
                     event = EVENT_WEBDAV_BUCKET_LIST_FAILED,
                     component = LOG_COMPONENT_PROTOCOLS,
@@ -856,6 +863,35 @@ where
         }
     }
 
+    fn bucket_entries(output: ListBucketsOutput) -> Vec<WebDavDirEntry> {
+        output
+            .buckets
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|bucket| {
+                let name = bucket.name?;
+                let modified = bucket
+                    .creation_date
+                    .map(|date| {
+                        let date: time::OffsetDateTime = date.into();
+                        SystemTime::from(date)
+                    })
+                    .unwrap_or_else(SystemTime::now);
+                Some(WebDavDirEntry {
+                    name,
+                    metadata: WebDavMetaData {
+                        size: 0,
+                        modified,
+                        created: modified,
+                        is_dir: true,
+                        etag: None,
+                        content_type: None,
+                    },
+                })
+            })
+            .collect()
+    }
+
     /// List objects in a bucket
     async fn list_objects(&self, bucket: &str, prefix: Option<&str>) -> FsResult<Vec<WebDavDirEntry>> {
         // Authorize the operation
@@ -1715,8 +1751,9 @@ where
 mod tests {
     use super::WebDavDriver;
     use crate::common::client::s3::StorageBackend as S3StorageBackend;
-    use crate::common::gateway::{S3Action, with_test_auth_override};
-    use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
+    use crate::common::dummy_storage::DummyBackend;
+    use crate::common::gateway::{S3Action, with_test_auth_override, with_test_iam_unavailable};
+    use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, test_session};
     use async_trait::async_trait;
     use bytes::Bytes;
     use dav_server::davpath::DavPath;
@@ -1906,6 +1943,134 @@ mod tests {
         WebDavDriver::new(DummyStorage, Arc::new(session_context))
     }
 
+    #[tokio::test]
+    async fn session_bucket_listing_does_not_require_global_list_permission() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_ok(ListBucketsOutput {
+            buckets: Some(vec![Bucket {
+                name: Some("allowed-bucket".to_string()),
+                ..Default::default()
+            }]),
+            ..Default::default()
+        });
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav))).with_request_context(
+            http::HeaderMap::from_iter([(http::header::USER_AGENT, http::HeaderValue::from_static("webdav-test"))]),
+            false,
+        );
+
+        let entries = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect("session-aware backend should own bucket filtering");
+
+        assert_eq!(entries.len(), 1);
+        assert_eq!(entries[0].name, "allowed-bucket");
+        let (headers, secure_transport) = storage
+            .last_session_list_context()
+            .expect("request context should be forwarded");
+        assert_eq!(headers.get("user-agent").expect("user agent"), "webdav-test");
+        assert!(!secure_transport);
+    }
+
+    #[tokio::test]
+    async fn list_buckets_maps_typed_access_denied_to_forbidden() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_err(s3s::S3Error::with_message(s3s::S3ErrorCode::AccessDenied, "policy denied"));
+        let driver = WebDavDriver::new(storage, Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("bucket listing should be denied");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
+    #[tokio::test]
+    async fn list_buckets_does_not_classify_error_text_as_access_denied() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_err(s3s::S3Error::with_message(
+            s3s::S3ErrorCode::InternalError,
+            "AccessDenied appears only in the message",
+        ));
+        let driver = WebDavDriver::new(storage, Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("bucket listing should fail");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+    }
+
+    #[tokio::test]
+    async fn global_list_permission_keeps_the_legacy_backend_path() {
+        let storage = DummyBackend::new();
+        storage.queue_list_buckets_ok(ListBucketsOutput {
+            buckets: Some(vec![Bucket {
+                name: Some("legacy-bucket".to_string()),
+                ..Default::default()
+            }]),
+            ..Default::default()
+        });
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)));
+
+        let entries = with_test_auth_override(|_, _, _| true, driver.list_buckets())
+            .await
+            .expect("globally authorized legacy backend should keep working");
+
+        assert_eq!(entries[0].name, "legacy-bucket");
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn legacy_bucket_list_error_is_a_general_failure() {
+        let storage = DummyBackend::new();
+        storage.queue_list_buckets_err(crate::common::dummy_storage::DummyError::Injected("backend failed".to_string()));
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)));
+
+        let error = with_test_auth_override(|_, _, _| true, driver.list_buckets())
+            .await
+            .expect_err("legacy backend error should fail the listing");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn iam_unavailable_does_not_enter_the_session_fallback() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_ok(ListBucketsOutput::default());
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_iam_unavailable(driver.list_buckets())
+            .await
+            .expect_err("IAM outage must fail closed");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn missing_request_context_fails_closed() {
+        let error = with_test_auth_override(|_, _, _| false, driver().list_buckets())
+            .await
+            .expect_err("bucket listing should require the original request context");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
+    #[tokio::test]
+    async fn backend_without_session_listing_fails_closed() {
+        let driver = driver().with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("default session listing must deny the request");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
     #[derive(Default)]
     struct RecordingStorageState {
         objects: HashMap<(String, String), Vec<u8>>,

crates/protocols/src/webdav/server.rs

@@ -19,6 +19,8 @@ use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, is_tem
 use bytes::Bytes;
 use dav_server::DavHandler;
 use dav_server::fakels::FakeLs;
+use http::header::{AUTHORIZATION, REFERER, USER_AGENT};
+use http::{HeaderMap, HeaderValue};
 use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
 use hyper::body::Body as HttpBody;
 use hyper::server::conn::http1;
@@ -59,6 +61,20 @@ const EVENT_WEBDAV_CONNECTION_CAP_STATE: &str = "webdav_connection_cap_state";
 /// materialise a whole object in memory for every GET.
 type WebDavBody = Pin<Box<dyn HttpBody<Data = Bytes, Error = io::Error> + Send>>;
 
+fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
+    let mut policy_headers = HeaderMap::new();
+    for name in [USER_AGENT, REFERER] {
+        if let Some(value) = headers.get(&name) {
+            policy_headers.insert(name, value.clone());
+        }
+    }
+
+    let mut authorization = HeaderValue::from_static("Basic");
+    authorization.set_sensitive(true);
+    policy_headers.insert(AUTHORIZATION, authorization);
+    policy_headers
+}
+
 /// WebDAV server implementation
 pub struct WebDavServer<S>
 where
@@ -216,7 +232,7 @@ where
                                         match timeout(request_timeout, acceptor.accept(stream)).await {
                                             Ok(Ok(tls_stream)) => {
                                                 let io = TokioIo::new(tls_stream);
-                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, max_body_size, request_timeout).await {
+                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout).await {
                                                     debug!(
                                                         event = EVENT_WEBDAV_CONNECTION_STATE,
                                                         component = LOG_COMPONENT_PROTOCOLS,
@@ -254,7 +270,7 @@ where
                                         }
                                     } else {
                                         let io = TokioIo::new(stream);
-                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, max_body_size, request_timeout).await {
+                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout).await {
                                             debug!(
                                                 event = EVENT_WEBDAV_CONNECTION_STATE,
                                                 component = LOG_COMPONENT_PROTOCOLS,
@@ -313,6 +329,7 @@ where
         io: TokioIo<I>,
         storage: S,
         source_ip: IpAddr,
+        secure_transport: bool,
         max_body_size: u64,
         request_timeout: Duration,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
@@ -321,7 +338,7 @@ where
     {
         let service = service_fn(move |req: Request<hyper::body::Incoming>| {
             let storage = storage.clone();
-            async move { Self::handle_request(req, storage, source_ip, max_body_size, request_timeout).await }
+            async move { Self::handle_request(req, storage, source_ip, secure_transport, max_body_size, request_timeout).await }
         });
 
         // A peer that opens a connection and dribbles (or never finishes)
@@ -341,6 +358,7 @@ where
         req: Request<hyper::body::Incoming>,
         storage: S,
         source_ip: IpAddr,
+        secure_transport: bool,
         max_body_size: u64,
         request_timeout: Duration,
     ) -> Result<Response<WebDavBody>, Infallible> {
@@ -398,7 +416,8 @@ where
         };
 
         // Create WebDAV driver with session context
-        let driver = WebDavDriver::new(storage, Arc::new(session_context));
+        let driver = WebDavDriver::new(storage, Arc::new(session_context))
+            .with_request_context(policy_request_headers(req.headers()), secure_transport);
 
         // Build DAV handler with boxed filesystem
         let dav_handler = DavHandler::builder()
@@ -883,6 +902,30 @@ mod tests {
             .expect("build get request")
     }
 
+    #[test]
+    fn policy_headers_drop_credentials_and_s3_auth_spoofing() {
+        let mut headers = HeaderMap::new();
+        headers.insert(AUTHORIZATION, HeaderValue::from_static("Basic dXNlcjpwYXNzd29yZA=="));
+        headers.insert(USER_AGENT, HeaderValue::from_static("webdav-client"));
+        headers.insert(REFERER, HeaderValue::from_static("https://example.test/"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"));
+        headers.insert("x-amz-signature-age", HeaderValue::from_static("0"));
+
+        let policy_headers = policy_request_headers(&headers);
+
+        assert_eq!(policy_headers.get(AUTHORIZATION).expect("authorization marker"), "Basic");
+        assert!(
+            policy_headers
+                .get(AUTHORIZATION)
+                .expect("authorization marker")
+                .is_sensitive()
+        );
+        assert_eq!(policy_headers.get(USER_AGENT).expect("user agent"), "webdav-client");
+        assert_eq!(policy_headers.get(REFERER).expect("referer"), "https://example.test/");
+        assert!(!policy_headers.contains_key("x-amz-content-sha256"));
+        assert!(!policy_headers.contains_key("x-amz-signature-age"));
+    }
+
     /// R03-CAN-051 / R03-CAN-067 / R05-CAN-094: a chunked upload declares no
     /// Content-Length, so the limit has to hold on the bytes actually read.
     #[tokio::test]
@@ -959,6 +1002,7 @@ mod tests {
             TokioIo::new(server),
             StubStorage,
             TEST_IP,
+            false,
             1024,
             Duration::from_secs(30),
         ));

crates/s3select-query/src/lib.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![recursion_limit = "256"]
+
 pub mod data_source;
 pub mod dispatcher;
 pub mod execution;

crates/scanner/src/remote_scanner/stream.rs

@@ -22,7 +22,7 @@ use crate::scanner_io::{
 use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
 use crate::{
     DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo,
-    DataUsageScanPlanDigest, Disk, ScannerDiskExt as _, ScannerError, StorageError, resolve_scanner_object_store_handle,
+    DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle,
 };
 use hmac::{Hmac, KeyInit, Mac};
 use rustfs_common::heal_channel::HealScanMode;

crates/scanner/src/scanner.rs

@@ -1081,18 +1081,6 @@ async fn run_data_scanner_cycle(
         }
     };
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
-    let storeapi_clone = storeapi.clone();
-    let ctx_clone = ctx.clone();
-    let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
-        store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
-            ctx_clone,
-            storeapi_clone,
-            receiver,
-            Some(leader_epoch),
-            Some(usage_persist_baseline),
-        )
-        .await
-    }));
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
     let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
@@ -1107,47 +1095,78 @@ async fn run_data_scanner_cycle(
             scan_mode,
         )
         .await;
+    let publication_defer_reason = match &scan_result {
+        Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
+        Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+    };
     let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
-    let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await
-    {
-        DataUsagePersistTaskResult::Completed(outcome) => outcome,
-        DataUsagePersistTaskResult::JoinFailed(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                cycle = cycle_info.current,
-                state = "usage_persist_task_failed",
-                error = %err,
-                "Scanner data usage persistence task failed"
-            );
-            DataUsagePersistOutcome::Failed
+    let usage_persist_outcome = match publication_defer_reason {
+        Some(reason) => {
+            drop(receiver);
+            DataUsagePersistOutcome::Deferred(reason)
         }
-        DataUsagePersistTaskResult::Cancelled => {
-            debug!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                cycle = cycle_info.current,
-                state = "usage_persist_task_cancelled",
-                "Scanner data usage persistence task cancelled"
-            );
-            DataUsagePersistOutcome::Failed
-        }
-        DataUsagePersistTaskResult::TimedOut => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                cycle = cycle_info.current,
-                timeout = ?usage_persist_timeout,
-                state = "usage_persist_task_timed_out",
-                "Scanner data usage persistence task timed out"
-            );
-            DataUsagePersistOutcome::Failed
+        None => {
+            // ScannerIO emits its complete or observational update only after
+            // all set workers finish. Persist after the final activity fence;
+            // this also avoids blocking the scanner on a denied publication.
+            let storeapi_clone = storeapi.clone();
+            let ctx_clone = ctx.clone();
+            let route_probe_store = storeapi.clone();
+            let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
+                store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+                    ctx_clone,
+                    storeapi_clone,
+                    receiver,
+                    Some(leader_epoch),
+                    Some(usage_persist_baseline),
+                    move || {
+                        let storeapi = route_probe_store.clone();
+                        async move { storeapi.scanner_data_usage_publication_blocked().await }
+                    },
+                )
+                .await
+            }));
+            match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await {
+                DataUsagePersistTaskResult::Completed(outcome) => outcome,
+                DataUsagePersistTaskResult::JoinFailed(err) => {
+                    error!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        cycle = cycle_info.current,
+                        state = "usage_persist_task_failed",
+                        error = %err,
+                        "Scanner data usage persistence task failed"
+                    );
+                    DataUsagePersistOutcome::Failed
+                }
+                DataUsagePersistTaskResult::Cancelled => {
+                    debug!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        cycle = cycle_info.current,
+                        state = "usage_persist_task_cancelled",
+                        "Scanner data usage persistence task cancelled"
+                    );
+                    DataUsagePersistOutcome::Failed
+                }
+                DataUsagePersistTaskResult::TimedOut => {
+                    error!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        cycle = cycle_info.current,
+                        timeout = ?usage_persist_timeout,
+                        state = "usage_persist_task_timed_out",
+                        "Scanner data usage persistence task timed out"
+                    );
+                    DataUsagePersistOutcome::Failed
+                }
+            }
         }
     };
     let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
@@ -1191,33 +1210,51 @@ async fn run_data_scanner_cycle(
         mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Failed;
     }
-    if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
-        warn!(
-            target: "rustfs::scanner",
-            event = EVENT_SCANNER_CYCLE_STATE,
-            component = LOG_COMPONENT_SCANNER,
-            subsystem = LOG_SUBSYSTEM_RUNTIME,
-            cycle = cycle_info.current,
-            required_cycle,
-            state = "cache_cycle_ahead",
-            "Scanner cycle is recovering to a newer durable cache generation"
-        );
-        emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-        return if persist_required_scanner_cycle_floor(
-            ctx,
-            storeapi.clone(),
-            cycle_info,
-            cycle_revision,
-            leader_epoch,
-            required_cycle,
-            &mut cycle_metrics_guard,
-        )
-        .await
-        {
-            ScannerCycleOutcome::Partial
-        } else {
-            ScannerCycleOutcome::Failed
-        };
+    match scanner_cycle_pre_commit_outcome(scan_cycle_result.required_cycle_floor(), &usage_persist_outcome) {
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => {
+            warn!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_CYCLE_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                cycle = cycle_info.current,
+                required_cycle,
+                state = "cache_cycle_ahead",
+                "Scanner cycle is recovering to a newer durable cache generation"
+            );
+            emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
+            return if persist_required_scanner_cycle_floor(
+                ctx,
+                storeapi.clone(),
+                cycle_info,
+                cycle_revision,
+                leader_epoch,
+                required_cycle,
+                &mut cycle_metrics_guard,
+            )
+            .await
+            {
+                ScannerCycleOutcome::Partial
+            } else {
+                ScannerCycleOutcome::Failed
+            };
+        }
+        Some(ScannerCyclePreCommitOutcome::Deferred(reason)) => {
+            info!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_CYCLE_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                cycle = cycle_info.current,
+                reason = reason.as_str(),
+                state = "deferred",
+                "Scanner cycle deferred before data usage publication"
+            );
+            emit_scan_cycle_deferred(cycle_start.elapsed());
+            mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
+            return ScannerCycleOutcome::Deferred(reason);
+        }
+        None => {}
     }
     if usage_persist_outcome == DataUsagePersistOutcome::Failed {
         error!(
@@ -1804,14 +1841,13 @@ async fn run_data_scanner_with_maintenance_state(
             wait_plan.delay,
             activity_poll_interval,
             &mut scanner_activity_seen,
-            ScannerCycleObservedGenerations {
-                // A non-converged cycle holds further activity notifications
-                // until its bounded retry timer to avoid an unbroken scan loop.
-                dirty_usage: convergence_retry_interval.is_none().then_some(dirty_usage_generation_seen),
-                runtime_config: runtime_config_generation_seen,
-                maintenance: maintenance_generation_before_wait,
-                defer_cluster_activity: convergence_retry_interval.is_some(),
-            },
+            ScannerCycleObservedGenerations::for_wait(
+                &runtime_config,
+                convergence_retry_interval,
+                dirty_usage_generation_seen,
+                runtime_config_generation_seen,
+                maintenance_generation_before_wait,
+            ),
             || guard.is_lock_lost(),
             || probe_scanner_activity(storeapi.as_ref(), distributed),
         )
@@ -2001,6 +2037,56 @@ impl Drop for ScannerScanModeGuard {
     }
 }
 
+async fn final_data_usage_publication_defer_reason(
+    storeapi: &ECStore,
+    status: ScannerCycleStatus,
+) -> Option<ScannerCycleDeferReason> {
+    match status {
+        ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
+            if storeapi.scanner_data_usage_publication_blocked().await {
+                return Some(ScannerCycleDeferReason::DataMovement);
+            }
+            if status == ScannerCycleStatus::Complete {
+                let distributed = storeapi.setup_is_dist_erasure().await;
+                match probe_scanner_activity(storeapi, distributed).await {
+                    Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) => None,
+                    Ok(_) => Some(ScannerCycleDeferReason::DataMovement),
+                    Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+                }
+            } else {
+                // A superseded cycle is explicitly observational and cannot
+                // replace the authoritative snapshot. It may still be
+                // persisted as a convergence baseline for the next cycle.
+                None
+            }
+        }
+        ScannerCycleStatus::Deferred(reason) => Some(reason),
+        // Incomplete cycles do not publish a usage snapshot. Keep the
+        // decision permissive so existing partial-cycle handling remains
+        // unchanged if a future scanner path emits a bookkeeping update.
+        ScannerCycleStatus::Incomplete => None,
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum ScannerCyclePreCommitOutcome {
+    RecoverCacheCycle(u64),
+    Deferred(ScannerCycleDeferReason),
+}
+
+fn scanner_cycle_pre_commit_outcome(
+    required_cycle_floor: Option<u64>,
+    usage_persist_outcome: &DataUsagePersistOutcome,
+) -> Option<ScannerCyclePreCommitOutcome> {
+    // Keep the publication barrier fail-closed: `.bloomcycle.bin` uses the
+    // same routed writer and its floor must remain pending while data movement
+    // hides the source pool.
+    match usage_persist_outcome {
+        DataUsagePersistOutcome::Deferred(reason) => Some(ScannerCyclePreCommitOutcome::Deferred(*reason)),
+        _ => required_cycle_floor.map(ScannerCyclePreCommitOutcome::RecoverCacheCycle),
+    }
+}
+
 fn scanner_cycle_completion_outcome(
     scan_status: ScannerCycleStatus,
     usage_persist_outcome: DataUsagePersistOutcome,
@@ -2008,6 +2094,7 @@ fn scanner_cycle_completion_outcome(
     has_failed_dirty_usage: bool,
 ) -> ScannerCycleOutcome {
     match (scan_status, usage_persist_outcome) {
+        (_, DataUsagePersistOutcome::Deferred(reason)) => ScannerCycleOutcome::Deferred(reason),
         (_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
         (ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
             if !has_dirty_usage && !has_failed_dirty_usage =>

crates/scanner/src/scanner/activity.rs

@@ -229,6 +229,27 @@ pub(super) struct ScannerCycleObservedGenerations {
     pub(super) defer_cluster_activity: bool,
 }
 
+impl ScannerCycleObservedGenerations {
+    pub(super) fn for_wait(
+        runtime_config: &ScannerRuntimeConfig,
+        convergence_retry_interval: Option<Duration>,
+        dirty_usage_generation_seen: u64,
+        runtime_config_generation: u64,
+        maintenance_generation: u64,
+    ) -> Self {
+        Self {
+            // An explicit cycle override is a duty-cycle policy; dirty usage
+            // wakes stay on the default adaptive path so the interval holds.
+            dirty_usage: (convergence_retry_interval.is_none()
+                && runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default)
+                .then_some(dirty_usage_generation_seen),
+            runtime_config: runtime_config_generation,
+            maintenance: maintenance_generation,
+            defer_cluster_activity: convergence_retry_interval.is_some(),
+        }
+    }
+}
+
 pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
 
 #[derive(Clone, Debug, PartialEq, Eq)]

crates/scanner/src/scanner/tests.rs

@@ -153,6 +153,7 @@ struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
+    object_not_found_put_number: Mutex<HashMap<String, usize>>,
     error_after_commit_put_number: Mutex<HashMap<String, usize>>,
     interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
     cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
@@ -224,6 +225,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
         if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
             return Err(EcstoreError::other("injected put failure"));
         }
+        if self.object_not_found_put_number.lock().await.get(&key) == Some(&put_count) {
+            return Err(EcstoreError::ObjectNotFound(bucket.to_string(), object.to_string()));
+        }
 
         let interleaving_data = {
             let mut interleaving_puts = self.interleaving_puts.lock().await;
@@ -1431,6 +1435,170 @@ async fn test_store_data_usage_in_backend_preserves_newer_snapshot() {
     assert_eq!(outcome, DataUsagePersistOutcome::Current);
 }
 
+#[tokio::test]
+async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier() {
+    for (route_blocked, expected) in [
+        (true, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)),
+        (false, DataUsagePersistOutcome::Failed),
+    ] {
+        let store = Arc::new(MemoryConfigStore::default());
+        let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+        let baseline = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
+        let baseline_data = serde_json::to_vec(&baseline).expect("baseline usage snapshot should encode");
+        store.objects.lock().await.insert(key.clone(), baseline_data.clone());
+        store.revisions.lock().await.insert(key.clone(), 1);
+        store.object_not_found_put_number.lock().await.insert(key.clone(), 1);
+
+        let (sender, receiver) = mpsc::channel(1);
+        sender
+            .send(complete_usage_with_bucket_count(
+                Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
+                2,
+            ))
+            .await
+            .expect("new usage snapshot should enqueue");
+        drop(sender);
+        let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let route_probe_calls = probe_calls.clone();
+
+        let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+            CancellationToken::new(),
+            store.clone(),
+            receiver,
+            None,
+            Some(DataUsagePersistBaseline {
+                data: Some(Bytes::from(baseline_data.clone())),
+                revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
+            }),
+            move || {
+                let probe_calls = route_probe_calls.clone();
+                async move {
+                    let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                    route_blocked && call > 1
+                }
+            },
+        )
+        .await;
+
+        assert_eq!(outcome, expected);
+        assert_eq!(
+            probe_calls.load(std::sync::atomic::Ordering::SeqCst),
+            3,
+            "ObjectNotFound must be followed by a fresh route-barrier probe"
+        );
+        assert_eq!(
+            store.objects.lock().await.get(&key),
+            Some(&baseline_data),
+            "a route failure must not replace the authoritative baseline"
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
+    for observational in [false, true] {
+        let store = Arc::new(MemoryConfigStore::default());
+        let target_path = if observational {
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
+        } else {
+            DATA_USAGE_OBJ_NAME_PATH.as_str()
+        };
+        let target_key = memory_config_key(RUSTFS_META_BUCKET, target_path);
+        let mut incoming = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
+        incoming.usage_snapshot_converged = Some(!observational);
+        let (sender, receiver) = mpsc::channel(1);
+        sender.send(incoming).await.expect("usage snapshot should enqueue");
+        drop(sender);
+
+        let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+            CancellationToken::new(),
+            store.clone(),
+            receiver,
+            None,
+            Some(DataUsagePersistBaseline {
+                data: None,
+                revision: DataUsageCacheRevision::Missing,
+            }),
+            || async { true },
+        )
+        .await;
+
+        assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+        assert!(!store.objects.lock().await.contains_key(&target_key));
+        assert_eq!(
+            store.put_counts.lock().await.get(&target_key),
+            None,
+            "the final pool-state fence must run before the first PUT"
+        );
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn test_usage_route_barrier_precedes_durable_reconciliation() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let snapshot = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
+    let snapshot_data = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
+    let (sender, receiver) = mpsc::channel(1);
+    sender.send(snapshot).await.expect("usage snapshot should enqueue");
+    drop(sender);
+
+    let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        CancellationToken::new(),
+        store.clone(),
+        receiver,
+        None,
+        Some(DataUsagePersistBaseline {
+            data: Some(Bytes::from(snapshot_data)),
+            revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
+        }),
+        || async { true },
+    )
+    .await;
+
+    assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    assert_eq!(store.put_counts.lock().await.get(&key), None);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_deferred_usage_save_keeps_last_real_save_metric() {
+    let metrics = global_metrics();
+    metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
+    let before = metrics.report().await.usage_freshness;
+
+    let store = Arc::new(MemoryConfigStore::default());
+    let (sender, receiver) = mpsc::channel(1);
+    sender
+        .send(complete_usage_with_bucket_count(
+            Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
+            1,
+        ))
+        .await
+        .expect("usage snapshot should enqueue");
+    drop(sender);
+
+    let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        CancellationToken::new(),
+        store,
+        receiver,
+        None,
+        Some(DataUsagePersistBaseline {
+            data: None,
+            revision: DataUsageCacheRevision::Missing,
+        }),
+        || async { true },
+    )
+    .await;
+
+    assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    let after = metrics.report().await.usage_freshness;
+    assert_eq!(after.last_usage_save_result, before.last_usage_save_result);
+    assert_eq!(after.last_usage_save_result_code, before.last_usage_save_result_code);
+    assert_eq!(after.last_usage_save_unix_secs, before.last_usage_save_unix_secs);
+}
+
 #[tokio::test]
 async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2325,6 +2493,15 @@ async fn test_store_data_usage_in_backend_reports_missing_snapshot() {
 
 #[test]
 fn test_scanner_cycle_completion_prioritizes_persist_failure() {
+    assert_eq!(
+        scanner_cycle_completion_outcome(
+            ScannerCycleStatus::Complete,
+            DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement),
+            true,
+            false,
+        ),
+        ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
+    );
     assert_eq!(
         scanner_cycle_completion_outcome(
             ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
@@ -2421,6 +2598,33 @@ fn test_scanner_cycle_completion_prioritizes_persist_failure() {
     );
 }
 
+#[test]
+fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
+    for reason in [
+        ScannerCycleDeferReason::DataMovement,
+        ScannerCycleDeferReason::ActivityBaselineUnavailable,
+    ] {
+        let deferred = DataUsagePersistOutcome::Deferred(reason);
+        assert_eq!(
+            scanner_cycle_pre_commit_outcome(Some(19), &deferred),
+            Some(ScannerCyclePreCommitOutcome::Deferred(reason)),
+            "a blocked publication must not persist the routed scanner cycle floor"
+        );
+        assert_eq!(
+            scanner_cycle_pre_commit_outcome(None, &deferred),
+            Some(ScannerCyclePreCommitOutcome::Deferred(reason))
+        );
+    }
+    assert_eq!(
+        scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Saved),
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
+    );
+    assert_eq!(
+        scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Failed),
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
+    );
+}
+
 #[test]
 #[serial]
 fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
@@ -2448,6 +2652,23 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
     assert!(!crate::scanner_io::dirty_usage_buckets_pending());
 }
 
+#[test]
+#[serial]
+fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+    crate::scanner_io::record_dirty_usage_bucket("photos");
+    let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
+    let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
+
+    let (outcome, _, acknowledgements) =
+        finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+
+    assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    assert!(acknowledgements.is_empty());
+    assert!(crate::scanner_io::dirty_usage_buckets_pending());
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+}
+
 #[tokio::test]
 async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
     let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
@@ -3107,6 +3328,31 @@ fn clean_idle_backoff_requires_activity_probes() {
     assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config));
 }
 
+#[test]
+fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
+    let default_config = ScannerRuntimeConfig::default();
+
+    let default_observed = ScannerCycleObservedGenerations::for_wait(&default_config, None, 7, 11, 13);
+    assert_eq!(default_observed.dirty_usage, Some(7));
+    assert_eq!(default_observed.runtime_config, 11);
+    assert_eq!(default_observed.maintenance, 13);
+    assert!(!default_observed.defer_cluster_activity);
+
+    let retry_observed = ScannerCycleObservedGenerations::for_wait(&default_config, Some(Duration::from_secs(11)), 7, 11, 13);
+    assert_eq!(retry_observed.dirty_usage, None);
+    assert!(retry_observed.defer_cluster_activity);
+
+    for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] {
+        let explicit_cycle = ScannerRuntimeConfig {
+            cycle_interval_source: source,
+            ..default_config.clone()
+        };
+        let explicit_observed = ScannerCycleObservedGenerations::for_wait(&explicit_cycle, None, 7, 11, 13);
+        assert_eq!(explicit_observed.dirty_usage, None);
+        assert!(!explicit_observed.defer_cluster_activity);
+    }
+}
+
 #[test]
 #[serial]
 fn clean_idle_cap_preserves_default_bitrot_coverage_window() {

crates/scanner/src/scanner/usage_store.rs

@@ -22,6 +22,10 @@ pub(super) enum DataUsagePersistOutcome {
     AlreadyDurable,
     PriorCycleDurable,
     Saved,
+    /// The metadata route is temporarily unavailable (for example while a
+    /// terminal decommission state keeps the source pool suspended).  The
+    /// caller must retry without acknowledging dirty usage.
+    Deferred(ScannerCycleDeferReason),
     Failed,
 }
 
@@ -92,10 +96,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch(
 pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
     ctx: CancellationToken,
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
-    mut receiver: mpsc::Receiver<DataUsageInfo>,
+    receiver: mpsc::Receiver<DataUsageInfo>,
     leader_epoch: Option<u64>,
     initial_baseline: Option<DataUsagePersistBaseline>,
 ) -> DataUsagePersistOutcome {
+    store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        ctx,
+        storeapi,
+        receiver,
+        leader_epoch,
+        initial_baseline,
+        || async { false },
+    )
+    .await
+}
+
+pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
+    ctx: CancellationToken,
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    mut receiver: mpsc::Receiver<DataUsageInfo>,
+    leader_epoch: Option<u64>,
+    initial_baseline: Option<DataUsagePersistBaseline>,
+    route_probe: F,
+) -> DataUsagePersistOutcome
+where
+    F: Fn() -> Fut + Send + Sync,
+    Fut: Future<Output = bool> + Send,
+{
     let mut outcome = DataUsagePersistOutcome::NoUpdate;
     let mut next_baseline = initial_baseline;
 
@@ -113,6 +140,19 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
         } else {
             DATA_USAGE_OBJ_NAME_PATH.as_str()
         };
+        if route_probe().await {
+            debug!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                path = %target_path,
+                state = "publication_blocked_before_reconcile",
+                "Scanner data usage publication deferred by the pool-state fence"
+            );
+            outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+            break;
+        }
 
         if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
             let authoritative_data = match next_baseline.as_ref() {
@@ -275,6 +315,18 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
             if ctx.is_cancelled() {
                 break 'updates;
             }
+            if route_probe().await {
+                debug!(
+                    target: "rustfs::scanner",
+                    event = EVENT_SCANNER_PERSIST_STATE,
+                    component = LOG_COMPONENT_SCANNER,
+                    subsystem = LOG_SUBSYSTEM_RUNTIME,
+                    path = %target_path,
+                    state = "publication_blocked_before_save",
+                    "Scanner data usage publication deferred by the final pool-state fence"
+                );
+                break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+            }
 
             let done_save = Metrics::time(Metric::SaveUsage);
             let save_result = save_config_shared_with_preconditions(
@@ -313,6 +365,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
                         "Scanner data usage CAS conflict will be reconciled"
                     );
                 }
+                Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
+                    let route_blocked = route_probe().await;
+                    if route_blocked {
+                        warn!(
+                            target: "rustfs::scanner",
+                            event = EVENT_SCANNER_PERSIST_STATE,
+                            component = LOG_COMPONENT_SCANNER,
+                            subsystem = LOG_SUBSYSTEM_RUNTIME,
+                            path = %target_path,
+                            state = "publication_deferred",
+                            error = %e,
+                            "Scanner data usage route is blocked by data movement; retrying later"
+                        );
+                        break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+                    }
+                    error!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        path = %target_path,
+                        state = "save_failed",
+                        error = %e,
+                        "Scanner data usage save failed"
+                    );
+                    break DataUsagePersistOutcome::Failed;
+                }
                 Err(e) => {
                     error!(
                         target: "rustfs::scanner",
@@ -370,6 +449,13 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
                 outcome = DataUsagePersistOutcome::Failed;
                 continue;
             }
+            DataUsagePersistOutcome::Deferred(reason) => {
+                // A deferred publication is an intentional retryable state, not a
+                // failed save. Keep the last real save result so admin freshness
+                // reporting does not turn a pool-recovery fence into a false error.
+                outcome = DataUsagePersistOutcome::Deferred(reason);
+                break 'updates;
+            }
             DataUsagePersistOutcome::Saved => {
                 if observational {
                     invalidate_admin_data_usage_snapshot_cache().await;

crates/scanner/src/scanner_folder/item_actions.rs

@@ -274,18 +274,13 @@ impl ScannerItem {
     /// Transform meta directory by splitting prefix and extracting object name
     /// This converts a directory path like "bucket/dir1/dir2/file" to prefix="bucket/dir1/dir2" and object_name="file"
     pub fn transform_meta_dir(&mut self) {
-        let prefix = self.prefix.clone(); // Clone to avoid borrow checker issues
-        let split: Vec<&str> = prefix.split(SLASH_SEPARATOR).collect();
-
-        if split.len() > 1 {
-            let prefix_parts: Vec<&str> = split[..split.len() - 1].to_vec();
-            self.prefix = path_join_buf(&prefix_parts);
+        let prefix = std::mem::take(&mut self.prefix);
+        if let Some((parent, object_name)) = prefix.rsplit_once(SLASH_SEPARATOR) {
+            self.prefix = path_join_buf(&[parent]);
+            self.object_name = object_name.to_string();
         } else {
-            self.prefix = String::new();
+            self.object_name = prefix;
         }
-
-        // Object name is the last element
-        self.object_name = split.last().unwrap_or(&"").to_string();
     }
 
     pub(super) fn metadata_object_path(&self) -> String {
@@ -301,13 +296,14 @@ impl ScannerItem {
         versioning_config: VersioningConfiguration,
         size_summary: &mut SizeSummary,
     ) {
+        let object_path = self.object_path();
         if object_infos.is_empty() {
             debug!(
                 target: "rustfs::scanner::folder",
                 event = EVENT_SCANNER_LIFECYCLE_ACTION,
                 component = LOG_COMPONENT_SCANNER,
                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                object_path = %self.object_path(),
+                object_path = %object_path,
                 state = "no_object_versions",
                 "Scanner lifecycle action skipped"
             );
@@ -318,7 +314,7 @@ impl ScannerItem {
             event = EVENT_SCANNER_LIFECYCLE_ACTION,
             component = LOG_COMPONENT_SCANNER,
             subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-            object_path = %self.object_path(),
+            object_path = %object_path,
             state = "started",
             "Scanner lifecycle evaluation started"
         );
@@ -360,7 +356,7 @@ impl ScannerItem {
                 event = EVENT_SCANNER_LIFECYCLE_ACTION,
                 component = LOG_COMPONENT_SCANNER,
                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                object_path = %self.object_path(),
+                object_path = %object_path,
                 state = "no_lifecycle_config",
                 "Scanner lifecycle action finished without lifecycle rules"
             );
@@ -385,7 +381,7 @@ impl ScannerItem {
                     event = EVENT_SCANNER_LIFECYCLE_ACTION,
                     component = LOG_COMPONENT_SCANNER,
                     subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                    object_path = %self.object_path(),
+                    object_path = %object_path,
                     state = "evaluate_failed",
                     error = %e,
                     "Scanner lifecycle action evaluation failed"
@@ -502,7 +498,7 @@ impl ScannerItem {
                         emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
                         if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
                             done_ilm(1)();
-                            if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
+                            if !versioning_config.prefix_enabled(&object_path) && event.action == IlmAction::DeleteAction {
                                 remaining_versions -= 1;
                                 size = 0;
                             }
@@ -570,7 +566,7 @@ impl ScannerItem {
                 trace_emit(|| {
                     TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
                         .with_bucket(self.bucket.as_str())
-                        .with_object(self.object_path())
+                        .with_object(object_path.as_str())
                         .with_duration(trace_started_at.elapsed())
                         .with_attr("state", state)
                         .with_attr("action", action.as_str())
@@ -889,3 +885,48 @@ pub(super) async fn contains_erasure_part_file(path: &str) -> Result<bool, Scann
 
     Ok(false)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn scanner_item_with_prefix(prefix: &str) -> ScannerItem {
+        ScannerItem {
+            path: String::new(),
+            bucket: "bucket".to_string(),
+            prefix: prefix.to_string(),
+            object_name: String::new(),
+            file_type: std::fs::metadata(std::env::temp_dir())
+                .expect("temp dir metadata should be readable")
+                .file_type(),
+            lifecycle: None,
+            object_lock: None,
+            replication: None,
+            heal_enabled: false,
+            heal_bitrot: false,
+            debug: false,
+        }
+    }
+
+    #[test]
+    fn transform_meta_dir_splits_parent_and_object_without_extra_components() {
+        let mut item = scanner_item_with_prefix("bucket/prefix/object");
+
+        item.transform_meta_dir();
+
+        assert_eq!(item.prefix, "bucket/prefix");
+        assert_eq!(item.object_name, "object");
+        assert_eq!(item.object_path(), "bucket/prefix/object");
+    }
+
+    #[test]
+    fn transform_meta_dir_moves_single_component_into_object_name() {
+        let mut item = scanner_item_with_prefix("object");
+
+        item.transform_meta_dir();
+
+        assert_eq!(item.prefix, "");
+        assert_eq!(item.object_name, "object");
+        assert_eq!(item.object_path(), "object");
+    }
+}

crates/scanner/src/scanner_io/guards.rs

@@ -147,11 +147,19 @@ impl Drop for DiskBucketScanActiveGuard {
 
 pub(super) struct BucketDriveFailureGuard {
     failed: bool,
+    source: rustfs_common::metrics::ScannerWorkSource,
+    bucket: String,
+    drive: String,
 }
 
 impl BucketDriveFailureGuard {
-    pub(super) fn new() -> Self {
-        Self { failed: true }
+    pub(super) fn new(source: rustfs_common::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
+        Self {
+            failed: true,
+            source,
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        }
     }
 
     pub(super) fn mark_not_failed(&mut self) {
@@ -161,6 +169,7 @@ impl BucketDriveFailureGuard {
 
 impl Drop for BucketDriveFailureGuard {
     fn drop(&mut self) {
+        global_metrics().record_scan_bucket_drive_end(self.source, &self.bucket, &self.drive);
         if self.failed {
             global_metrics().record_scan_bucket_drive_failure();
         }
@@ -272,3 +281,28 @@ pub(super) fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error)
 pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
     Error::other(format!("{stage} task join failed: {err}"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_common::metrics::{ScannerWorkSource, global_metrics};
+
+    #[test]
+    fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_active_lifecycle_test__";
+        let drive = "/__guard_active_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+        {
+            let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            guard.mark_not_failed();
+        }
+        assert!(
+            !global_metrics()
+                .scanner_runtime_details_report()
+                .active_bucket_drive_scans
+                .iter()
+                .any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
+        );
+    }
+}

crates/scanner/src/scanner_io/io_cycle.rs

@@ -49,6 +49,25 @@ impl ScannerIOCycle for ECStore {
     ) -> Result<ScannerCycleResult> {
         let child_token = ctx.child_token();
 
+        // Check the local pool metadata before listing buckets. A failed or
+        // canceled decommission remains suspended after its worker exits, so
+        // starting a scan in that state could build a snapshot that cannot be
+        // routed to the authoritative metadata object.
+        if self.scanner_data_usage_publication_blocked().await {
+            debug!(
+                target: "rustfs::scanner::io",
+                event = EVENT_SCANNER_SET_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_IO,
+                state = "cycle_data_usage_route_blocked",
+                "Scanner cycle deferred while data usage metadata remains hidden by data movement"
+            );
+            return Ok(ScannerCycleResult::new(
+                ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
+                None,
+            ));
+        }
+
         let distributed = self.setup_is_dist_erasure().await;
         let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
             ScannerActivityPreflight::Ready(snapshot) => snapshot,

crates/scanner/src/scanner_io/io_disk.rs

@@ -42,7 +42,8 @@ impl ScannerIODisk for Disk {
             return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
         }
 
-        let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
+        let metadata_object_path = item.object_path();
+        let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
             Ok(data) => data,
             Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
                 return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -51,23 +52,23 @@ impl ScannerIODisk for Disk {
                 return Err(scanner_metadata_transient_error(
                     format!("failed to read metadata: {e}"),
                     &item.bucket,
-                    &item.object_path(),
+                    &metadata_object_path,
                 ));
             }
         };
 
         item.transform_meta_dir();
+        let object_path = item.object_path();
 
-        let meta = FileMeta::load(&data).map_err(|e| {
-            scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
-        })?;
-        let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
+        let meta = FileMeta::load(&data)
+            .map_err(|e| scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &object_path))?;
+        let fivs = match meta.get_file_info_versions(item.bucket.as_str(), object_path.as_str(), false) {
             Ok(versions) => versions,
             Err(e) => {
                 return Err(scanner_metadata_corrupt_error(
                     format!("failed to resolve file info versions: {e}"),
                     &item.bucket,
-                    &item.object_path(),
+                    &object_path,
                 ));
             }
         };
@@ -91,17 +92,17 @@ impl ScannerIODisk for Disk {
                 VersioningConfiguration::default()
             }
         };
-        let versioned = versioning_config.versioned(&item.object_path());
+        let versioned = versioning_config.versioned(&object_path);
 
         let object_infos = fivs
             .versions
             .iter()
-            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
+            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
             .collect::<Vec<ObjectInfo>>();
         let free_version_infos = fivs
             .free_versions
             .iter()
-            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
+            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
             .collect::<Vec<ObjectInfo>>();
 
         let mut size_summary = SizeSummary::default();
@@ -147,8 +148,12 @@ impl ScannerIODisk for Disk {
         let drive_start = std::time::Instant::now();
         let bucket = cache.info.name.clone();
         let disk_path = self.path().to_string_lossy().to_string();
-        global_metrics().record_scan_bucket_drive_start();
-        let mut failure_guard = BucketDriveFailureGuard::new();
+        let source = match scan_mode {
+            HealScanMode::Deep => rustfs_common::metrics::ScannerWorkSource::Bitrot,
+            HealScanMode::Normal | HealScanMode::Unknown => rustfs_common::metrics::ScannerWorkSource::Usage,
+        };
+        global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
+        let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
         let _guard = self.start_scan();
 
         let mut cache = cache;
@@ -196,32 +201,32 @@ impl ScannerIODisk for Disk {
         match result {
             Ok(mut data_usage_info) => {
                 done_drive();
-                emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
                 data_usage_info.info.last_update = Some(SystemTime::now());
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
             }
             Err(ScannerError::PartialCache(mut partial_cache)) => {
                 done_drive();
-                emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                 partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
             }
             Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
                 done_drive();
-                emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                 partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
             }
             Err(e) => {
                 if ctx.is_cancelled() {
-                    emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                    emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                     failure_guard.mark_not_failed();
                 } else {
                     done_drive();
-                    emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
+                    emit_scan_bucket_drive_complete(source, false, &bucket, &disk_path, drive_start.elapsed());
                 }
                 Err(StorageError::other(format!("Failed to scan data folder: {e}")))
             }

crates/scanner/src/scanner_io/tests.rs

@@ -17,7 +17,9 @@ use super::io_disk::tier_stats_template;
 use super::*;
 use crate::scanner_budget::ScannerCycleBudgetConfig;
 use crate::scanner_folder::ScannerItem;
-use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
+use crate::storage_api::owner::{
+    EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
+};
 use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
 use crate::{
     DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
@@ -182,6 +184,39 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
     assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
 }
 
+#[tokio::test]
+#[serial]
+async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
+    let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+    for decommission in [
+        EcstorePoolDecommissionInfo {
+            failed: true,
+            ..Default::default()
+        },
+        EcstorePoolDecommissionInfo {
+            canceled: true,
+            ..Default::default()
+        },
+    ] {
+        store.pool_meta.write().await.pools[0].decommission = Some(decommission);
+        assert!(store.scanner_data_usage_publication_blocked().await);
+
+        let ctx = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
+        let (updates, mut receiver) = mpsc::channel(1);
+        let result = tokio::time::timeout(
+            Duration::from_secs(30),
+            ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
+        )
+        .await
+        .expect("terminal-decommission-deferred scanner cycle should finish")
+        .expect("terminal-decommission-deferred scanner cycle should succeed");
+
+        assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
+        assert!(receiver.recv().await.is_none(), "blocked cycle must not publish usage");
+    }
+}
+
 #[tokio::test]
 async fn data_usage_publish_fails_when_receiver_is_closed() {
     let (updates, receiver) = mpsc::channel(1);
@@ -236,6 +271,10 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
     assert_eq!(bucket_usage.size, 11);
     assert_eq!(usage.objects_total_count, 2);
     assert_eq!(usage.objects_total_size, 11);
+    assert!(
+        receiver.recv().await.is_none(),
+        "a scanner cycle must publish at most one terminal usage snapshot"
+    );
 }
 
 #[tokio::test]

crates/scanner/src/storage_api.rs

@@ -47,6 +47,8 @@ pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys
 pub(crate) use rustfs_ecstore::api::cache::{
     ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
 };
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::api::capacity::PoolDecommissionInfo as EcstorePoolDecommissionInfo;
 pub(crate) use rustfs_ecstore::api::capacity::{
     is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
     path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
@@ -127,9 +129,9 @@ pub(crate) mod owner {
     #[cfg(test)]
     pub(crate) use super::{
         EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
-        EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
-        EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
-        ecstore_new_disk,
+        EcstoreInstanceContext, EcstorePoolDecommissionInfo, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo,
+        EcstoreRebalanceMeta, EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys,
+        ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
     };
 }
 

crates/test-utils/src/lib.rs

@@ -245,6 +245,18 @@ impl TestECStoreEnvBuilder {
             .await
             .expect("build test ECStore");
 
+        // The production bootstrap only persists pool.bin from the elected
+        // first cluster node.  Test stores intentionally have no cluster
+        // election, but heal-format still requires that durable fence before
+        // it can write any disk format.  Materialize the validated topology
+        // here so the shared fixture models a ready single-node store.
+        let mut pool_meta = ecstore.pool_meta.read().await.clone();
+        pool_meta.dont_save = false;
+        pool_meta
+            .save(ecstore.pools.clone())
+            .await
+            .expect("persist test pool metadata");
+
         if self.init_bucket_metadata {
             let buckets_list = ecstore
                 .list_bucket(&BucketOptions {

docs/operations/scanner-runtime-controls.md

@@ -70,6 +70,11 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
 `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
 needs a precise override.
 
+An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
+cadence: dirty-usage notifications do not bypass that configured interval.
+The default adaptive policy continues to use dirty-usage notifications to wake
+the scanner between timer-driven cycles.
+
 ## Single-disk clean-idle scheduling
 
 An erasure single-disk deployment using the built-in cycle and bitrot defaults

docs/testing/metrics-1649-prometheus-smoke.md

@@ -0,0 +1,44 @@
+# Backlog #1649 Prometheus smoke
+
+`scripts/prometheus_metrics_1649_smoke.py` is a read-only environment check for
+the metric dimensions delivered by backlog #1649 and issues #1650-#1653. It
+uses Prometheus' instant-query API and does not start, stop, reconfigure, or
+load RustFS nodes.
+
+Run the parser and selector self-test without a live environment:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py --self-test
+```
+
+For a live cluster, pass a Prometheus base URL (or its `/api/v1/query`
+endpoint), one or more expected server label values, and the built-in profile:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py \
+  --query-url http://prometheus.example:9090 \
+  --profile backlog-1649 \
+  --server rustfs-node1 \
+  --server rustfs-node2
+```
+
+The profile checks the disk, scanner, ILM, audit, and notification series and
+their required labels. It also requires the legacy aggregate audit and
+notification series, so an additive label change cannot silently break
+existing dashboards.
+
+Dynamic series retirement is checked with an exact label set after the
+scheduler retirement window has elapsed:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py \
+  --query-url http://prometheus.example:9090 \
+  --retired 'rustfs_scanner_bucket_drive_result_total|server=node1,bucket=removed,drive=d1,result=success' \
+  --retired 'rustfs_audit_total_messages_by_server|server=node1,target_id=removed'
+```
+
+`--require METRIC|key=value,...` requires a matching series;
+`--require-labels METRIC|key1,key2` requires every returned series to carry
+the named labels. Use `--bearer` for a bearer token or `--basic` for a
+`username:password` credential when Prometheus is protected. Do not put
+credentials in committed commands, logs, or issue comments.

rustfs/src/admin/handlers/inspect_archive.rs

@@ -643,7 +643,7 @@ mod tests {
     fn decode_hex_fixture(value: &str) -> Vec<u8> {
         value
             .split_ascii_whitespace()
-            .flat_map(|line| line.as_bytes().chunks_exact(2))
+            .flat_map(|line| line.as_bytes().as_chunks::<2>().0.iter())
             .map(|pair| {
                 let pair = std::str::from_utf8(pair).expect("fixture contains ASCII hex");
                 u8::from_str_radix(pair, 16).expect("fixture contains valid hex")

rustfs/src/admin/router.rs

@@ -2694,7 +2694,7 @@ async fn ssec_passthrough_probe_object(
     let head = target_client
         .head_object(target_bucket, probe_key, head_version)
         .await
-        .map_err(S3ClientError::from)?;
+        .map_err(|err| S3ClientError::from(*err))?;
 
     Ok(ReplicationSsecProbeOutcome {
         evidence_present: head.sse_customer_algorithm().is_some_and(|algorithm| !algorithm.is_empty()),

rustfs/src/app/object_usecase.rs

@@ -3683,6 +3683,13 @@ where
 
     let authorization_headers = pax_headers.clone();
 
+    if let Some(value) = pax_headers.remove("x-amz-tagging") {
+        let value = value
+            .to_str()
+            .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball object tagging value"))?;
+        metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), value.to_owned());
+    }
+
     let object_lock_mode = pax_headers
         .remove(AMZ_OBJECT_LOCK_MODE_LOWER)
         .map(|value| {
@@ -3962,6 +3969,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
     opts.version_id.is_none() && opts.versioned && !opts.version_suspended
 }
 
+/// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
+/// distributed delete path instead of its usual typed missing-object error.
+fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
+    is_err_object_not_found(error)
+        || is_err_version_not_found(error)
+        || matches!(error, StorageError::Io(source) if source.kind() == std::io::ErrorKind::NotFound)
+}
+
 /// Bounded concurrency for the per-object pre-delete stat fanout in
 /// `execute_delete_objects` (backlog#929 / HP-8). Keeps the metadata reads for
 /// a 1000-key batch from serializing while capping the disk fanout pressure.
@@ -4023,6 +4038,27 @@ fn delete_response_version_id(version_id: Option<Uuid>, synthetic_version_id: bo
     }
 }
 
+fn reduce_delete_objects_result<'a>(
+    object: &ObjectToDelete,
+    deleted: &'a StorageDeletedObject,
+    error: Option<&EcstoreError>,
+    synthetic_version_id: bool,
+) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
+    match error {
+        None => Ok(deleted),
+        Some(error) if is_delete_objects_not_found(error) => Ok(deleted),
+        Some(error) => {
+            let api_error = ApiError::from(error.clone());
+            Err(s3s::dto::Error {
+                code: Some(api_error.code.as_str().to_string()),
+                key: Some(object.object_name.clone()),
+                message: Some(api_error.message),
+                version_id: delete_response_version_id(object.version_id, synthetic_version_id),
+            })
+        }
+    }
+}
+
 fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
     let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
         .map(|value| normalize_snowball_prefix(&value))
@@ -8469,39 +8505,31 @@ impl DefaultObjectUsecase {
         for (i, err) in errs.iter().enumerate() {
             let didx = object_to_delete_idx[i];
 
-            if err.is_none()
-                || err
-                    .clone()
-                    .is_some_and(|v| is_err_object_not_found(&v) || is_err_version_not_found(&v))
-            {
-                delete_results[didx].delete_object = Some(dobjs[i].clone());
-                let (versioned, version_suspended) = object_versioning[i];
-                let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
-                if creates_delete_marker {
-                    record_bucket_delete_marker_memory(&bucket).await;
-                } else {
-                    let size = object_sizes[i].max(0) as u64;
-                    record_bucket_object_delete_memory(
-                        &bucket,
-                        size,
-                        existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
-                    )
-                    .await;
+            match reduce_delete_objects_result(
+                &object_to_delete[i],
+                &dobjs[i],
+                err.as_ref(),
+                delete_results[didx].synthetic_version_id,
+            ) {
+                Ok(deleted_object) => {
+                    delete_results[didx].delete_object = Some(deleted_object.clone());
+                    let (versioned, version_suspended) = object_versioning[i];
+                    let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
+                    if creates_delete_marker {
+                        record_bucket_delete_marker_memory(&bucket).await;
+                    } else {
+                        let size = object_sizes[i].max(0) as u64;
+                        record_bucket_object_delete_memory(
+                            &bucket,
+                            size,
+                            existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
+                        )
+                        .await;
+                    }
+                }
+                Err(error) => {
+                    delete_results[didx].error = Some(error);
                 }
-                continue;
-            }
-
-            if let Some(err) = err.clone() {
-                let api_error = ApiError::from(err);
-                delete_results[didx].error = Some(s3s::dto::Error {
-                    code: Some(api_error.code.as_str().to_string()),
-                    key: Some(object_to_delete[i].object_name.clone()),
-                    message: Some(api_error.message),
-                    version_id: delete_response_version_id(
-                        object_to_delete[i].version_id,
-                        delete_results[didx].synthetic_version_id,
-                    ),
-                });
             }
         }
 
@@ -10740,6 +10768,7 @@ mod tests {
         let mut record = pax_record("minio.metadata.Content-Type", b"text/plain");
         record.extend(pax_record("minio.metadata.X-Amz-Meta-Owner", b"alice"));
         record.extend(pax_record("minio.metadata.project", b"alpha-demo"));
+        record.extend(pax_record("minio.metadata.x-amz-tagging", b"classification=public"));
         record.extend(pax_record("minio.versionId", Uuid::nil().to_string().as_bytes()));
         record.extend(pax_record("minio.metadata.x-amz-replication-status", b"REPLICA"));
         record.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"GOVERNANCE"));
@@ -10778,6 +10807,8 @@ mod tests {
         assert_eq!(metadata.get("content-type").map(String::as_str), Some("text/plain"));
         assert_eq!(metadata.get("owner").map(String::as_str), Some("alice"));
         assert_eq!(metadata.get("project").map(String::as_str), Some("alpha-demo"));
+        assert_eq!(metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("classification=public"));
+        assert!(!metadata.contains_key("x-amz-tagging"));
         assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
         assert_eq!(
             metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER).map(String::as_str),
@@ -17682,6 +17713,35 @@ mod tests {
         assert_eq!(internal_version_id, None);
     }
 
+    #[test]
+    fn delete_objects_treats_raw_io_not_found_as_idempotent() {
+        assert!(is_delete_objects_not_found(&StorageError::FileNotFound));
+        assert!(is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
+            std::io::ErrorKind::NotFound,
+        ))));
+        assert!(!is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
+            std::io::ErrorKind::PermissionDenied,
+        ))));
+        assert!(!is_delete_objects_not_found(&StorageError::DiskNotFound));
+    }
+
+    #[test]
+    fn delete_objects_result_reducer_reports_raw_not_found_as_deleted() {
+        let object = ObjectToDelete {
+            object_name: "missing-key".to_string(),
+            ..Default::default()
+        };
+        let deleted = StorageDeletedObject {
+            object_name: object.object_name.clone(),
+            ..Default::default()
+        };
+        let error = StorageError::Io(std::io::Error::from(std::io::ErrorKind::NotFound));
+
+        let deleted = reduce_delete_objects_result(&object, &deleted, Some(&error), false)
+            .expect("raw not-found must produce a deleted result");
+        assert_eq!(deleted.object_name, "missing-key");
+    }
+
     #[test]
     fn recursive_force_delete_requires_administrative_or_replica_context() {
         let mut headers = HeaderMap::new();

rustfs/src/bin/rustfs-cli.rs

@@ -17,6 +17,220 @@
 //! This binary shares RustFS's existing subcommand dispatcher and provides the
 //! documented entry point for offline tooling such as `inspect bucket-meta`.
 
-fn main() {
+use std::fs;
+use std::io::{Read as _, Write as _};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use rustfs::connect::offline::{OfflineEnrollment, OfflineKeyStore};
+
+/// Owner read/write only. The response names the key being enrolled and the
+/// challenge it answers; neither belongs to anyone else on the machine.
+#[cfg(unix)]
+const RESPONSE_MODE: u32 = 0o600;
+
+const USAGE: &str = "\
+Usage: rustfs-cli connect offline enroll --challenge <path|-> --output <path> [--key-dir <path>]
+
+Answers a Connect offline enrolment challenge without a network. Reads the
+challenge from a file or from stdin when the path is `-`, verifies it against the
+enrolment root compiled into this binary, mints the key being enrolled on first
+use, and writes the signed response.
+
+No secret is ever accepted on the command line.
+";
+
+fn main() -> ExitCode {
+    let arguments: Vec<String> = std::env::args().skip(1).collect();
+
+    // Offline enrolment is handled before the server dispatcher is reached, and
+    // the reason is the surface's whole point: `run_process` builds a Tokio
+    // runtime and enters the server's async main. An air-gapped enrolment must
+    // not start a runtime, a task, or anything that could open a socket, so the
+    // two paths cannot share an entry.
+    if matches!(
+        arguments.first().map(String::as_str),
+        Some("connect") if matches!(arguments.get(1).map(String::as_str), Some("offline"))
+    ) {
+        return match run_offline(&arguments[2..]) {
+            Ok(()) => ExitCode::SUCCESS,
+            Err(message) => {
+                eprintln!("rustfs-cli: {message}");
+                ExitCode::FAILURE
+            }
+        };
+    }
+
     rustfs::startup_entrypoint::run_process();
+
+    ExitCode::SUCCESS
+}
+
+fn run_offline(arguments: &[String]) -> Result<(), String> {
+    match arguments.first().map(String::as_str) {
+        Some("enroll") => enroll(&arguments[1..]),
+        Some(other) => Err(format!("unknown offline subcommand `{other}`\n\n{USAGE}")),
+        None => Err(format!("missing offline subcommand\n\n{USAGE}")),
+    }
+}
+
+fn enroll(arguments: &[String]) -> Result<(), String> {
+    let mut challenge_path: Option<String> = None;
+    let mut output_path: Option<String> = None;
+    let mut key_directory: Option<String> = None;
+
+    let mut index = 0;
+    while index < arguments.len() {
+        let flag = arguments[index].as_str();
+        let take_value = |name: &str| -> Result<String, String> {
+            arguments
+                .get(index + 1)
+                .cloned()
+                .ok_or_else(|| format!("`{name}` needs a value\n\n{USAGE}"))
+        };
+
+        match flag {
+            "--challenge" => challenge_path = Some(take_value("--challenge")?),
+            "--output" => output_path = Some(take_value("--output")?),
+            "--key-dir" => key_directory = Some(take_value("--key-dir")?),
+            "-h" | "--help" => {
+                println!("{USAGE}");
+                return Ok(());
+            }
+            other => return Err(format!("unknown option `{other}`\n\n{USAGE}")),
+        }
+
+        index += 2;
+    }
+
+    let challenge_path = challenge_path.ok_or_else(|| format!("`--challenge` is required\n\n{USAGE}"))?;
+    let output_path = output_path.ok_or_else(|| format!("`--output` is required\n\n{USAGE}"))?;
+    let key_directory = key_directory.unwrap_or_else(|| ".".to_string());
+
+    let challenge = read_challenge(&challenge_path)?;
+
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(|_| "the system clock is before the Unix epoch".to_string())?
+        .as_secs() as i64;
+
+    let verified = OfflineEnrollment::verify_challenge(&challenge, now).map_err(|error| error.to_string())?;
+
+    // First use mints the key; a retry answers with the one already enrolled,
+    // because the operator may already be carrying a response naming it.
+    let key = OfflineKeyStore::new(&key_directory)
+        .load_or_create()
+        .map_err(|error| error.to_string())?;
+
+    let mut device_nonce = [0u8; 32];
+    getrandom(&mut device_nonce)?;
+
+    let response = OfflineEnrollment::build_response(&verified, &key, &device_nonce, now).map_err(|error| error.to_string())?;
+
+    write_response(Path::new(&output_path), &response)?;
+
+    Ok(())
+}
+
+/// Reads the challenge from a file, or from stdin when the path is `-`.
+///
+/// A challenge is not a secret — it is signed, public, and carried in by hand —
+/// so accepting a path is safe. The response's key never arrives this way.
+fn read_challenge(path: &str) -> Result<Vec<u8>, String> {
+    if path == "-" {
+        let mut buffer = Vec::new();
+        std::io::stdin()
+            .read_to_end(&mut buffer)
+            .map_err(|error| format!("cannot read the challenge from stdin: {error}"))?;
+        return Ok(buffer);
+    }
+
+    fs::read(path).map_err(|error| format!("cannot read the challenge at {path}: {error}"))
+}
+
+/// Writes the response durably and atomically at mode 0600.
+///
+/// Not the no-clobber publish `IdentityStore` performs for a key: an operator
+/// who reruns an enrolment expects the response file to be replaced, whereas a
+/// second key would strand the first. Same durability, deliberately different
+/// publication rule.
+fn write_response(path: &Path, response: &[u8]) -> Result<(), String> {
+    let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty());
+    let temporary: PathBuf = match parent {
+        Some(parent) => parent.join(format!(".{}.tmp", file_name(path))),
+        None => PathBuf::from(format!(".{}.tmp", file_name(path))),
+    };
+
+    let mut options = fs::OpenOptions::new();
+    options.write(true).create(true).truncate(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt as _;
+        options.mode(RESPONSE_MODE);
+    }
+
+    let write = (|| -> std::io::Result<()> {
+        let mut file = options.open(&temporary)?;
+        file.write_all(response)?;
+
+        // The umask can only narrow the creation mode, so set the exact mode
+        // before the bytes become durable.
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt as _;
+            file.set_permissions(fs::Permissions::from_mode(RESPONSE_MODE))?;
+        }
+
+        file.sync_all()
+    })();
+
+    if let Err(error) = write {
+        let _ = fs::remove_file(&temporary);
+        return Err(format!("cannot write the response to {}: {error}", path.display()));
+    }
+
+    fs::rename(&temporary, path).map_err(|error| {
+        let _ = fs::remove_file(&temporary);
+        format!("cannot publish the response at {}: {error}", path.display())
+    })?;
+
+    if let Some(parent) = parent {
+        sync_directory(parent);
+    }
+
+    Ok(())
+}
+
+fn file_name(path: &Path) -> String {
+    path.file_name()
+        .map(|name| name.to_string_lossy().into_owned())
+        .unwrap_or_else(|| "response".to_string())
+}
+
+/// Fsync the directory so the renamed entry survives power loss. Directories
+/// cannot be opened for syncing on Windows, where this is a no-op.
+fn sync_directory(directory: &Path) {
+    #[cfg(unix)]
+    {
+        if let Ok(handle) = fs::File::open(directory) {
+            let _ = handle.sync_all();
+        }
+    }
+    #[cfg(not(unix))]
+    let _ = directory;
+}
+
+/// Fills `buffer` with operating-system randomness.
+///
+/// The device nonce must be unpredictable: it is what stops a captured response
+/// being replayed as a fresh one. Sourced through p256's pinned rand_core 0.6
+/// rather than the workspace `rand` 0.10, matching `identity.rs`; the two are
+/// different crate versions and only the pinned one is on p256's own path.
+fn getrandom(buffer: &mut [u8]) -> Result<(), String> {
+    use p256::elliptic_curve::rand_core::{OsRng, RngCore as _};
+
+    OsRng
+        .try_fill_bytes(buffer)
+        .map_err(|error| format!("the operating system random source failed: {error}"))
 }

rustfs/src/connect/mod.rs

@@ -27,6 +27,8 @@
 
 pub mod identity;
 pub mod identity_store;
+pub mod offline;
 
 pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
 pub use identity_store::{IdentityStore, StoreError};
+pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};

rustfs/src/connect/offline/enrollment.rs

@@ -0,0 +1,684 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Challenge verification and response production for offline enrolment.
+//!
+//! Two invariants carry the security of this surface and both are easy to break
+//! by accident:
+//!
+//! - Every signature is checked over the octets that arrived, never over a
+//!   re-serialised document. Parsing happens only to route the verification, and
+//!   nothing a parse yields is believed until the signature over those same
+//!   octets has verified.
+//! - The enrolment root is the constant in this file. It is never taken from a
+//!   challenge, a configuration file, or an operator prompt, so there is no
+//!   trust-on-first-use path an operator could be talked into.
+//!
+//! The order of the checks in [`OfflineEnrollment::verify_challenge`] is frozen
+//! by `verificationOrder.enrollmentChallenge` in
+//! `protocol/agent/v1/fixtures/offline-enrollment/trust-model.json`, and the
+//! signature encoding, the domain separation tags, and every rejection reason
+//! are frozen beside it. Reordering the checks changes which reason a given
+//! artifact produces, which is itself part of the contract.
+
+use base64::Engine as _;
+use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
+use p256::ecdsa::signature::{Signer as _, Verifier as _};
+use p256::ecdsa::{Signature, SigningKey, VerifyingKey};
+use p256::pkcs8::DecodePrivateKey as _;
+use serde::{Deserialize, Serialize};
+use sha2::{Digest as _, Sha256};
+use time::{Date, Month, OffsetDateTime, PrimitiveDateTime, Time};
+
+use crate::connect::identity::DeviceIdentity;
+
+/// The hosted enrolment root, compiled in. Both halves are pinned: the
+/// fingerprint identifies the root, and the point is what actually verifies the
+/// first link, so a build cannot be pointed at a different key by supplying one.
+const PINNED_ROOT_KEY_ID: &str = "df22e2806112debbe953672aafa186d699af0e97dd3fd2b09fa8359005fe348f";
+const PINNED_ROOT_PUBLIC_KEY: &str = "BFfx-K-FfEA5nK_Rz3IHacvRCkJyQ7JOd1geLyU6HKRZDgNezmVuKhvJ22VhemyjV__Gshk8JGGqOBzYPMD0p6s";
+
+/// Domain separation tags. A document that verifies under one of these must not
+/// be accepted for another artifact type, so the tag is part of the signature
+/// input rather than a property of the caller.
+const TAG_TRUST_LINK: &[u8] = b"rustfs-offline-trust-link-v1";
+const TAG_CHALLENGE: &[u8] = b"rustfs-offline-enrollment-challenge-v1";
+const TAG_RESPONSE: &[u8] = b"rustfs-offline-enrollment-response-v1";
+
+/// The single octet between the tag and the signed document.
+const DOMAIN_SEPARATOR: u8 = 0x00;
+
+const SIGNATURE_ALGORITHM: &str = "ES256";
+const PROTOCOL_VERSION: &str = "v1";
+const FORMAT_TRUST_LINK: &str = "rustfs.connect.offline.trustLink/1";
+const FORMAT_CHALLENGE: &str = "rustfs.connect.offline.enrollmentChallenge/1";
+const FORMAT_RESPONSE: &str = "rustfs.connect.offline.enrollmentResponse/1";
+
+/// DER SubjectPublicKeyInfo header for an uncompressed P-256 point. A keyId is
+/// the SHA-256 of this prefix followed by the 65 octet point, so the prefix is
+/// also how a device public key is recovered from its own DER encoding.
+const SPKI_PREFIX: [u8; 26] = [
+    0x30, 0x59, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03,
+    0x01, 0x07, 0x03, 0x42, 0x00,
+];
+
+/// Order of the P-256 group, and half of it. `r` and `s` must lie in `[1, n)`,
+/// and `s` additionally in `[1, n/2]`: ECDSA admits both `s` and `n - s`, and a
+/// signature with two spellings cannot serve as an artifact identity. Every
+/// ECDSA library accepts the malleated form, so the encoding layer rejects it.
+const GROUP_ORDER: [u8; 32] = [
+    0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbc, 0xe6, 0xfa, 0xad, 0xa7,
+    0x17, 0x9e, 0x84, 0xf3, 0xb9, 0xca, 0xc2, 0xfc, 0x63, 0x25, 0x51,
+];
+const MAX_S: [u8; 32] = [
+    0x7f, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xde, 0x73, 0x7d, 0x56, 0xd3,
+    0x8b, 0xcf, 0x42, 0x79, 0xdc, 0xe5, 0x61, 0x7e, 0x31, 0x92, 0xa8,
+];
+
+const SCALAR_OCTETS: usize = 32;
+const SIGNATURE_OCTETS: usize = 64;
+/// 64 octets as unpadded base64url. The length is checked before decoding so
+/// that `=` padding, the standard alphabet, DER, and a truncated value are all
+/// refused rather than repaired.
+const SIGNATURE_VALUE_CHARS: usize = 86;
+
+const PUBLIC_KEY_OCTETS: usize = 65;
+const PUBLIC_KEY_CHARS: usize = 87;
+/// SEC1 tag of an uncompressed point. Compressed and hybrid forms are refused.
+const UNCOMPRESSED_POINT: u8 = 0x04;
+
+const TIMESTAMP_CHARS: usize = 20;
+
+/// The chain is exactly two links: a pinned root issues the intermediate, and
+/// the intermediate issues the signing key. Roles are positional and the
+/// enumeration is closed.
+const CHAIN_LINK_COUNT: usize = 2;
+const CHAIN_ROLES: [&str; CHAIN_LINK_COUNT] = ["intermediate", "signing"];
+
+/// Skew allowed on the challenge window. A device may have no synchronised
+/// clock at all, so its own reading of "now" is advisory.
+const CLOCK_SKEW_TOLERANCE: i64 = 300;
+
+/// Longest life a challenge may claim. The issuer sets both ends of its own
+/// window, so the protocol bound is applied on top of the declared expiry
+/// rather than trusted from it.
+const MAX_CHALLENGE_LIFETIME: i64 = 604_800;
+
+/// A challenge that verified, with the fields the response has to echo.
+///
+/// Construction is the proof: a value of this type only exists after the chain
+/// closed on the pinned root and the challenge signature verified over the
+/// received octets.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct VerifiedChallenge {
+    pub challenge_id: String,
+    pub organization_name: String,
+    pub cluster_name: String,
+    pub nonce: String,
+    pub issued_at: String,
+    pub expires_at: String,
+    pub connect_key_id: String,
+    /// The signature value of the challenge, verbatim. It binds a response to
+    /// the one challenge it answers, so it is carried rather than recomputed.
+    pub challenge_proof: String,
+}
+
+/// Why an offline enrolment artifact was refused.
+///
+/// The variants are the frozen `reason` vocabulary of
+/// `fixtures/offline-enrollment/error-codes.json`, which spans both halves of
+/// the exchange. The device half implemented here produces the encoding, chain,
+/// version, and freshness reasons; the reasons that describe a response being
+/// evaluated against stored state — [`Self::ChallengeUnknown`],
+/// [`Self::ChallengeProofInvalid`], [`Self::DeviceProofInvalid`],
+/// [`Self::EnrollmentReplayed`], [`Self::OrganizationMismatch`], and
+/// [`Self::ClusterMismatch`] — are Connect's to raise and are named here so the
+/// two sides share one vocabulary.
+///
+/// No variant carries a payload: a rejection must never disclose key material,
+/// signature octets, nonces, or document bytes.
+#[derive(Debug, PartialEq, Eq, thiserror::Error)]
+pub enum EnrollmentError {
+    #[error("protocolVersion is missing, malformed, or names an unsupported major version")]
+    UnsupportedProtocol,
+
+    #[error("formatVersion is not a supported offline enrollment format")]
+    UnsupportedFormat,
+
+    #[error("the signature is not 64 octets of fixed-width r||s in unpadded base64url")]
+    SignatureMalformed,
+
+    #[error("the signature is not in its canonical low-S form")]
+    SignatureNotCanonical,
+
+    #[error("the signature does not verify over the received octets")]
+    SignatureInvalid,
+
+    #[error("the trust chain is not issued by a root pinned in this build")]
+    EnrollmentRootUnknown,
+
+    #[error("a trust link is invalid, misordered, or outside its validity at the challenge issuedAt")]
+    TrustChainInvalid,
+
+    #[error("connectKeyId is not the subject of the last trust link")]
+    ConnectKeyUnchained,
+
+    #[error("no issued challenge matches this challengeId")]
+    ChallengeUnknown,
+
+    #[error("the challenge is not yet valid at the evaluation time")]
+    ChallengeNotYetValid,
+
+    #[error("the challenge has expired at the evaluation time")]
+    ChallengeExpired,
+
+    #[error("the response nonce or challengeProof is not the one issued for this challenge")]
+    ChallengeProofInvalid,
+
+    #[error("the response does not prove possession of the device key it presents")]
+    DeviceProofInvalid,
+
+    #[error("the challenge was already consumed")]
+    EnrollmentReplayed,
+
+    #[error("the response names a different organization than the challenge it answers")]
+    OrganizationMismatch,
+
+    #[error("the response names a different cluster than the challenge it answers")]
+    ClusterMismatch,
+
+    /// The artifact could not be read as a signed enrolment document at all: the
+    /// envelope, the base64 of the signed octets, or a field the frozen order
+    /// reads before the signature verifies did not parse. The frozen reason set
+    /// has no code for a structurally unreadable document, so this variant maps
+    /// to none of them.
+    #[error("the offline enrollment document is not well formed")]
+    MalformedDocument,
+
+    /// A fault on this side of the exchange rather than in the artifact: the
+    /// device key did not round-trip through its own PKCS#8 encoding, or the
+    /// caller named an instant outside the representable calendar. Fails closed
+    /// because a half-produced response must never reach removable media.
+    #[error("the enrollment response could not be produced on this device")]
+    ResponseNotProduced,
+}
+
+impl EnrollmentError {
+    /// The frozen `reason` an operator and Connect both branch on.
+    ///
+    /// The `Display` message is prose and may be reworded; this is the stable
+    /// identifier, so nothing should parse the message instead. The two
+    /// variants with no frozen counterpart deliberately return codes outside
+    /// the frozen set rather than borrowing the nearest one, so a document that
+    /// simply failed to parse can never be reported as a signature or freshness
+    /// failure.
+    pub fn reason(&self) -> &'static str {
+        match self {
+            Self::UnsupportedProtocol => "UNSUPPORTED_PROTOCOL",
+            Self::UnsupportedFormat => "UNSUPPORTED_FORMAT",
+            Self::SignatureMalformed => "SIGNATURE_MALFORMED",
+            Self::SignatureNotCanonical => "SIGNATURE_NOT_CANONICAL",
+            Self::SignatureInvalid => "SIGNATURE_INVALID",
+            Self::EnrollmentRootUnknown => "ENROLLMENT_ROOT_UNKNOWN",
+            Self::TrustChainInvalid => "TRUST_CHAIN_INVALID",
+            Self::ConnectKeyUnchained => "CONNECT_KEY_UNCHAINED",
+            Self::ChallengeUnknown => "CHALLENGE_UNKNOWN",
+            Self::ChallengeNotYetValid => "CHALLENGE_NOT_YET_VALID",
+            Self::ChallengeExpired => "CHALLENGE_EXPIRED",
+            Self::ChallengeProofInvalid => "CHALLENGE_PROOF_INVALID",
+            Self::DeviceProofInvalid => "DEVICE_PROOF_INVALID",
+            Self::EnrollmentReplayed => "ENROLLMENT_REPLAYED",
+            Self::OrganizationMismatch => "ORGANIZATION_MISMATCH",
+            Self::ClusterMismatch => "CLUSTER_MISMATCH",
+            Self::MalformedDocument => "MALFORMED_DOCUMENT",
+            Self::ResponseNotProduced => "RESPONSE_NOT_PRODUCED",
+        }
+    }
+}
+
+/// A signed document, in the shape both directions carry it. `bytes` is
+/// standard padded base64 of the exact octets that were signed; nothing else is
+/// ever used as the signature input.
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SignedDocument {
+    bytes: String,
+    signature: DocumentSignature,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct DocumentSignature {
+    algorithm: String,
+    key_id: String,
+    value: String,
+}
+
+/// The three fields the frozen order permits reading before anything verifies.
+/// They route the verification and are not facts until it has.
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ChallengeRouting {
+    connect_key_id: String,
+    issued_at: String,
+    trust_chain: Vec<SignedDocument>,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ChallengeDocument {
+    format_version: String,
+    protocol_version: String,
+    challenge_id: String,
+    organization_name: String,
+    cluster_name: String,
+    nonce: String,
+    issued_at: String,
+    expires_at: String,
+    connect_key_id: String,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TrustLink {
+    format_version: String,
+    protocol_version: String,
+    role: String,
+    issuer_key_id: String,
+    subject_key_id: String,
+    subject_public_key: String,
+    not_before: String,
+    not_after: String,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ResponseDocument<'a> {
+    format_version: &'a str,
+    protocol_version: &'a str,
+    challenge_id: &'a str,
+    organization_name: &'a str,
+    cluster_name: &'a str,
+    challenge_nonce: &'a str,
+    challenge_proof: &'a str,
+    device_key_id: String,
+    device_public_key: String,
+    device_nonce: String,
+    produced_at: String,
+}
+
+/// The device half of the offline enrolment exchange: bytes in, bytes out.
+pub struct OfflineEnrollment;
+
+impl OfflineEnrollment {
+    /// Verify an enrolment challenge and return what a response must echo.
+    ///
+    /// `now_unix` is the device's reading of the current time, which the clock
+    /// skew tolerance treats as advisory.
+    pub fn verify_challenge(document: &[u8], now_unix: i64) -> Result<VerifiedChallenge, EnrollmentError> {
+        let envelope: SignedDocument = serde_json::from_slice(document).map_err(|_| EnrollmentError::MalformedDocument)?;
+
+        // Step 1: the encoding is checked before anything is decoded from it, so
+        // a DER, padded, truncated, out-of-range, or high-S signature is refused
+        // on its spelling rather than handed to a library that would accept it.
+        let signature = decode_signature(&envelope.signature)?;
+
+        // The octets that were transmitted. They are never re-serialised: every
+        // later step signs and parses this same buffer.
+        let bytes = BASE64_STANDARD
+            .decode(envelope.bytes.as_bytes())
+            .map_err(|_| EnrollmentError::MalformedDocument)?;
+
+        // Step 2: routing only.
+        let routing: ChallengeRouting = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::MalformedDocument)?;
+        let issued_at = parse_timestamp(&routing.issued_at)?;
+
+        // Steps 3 to 5.
+        let connect_key = verify_trust_chain(&routing.trust_chain, &routing.connect_key_id, issued_at)?;
+
+        // Step 6. The verification key comes from the chain, so `signature.keyId`
+        // is a label rather than an input: a value naming some other key simply
+        // fails to verify here.
+        if !verifies(&connect_key, TAG_CHALLENGE, &bytes, &signature) {
+            return Err(EnrollmentError::SignatureInvalid);
+        }
+
+        // Step 7: only now is the document read as a fact.
+        let challenge: ChallengeDocument = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::MalformedDocument)?;
+        if challenge.protocol_version != PROTOCOL_VERSION {
+            return Err(EnrollmentError::UnsupportedProtocol);
+        }
+        if challenge.format_version != FORMAT_CHALLENGE {
+            return Err(EnrollmentError::UnsupportedFormat);
+        }
+
+        // Step 8.
+        let expires_at = parse_timestamp(&challenge.expires_at)?;
+        check_challenge_window(issued_at, expires_at, now_unix)?;
+
+        Ok(VerifiedChallenge {
+            challenge_id: challenge.challenge_id,
+            organization_name: challenge.organization_name,
+            cluster_name: challenge.cluster_name,
+            nonce: challenge.nonce,
+            issued_at: challenge.issued_at,
+            expires_at: challenge.expires_at,
+            connect_key_id: challenge.connect_key_id,
+            challenge_proof: envelope.signature.value,
+        })
+    }
+
+    /// Build the signed response an operator carries back to Connect.
+    ///
+    /// `device_nonce` is the response's own replay value and must come from a
+    /// cryptographic source. The private key never appears in the result: only
+    /// the public point, its fingerprint, and a signature over the document
+    /// that presents them, which is what makes presenting the key safe.
+    pub fn build_response(
+        challenge: &VerifiedChallenge,
+        key: &DeviceIdentity,
+        device_nonce: &[u8; 32],
+        produced_at_unix: i64,
+    ) -> Result<Vec<u8>, EnrollmentError> {
+        let issued_at = parse_timestamp(&challenge.issued_at)?;
+        let expires_at = parse_timestamp(&challenge.expires_at)?;
+        // Connect re-checks producedAt against the same window, so a response
+        // outside it is refused here rather than written to media and rejected
+        // after the operator has carried it out.
+        check_challenge_window(issued_at, expires_at, produced_at_unix)?;
+
+        let point = device_public_point(key)?;
+        let produced_at = format_timestamp(produced_at_unix)?;
+
+        let document = ResponseDocument {
+            format_version: FORMAT_RESPONSE,
+            protocol_version: PROTOCOL_VERSION,
+            challenge_id: &challenge.challenge_id,
+            organization_name: &challenge.organization_name,
+            cluster_name: &challenge.cluster_name,
+            challenge_nonce: &challenge.nonce,
+            challenge_proof: &challenge.challenge_proof,
+            device_key_id: key_id(&point),
+            device_public_key: BASE64_URL_NO_PAD.encode(point),
+            device_nonce: BASE64_URL_NO_PAD.encode(device_nonce),
+            produced_at,
+        };
+
+        // Serialised once. These octets are what is signed and what is carried,
+        // so no second serialisation can disagree with the signature.
+        let bytes = serde_json::to_vec(&document).map_err(|_| EnrollmentError::ResponseNotProduced)?;
+        let signature = sign(key, TAG_RESPONSE, &bytes)?;
+
+        let envelope = SignedDocument {
+            bytes: BASE64_STANDARD.encode(&bytes),
+            signature: DocumentSignature {
+                algorithm: SIGNATURE_ALGORITHM.to_owned(),
+                key_id: document.device_key_id,
+                value: signature,
+            },
+        };
+
+        serde_json::to_vec(&envelope).map_err(|_| EnrollmentError::ResponseNotProduced)
+    }
+}
+
+/// Walk the chain from the pinned root to the signing key, returning the key
+/// `connect_key_id` names once the chain vouches for it.
+fn verify_trust_chain(
+    chain: &[SignedDocument],
+    connect_key_id: &str,
+    challenge_issued_at: i64,
+) -> Result<VerifyingKey, EnrollmentError> {
+    // The pinned root gate runs before the chain's shape is examined, so a
+    // chain that is internally consistent under a foreign root — exactly what
+    // trust on first use would have accepted — is refused for its root rather
+    // than for its length.
+    let first = chain.first().ok_or(EnrollmentError::EnrollmentRootUnknown)?;
+    let root = decode_trust_link(first)?;
+    if root.0.issuer_key_id != PINNED_ROOT_KEY_ID {
+        return Err(EnrollmentError::EnrollmentRootUnknown);
+    }
+
+    let [_, second] = chain else {
+        return Err(EnrollmentError::TrustChainInvalid);
+    };
+    let links = [root, decode_trust_link(second)?];
+
+    let mut issuer_key_id = PINNED_ROOT_KEY_ID.to_owned();
+    let (mut issuer_key, _) = decode_public_key(PINNED_ROOT_PUBLIC_KEY).ok_or(EnrollmentError::EnrollmentRootUnknown)?;
+
+    for (index, ((link, link_bytes), entry)) in links.iter().zip(chain).enumerate() {
+        if link.format_version != FORMAT_TRUST_LINK
+            || link.protocol_version != PROTOCOL_VERSION
+            || link.role != CHAIN_ROLES[index]
+            || link.issuer_key_id != issuer_key_id
+            // A link that names itself as its own issuer would let a stolen
+            // intermediate mint its own root.
+            || link.subject_key_id == link.issuer_key_id
+        {
+            return Err(EnrollmentError::TrustChainInvalid);
+        }
+
+        let (subject_key, subject_point) =
+            decode_public_key(&link.subject_public_key).ok_or(EnrollmentError::TrustChainInvalid)?;
+        if key_id(&subject_point) != link.subject_key_id {
+            return Err(EnrollmentError::TrustChainInvalid);
+        }
+
+        let signature = decode_signature(&entry.signature)?;
+        if !verifies(&issuer_key, TAG_TRUST_LINK, link_bytes, &signature) {
+            return Err(EnrollmentError::TrustChainInvalid);
+        }
+
+        // The issuer controls both ends of a link's window, so it is evaluated
+        // with no skew tolerance, and against the challenge's issuedAt rather
+        // than against the device clock: a challenge carries the chain that was
+        // valid when it was issued.
+        let not_before = parse_timestamp(&link.not_before)?;
+        let not_after = parse_timestamp(&link.not_after)?;
+        if challenge_issued_at < not_before || challenge_issued_at > not_after {
+            return Err(EnrollmentError::TrustChainInvalid);
+        }
+
+        issuer_key_id = link.subject_key_id.clone();
+        issuer_key = subject_key;
+    }
+
+    if issuer_key_id != connect_key_id {
+        return Err(EnrollmentError::ConnectKeyUnchained);
+    }
+
+    Ok(issuer_key)
+}
+
+/// Decode a link and keep the octets it was signed over: the signature is
+/// checked against these, never against a re-encoding of the parsed link.
+fn decode_trust_link(entry: &SignedDocument) -> Result<(TrustLink, Vec<u8>), EnrollmentError> {
+    let bytes = BASE64_STANDARD
+        .decode(entry.bytes.as_bytes())
+        .map_err(|_| EnrollmentError::MalformedDocument)?;
+    let link = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?;
+    Ok((link, bytes))
+}
+
+/// Check a signature's spelling and range, then admit it.
+///
+/// `r` and `s` are compared against the group order here rather than left to
+/// the ECDSA library, because a library that accepts high-S — every library
+/// does — would let a malleated copy of an artifact pass as a second artifact.
+fn decode_signature(signature: &DocumentSignature) -> Result<Signature, EnrollmentError> {
+    if signature.algorithm != SIGNATURE_ALGORITHM {
+        return Err(EnrollmentError::SignatureMalformed);
+    }
+
+    let value = signature.value.as_bytes();
+    if value.len() != SIGNATURE_VALUE_CHARS || !value.iter().all(|byte| is_base64url(*byte)) {
+        return Err(EnrollmentError::SignatureMalformed);
+    }
+
+    let decoded = BASE64_URL_NO_PAD
+        .decode(value)
+        .map_err(|_| EnrollmentError::SignatureMalformed)?;
+    let octets: [u8; SIGNATURE_OCTETS] = decoded
+        .as_slice()
+        .try_into()
+        .map_err(|_| EnrollmentError::SignatureMalformed)?;
+
+    // Big-endian octets of equal length order lexicographically exactly as the
+    // integers they spell, so a slice comparison is the range check.
+    let (r, s) = octets.split_at(SCALAR_OCTETS);
+    let out_of_range = |scalar: &[u8]| scalar.iter().all(|byte| *byte == 0) || scalar >= &GROUP_ORDER[..];
+    if out_of_range(r) || out_of_range(s) {
+        return Err(EnrollmentError::SignatureMalformed);
+    }
+    if s > &MAX_S[..] {
+        return Err(EnrollmentError::SignatureNotCanonical);
+    }
+
+    Signature::from_slice(&octets).map_err(|_| EnrollmentError::SignatureMalformed)
+}
+
+fn verifies(key: &VerifyingKey, tag: &[u8], bytes: &[u8], signature: &Signature) -> bool {
+    key.verify(&signature_input(tag, bytes), signature).is_ok()
+}
+
+fn signature_input(tag: &[u8], bytes: &[u8]) -> Vec<u8> {
+    let mut input = Vec::with_capacity(tag.len() + 1 + bytes.len());
+    input.extend_from_slice(tag);
+    input.push(DOMAIN_SEPARATOR);
+    input.extend_from_slice(bytes);
+    input
+}
+
+fn sign(key: &DeviceIdentity, tag: &[u8], bytes: &[u8]) -> Result<String, EnrollmentError> {
+    // `DeviceIdentity` publishes no general signing operation, so the key is
+    // rebuilt from its own PKCS#8 encoding; the encoding is wiped when the
+    // wrapper drops.
+    let pkcs8 = key.to_pkcs8_der().map_err(|_| EnrollmentError::ResponseNotProduced)?;
+    let signing_key = SigningKey::from_pkcs8_der(pkcs8.as_slice()).map_err(|_| EnrollmentError::ResponseNotProduced)?;
+
+    let signature: Signature = signing_key.sign(&signature_input(tag, bytes));
+    let canonical = signature.normalize_s().unwrap_or(signature);
+
+    Ok(BASE64_URL_NO_PAD.encode(canonical.to_bytes()))
+}
+
+/// The device's public point, recovered from the DER encoding the identity
+/// publishes so that one prefix constant governs both the fingerprint and the
+/// wire form.
+fn device_public_point(key: &DeviceIdentity) -> Result<[u8; PUBLIC_KEY_OCTETS], EnrollmentError> {
+    key.public_key_der()
+        .strip_prefix(&SPKI_PREFIX)
+        .and_then(|point| <[u8; PUBLIC_KEY_OCTETS]>::try_from(point).ok())
+        .ok_or(EnrollmentError::ResponseNotProduced)
+}
+
+/// Decode an uncompressed SEC1 point and check that it is on the curve.
+///
+/// The length and alphabet are checked before decoding so that a padded or
+/// standard-alphabet spelling is refused, and the point tag is checked so that
+/// the compressed and hybrid forms — which no keyId would match — cannot be
+/// spelled at all.
+fn decode_public_key(value: &str) -> Option<(VerifyingKey, [u8; PUBLIC_KEY_OCTETS])> {
+    let value = value.as_bytes();
+    if value.len() != PUBLIC_KEY_CHARS || !value.iter().all(|byte| is_base64url(*byte)) {
+        return None;
+    }
+
+    let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode(value).ok()?.try_into().ok()?;
+    if point[0] != UNCOMPRESSED_POINT {
+        return None;
+    }
+
+    VerifyingKey::from_sec1_bytes(&point).ok().map(|key| (key, point))
+}
+
+/// Lowercase SHA-256 hex of the DER SubjectPublicKeyInfo built from a 65 octet
+/// uncompressed point.
+fn key_id(point: &[u8]) -> String {
+    let mut digest = Sha256::new();
+    digest.update(SPKI_PREFIX);
+    digest.update(point);
+    hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower)
+}
+
+fn is_base64url(byte: u8) -> bool {
+    byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_'
+}
+
+/// Parse `YYYY-MM-DDTHH:MM:SSZ` into a Unix instant.
+///
+/// The shape is checked before the fields are read: offsets other than `Z` and
+/// fractional seconds are refused rather than normalised, so two producers
+/// cannot spell the same instant two ways.
+fn parse_timestamp(value: &str) -> Result<i64, EnrollmentError> {
+    let octets = value.as_bytes();
+    if octets.len() != TIMESTAMP_CHARS
+        || octets[4] != b'-'
+        || octets[7] != b'-'
+        || octets[10] != b'T'
+        || octets[13] != b':'
+        || octets[16] != b':'
+        || octets[19] != b'Z'
+    {
+        return Err(EnrollmentError::MalformedDocument);
+    }
+
+    let field = |range: std::ops::Range<usize>| -> Result<u32, EnrollmentError> {
+        let text = &value[range];
+        if !text.bytes().all(|byte| byte.is_ascii_digit()) {
+            return Err(EnrollmentError::MalformedDocument);
+        }
+        text.parse().map_err(|_| EnrollmentError::MalformedDocument)
+    };
+
+    let month = Month::try_from(field(5..7)? as u8).map_err(|_| EnrollmentError::MalformedDocument)?;
+    let date = Date::from_calendar_date(field(0..4)? as i32, month, field(8..10)? as u8)
+        .map_err(|_| EnrollmentError::MalformedDocument)?;
+    let clock = Time::from_hms(field(11..13)? as u8, field(14..16)? as u8, field(17..19)? as u8)
+        .map_err(|_| EnrollmentError::MalformedDocument)?;
+
+    Ok(PrimitiveDateTime::new(date, clock).assume_utc().unix_timestamp())
+}
+
+fn format_timestamp(unix: i64) -> Result<String, EnrollmentError> {
+    let moment = OffsetDateTime::from_unix_timestamp(unix).map_err(|_| EnrollmentError::ResponseNotProduced)?;
+    Ok(format!(
+        "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
+        moment.year(),
+        u8::from(moment.month()),
+        moment.day(),
+        moment.hour(),
+        moment.minute(),
+        moment.second()
+    ))
+}
+
+/// `at` must fall within `[issuedAt - 300, expiresAt + 300]`.
+///
+/// The declared expiry is capped at the protocol's maximum challenge lifetime
+/// because the issuer sets both ends of its own window; a challenge claiming a
+/// longer life expires at the bound.
+fn check_challenge_window(issued_at: i64, expires_at: i64, at: i64) -> Result<(), EnrollmentError> {
+    if at < issued_at.saturating_sub(CLOCK_SKEW_TOLERANCE) {
+        return Err(EnrollmentError::ChallengeNotYetValid);
+    }
+
+    let effective_expiry = expires_at.min(issued_at.saturating_add(MAX_CHALLENGE_LIFETIME));
+    if at > effective_expiry.saturating_add(CLOCK_SKEW_TOLERANCE) {
+        return Err(EnrollmentError::ChallengeExpired);
+    }
+
+    Ok(())
+}

rustfs/src/connect/offline/key_store.rs

@@ -0,0 +1,72 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! On-disk home of the offline enrollment key.
+//!
+//! An air-gapped device enrols with a key that is not its online device
+//! identity: the online key is minted during a registration exchange this
+//! device cannot perform, and an operator who carries an enrolment response out
+//! on removable media is enrolling exactly one key that Connect will pin. Losing
+//! it means asking for a fresh challenge, so it is written durably and published
+//! exactly once.
+//!
+//! The durability protocol is not reimplemented here. [`IdentityStore`] already
+//! seals a P-256 key at mode 0600, fsyncs it, and publishes it through a
+//! no-clobber link so a retry or a concurrent start converges on one key; it is
+//! pointed at a directory of this key's own rather than generalised into a
+//! key-store abstraction that would have to describe both lifecycles.
+
+use std::path::{Path, PathBuf};
+
+use super::super::identity::DeviceIdentity;
+use super::super::identity_store::{IdentityStore, StoreError};
+
+/// Subdirectory holding the offline enrolment key, kept apart from the online
+/// device identity so neither can be read in place of the other.
+const OFFLINE_DIRECTORY: &str = "offline";
+
+/// The offline enrolment key of one deployment.
+#[derive(Clone, Debug)]
+pub struct OfflineKeyStore {
+    inner: IdentityStore,
+}
+
+impl OfflineKeyStore {
+    pub fn new(directory: impl AsRef<Path>) -> Self {
+        Self {
+            inner: IdentityStore::new(directory.as_ref().join(OFFLINE_DIRECTORY)),
+        }
+    }
+
+    pub fn key_path(&self) -> PathBuf {
+        self.inner.key_path()
+    }
+
+    /// Return the stored key, or `None` when this deployment has never enrolled
+    /// offline. Reading never creates one, so a deployment that only ever
+    /// registers online holds no offline key.
+    pub fn load(&self) -> Result<Option<DeviceIdentity>, StoreError> {
+        self.inner.load()
+    }
+
+    /// Return the stored key, generating and publishing one the first time.
+    ///
+    /// A second enrolment attempt returns the original key rather than minting a
+    /// replacement: the operator may already be carrying a response for it, and
+    /// two keys would mean the response and the device disagree about which one
+    /// Connect pinned.
+    pub fn load_or_create(&self) -> Result<DeviceIdentity, StoreError> {
+        self.inner.load_or_create()
+    }
+}

rustfs/src/connect/offline/mod.rs

@@ -0,0 +1,35 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Offline enrolment: joining a Connect tenant without a network.
+//!
+//! An air-gapped cluster cannot perform the registration exchange, so an
+//! operator carries a signed challenge in and a signed response out. The device
+//! half of that exchange lives here: verifying the challenge against a root
+//! whose fingerprint is compiled into this binary, minting the key being
+//! enrolled, and signing the response.
+//!
+//! Nothing here opens a socket. That is the point of the surface, and it is
+//! asserted rather than assumed: the enrolment path takes bytes and returns
+//! bytes.
+//!
+//! The trust model, the signing convention, and every rejection reason are
+//! frozen by `protocol/agent/v1/fixtures/offline-enrollment/` and by
+//! `docs/adr/0009-offline-signing.md` on the Connect side.
+
+pub mod enrollment;
+pub mod key_store;
+
+pub use enrollment::{EnrollmentError, OfflineEnrollment, VerifiedChallenge};
+pub use key_store::OfflineKeyStore;

rustfs/src/protocols/client.rs

@@ -13,10 +13,16 @@
 // limitations under the License.
 
 use crate::runtime_sources::current_action_credentials;
+#[cfg(feature = "webdav")]
+use crate::shared_types::RemoteAddr;
 use crate::storage_api::protocols::client::{FS, ReqInfo, RequestContext};
 use http::{HeaderMap, Method};
 use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
 use rustfs_credentials;
+#[cfg(feature = "webdav")]
+use rustfs_protocols::common::SessionContext;
+#[cfg(feature = "webdav")]
+use rustfs_trusted_proxies::ClientInfo;
 use rustfs_utils::MaskedAccessKey;
 use s3s::dto::*;
 use s3s::{S3, S3Request, S3Result};
@@ -90,6 +96,50 @@ fn trace_protocol_request(operation: &str, bucket: Option<&str>, object: Option<
     );
 }
 
+#[cfg(feature = "webdav")]
+fn session_list_buckets_request(
+    input: ListBucketsInput,
+    session_context: &SessionContext,
+    request_headers: &HeaderMap,
+    secure_transport: bool,
+) -> S3Request<ListBucketsInput> {
+    let credentials = &session_context.principal.user_identity.credentials;
+    let mut extensions = http::Extensions::default();
+    let remote_addr = std::net::SocketAddr::new(session_context.source_ip, 0);
+    extensions.insert(Some(RemoteAddr(remote_addr)));
+    let mut client_info = ClientInfo::direct(remote_addr);
+    client_info.forwarded_proto = Some(if secure_transport { "https" } else { "http" }.to_string());
+    extensions.insert(client_info);
+
+    let is_owner = current_action_credentials().is_some_and(|global_cred| credentials.access_key == global_cred.access_key);
+    extensions.insert(ReqInfo {
+        cred: Some(credentials.clone()),
+        is_owner,
+        bucket: None,
+        object: None,
+        version_id: None,
+        replication_request_authorized: false,
+        region: None,
+        request_context: Some(RequestContext::fallback()),
+        suppress_denial_log: false,
+    });
+
+    S3Request {
+        input,
+        method: Method::GET,
+        uri: http::Uri::from_static("/"),
+        headers: request_headers.clone(),
+        extensions,
+        credentials: Some(s3s::auth::Credentials {
+            access_key: credentials.access_key.clone(),
+            secret_key: credentials.secret_key.clone().into(),
+        }),
+        region: None,
+        service: None,
+        trailing_headers: None,
+    }
+}
+
 fn build_bucket_uri(bucket: &str, query: &[(&str, Option<&str>)]) -> S3Result<http::Uri> {
     let mut uri = format!("/{}", encode_path_segment(bucket));
     let mut first = true;
@@ -469,6 +519,29 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        session_context: &SessionContext,
+        request_headers: &HeaderMap,
+        secure_transport: bool,
+    ) -> Result<ListBucketsOutput, Self::Error> {
+        trace!(
+            event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
+            component = LOG_COMPONENT_PROTOCOLS,
+            subsystem = LOG_SUBSYSTEM_STORAGE_CLIENT,
+            operation = "list_buckets",
+            access_key = %MaskedAccessKey(&session_context.principal.user_identity.credentials.access_key),
+            "Protocol storage client request"
+        );
+
+        let input = ListBucketsInput::builder().build().map_err(|e| {
+            s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build ListBucketsInput: {}", e))
+        })?;
+        let request = session_list_buckets_request(input, session_context, request_headers, secure_transport);
+        self.fs.list_buckets(request).await.map(|response| response.output)
+    }
+
     async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error> {
         trace_protocol_request("create_bucket", Some(bucket), None);
 
@@ -872,6 +945,60 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[cfg(feature = "webdav")]
+    #[test]
+    fn request_extensions_preserve_authenticated_identity_and_source_ip() {
+        use std::collections::HashMap;
+        use std::net::{IpAddr, Ipv4Addr};
+
+        let claims = HashMap::from([("parent".to_string(), serde_json::json!("alice"))]);
+        let credentials = rustfs_credentials::Credentials {
+            access_key: "service-account".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: "session-token".to_string(),
+            parent_user: "alice".to_string(),
+            groups: Some(vec!["developers".to_string()]),
+            claims: Some(claims.clone()),
+            ..Default::default()
+        };
+        let source_ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10));
+
+        let identity = rustfs_policy::auth::UserIdentity {
+            credentials: credentials.clone(),
+            ..Default::default()
+        };
+        let principal = rustfs_protocols::common::ProtocolPrincipal::new(std::sync::Arc::new(identity));
+        let session_context = SessionContext::new(principal, rustfs_protocols::Protocol::WebDav, source_ip);
+        let mut headers = HeaderMap::new();
+        headers.insert("user-agent", http::HeaderValue::from_static("webdav-client"));
+        let request = session_list_buckets_request(ListBucketsInput::default(), &session_context, &headers, true);
+        let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
+        let copied = request_info.cred.as_ref().expect("credentials should be present");
+        let remote_addr = request
+            .extensions
+            .get::<Option<RemoteAddr>>()
+            .and_then(Option::as_ref)
+            .expect("remote address should be present");
+        let client_info = request.extensions.get::<ClientInfo>().expect("client info should be present");
+
+        assert_eq!(copied.access_key, credentials.access_key);
+        assert_eq!(copied.secret_key, credentials.secret_key);
+        assert_eq!(copied.session_token, credentials.session_token);
+        assert_eq!(copied.parent_user, credentials.parent_user);
+        assert_eq!(copied.groups, credentials.groups);
+        assert_eq!(copied.claims, Some(claims));
+        assert_eq!(remote_addr.0.ip(), source_ip);
+        assert_eq!(client_info.real_ip, source_ip);
+        assert_eq!(client_info.forwarded_proto.as_deref(), Some("https"));
+        assert_eq!(request.headers.get("user-agent").expect("user agent"), "webdav-client");
+
+        let insecure_request = session_list_buckets_request(ListBucketsInput::default(), &session_context, &headers, false);
+        let insecure_client_info = insecure_request
+            .extensions
+            .get::<ClientInfo>()
+            .expect("client info should be present");
+        assert_eq!(insecure_client_info.forwarded_proto.as_deref(), Some("http"));
+    }
 
     #[test]
     fn build_object_uri_encodes_key_segments_without_flattening_slashes() {

rustfs/src/server/http.rs

@@ -1537,7 +1537,7 @@ fn process_connection(
                 None
             }
         };
-        // ── Canonical Middleware Stack Order (outermost → innermost) ──
+        // ── Canonical External Middleware Stack Order (outermost → innermost) ──
         // This order MUST be preserved across refactorings.
         // Only AddExtensionLayer (layers 1-2) are per-connection; most remaining layers are stateless.
         //
@@ -1565,6 +1565,8 @@ fn process_connection(
         // 22. PublicHealthEndpointLayer              — handles public health before s3s host parsing
         // 23. VirtualHostStyleHintLayer              — actionable error for unroutable virtual-hosted-style (conditional)
         // 24. DoubleSlashListBucketsCompatLayer      — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
+        // The internode lane below intentionally keeps only the shared
+        // transport/auth/observability subset needed by `/rustfs/rpc/...`.
         // ─────────────────────────────────────────────────────────────
         let build_external_stack = |service| {
             ServiceBuilder::new()
@@ -1747,16 +1749,9 @@ fn process_connection(
                 .layer(PropagateRequestIdLayer::x_request_id())
                 .layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
                 .option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
-                .layer(S3ErrorMessageCompatLayer)
-                .layer(IcebergRestErrorCompatLayer)
-                .layer(ObjectAttributesEtagFixLayer)
-                .layer(ConditionalCorsLayer::new())
-                .option_layer(if is_console { Some(RedirectLayer) } else { None })
-                .layer(BodylessStatusFixLayer)
-                .layer(HeadRequestBodyFixLayer)
-                .layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
-                .option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
-                .layer(DoubleSlashListBucketsCompatLayer)
+                // The internode lane only serves `/rustfs/rpc/...` gRPC requests.
+                // Keep safety/observability layers above, but leave S3/REST
+                // compatibility rewrites on the external lane.
                 .service(service)
         };
         let external_stack_service = build_external_stack(external_service);

rustfs/src/storage/rpc/http_service.rs

@@ -1500,7 +1500,7 @@ where
         return write_body_chunks_to_writer(body, writer).await;
     };
 
-    let expected_size = (!query.append && query.size >= 0)
+    let expected_size = (!query.append && query.size > 0)
         .then(|| {
             u64::try_from(query.size)
                 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "put_file auth size cannot be represented"))
@@ -2325,6 +2325,39 @@ mod tests {
         assert_eq!(writer, b"append-data");
     }
 
+    #[tokio::test]
+    async fn put_file_auth_zero_size_create_uses_trailing_auth_record() {
+        let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-body-test-secret".to_string());
+        let nonce = uuid::Uuid::parse_str("43434343-4444-4555-8666-777777777777").expect("nonce");
+        let url = concat!(
+            "/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
+            "&append=false&size=0&put_file_auth=digest-trailer-v1&put_file_nonce=43434343-4444-4555-8666-777777777777"
+        );
+        let digest = hex_simd::encode_to_string(sha2::Sha256::digest(b"unknown-size-data"), hex_simd::AsciiCase::Lower);
+        let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &digest).expect("trailer should build");
+        let query = PutFileQuery {
+            disk: "disk-a".to_string(),
+            volume: "bucket".to_string(),
+            path: "object/part.1".to_string(),
+            append: false,
+            size: 0,
+            put_file_auth: Some("digest-trailer-v1".to_string()),
+            put_file_nonce: Some(nonce),
+            put_file_server_epoch: Some(*super::PUT_FILE_CAPABILITY_SERVER_EPOCH),
+        };
+        let mut payload = b"unknown-size-data".to_vec();
+        payload.extend_from_slice(&trailer);
+        let body = iter(vec![Ok::<Bytes, io::Error>(Bytes::from(payload))]);
+        let mut writer = Vec::new();
+
+        let copied = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
+            .await
+            .expect("zero-size create body should verify");
+
+        assert_eq!(copied, 17);
+        assert_eq!(writer, b"unknown-size-data");
+    }
+
     #[tokio::test]
     async fn put_file_auth_append_body_rejects_missing_trailer() {
         let nonce = uuid::Uuid::parse_str("44444444-5555-4666-8777-888888888888").expect("nonce");

rustfs/tests/connect_offline_enrollment.rs

@@ -0,0 +1,951 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Offline enrollment conformance against the frozen Connect fixtures.
+//!
+//! The device half of the air-gapped exchange verifies a challenge Connect
+//! signed and produces a response Connect will verify. Neither side can talk to
+//! the other while it does so, which means every disagreement about encoding,
+//! trust, or clock windows surfaces as a failed enrollment in the field rather
+//! than as an error at development time. The fixtures under
+//! `protocol/agent/v1/fixtures/offline-enrollment/` are the shared statement of
+//! what both sides must do, so this suite replays them rather than restating
+//! them: accept vectors must be accepted with the fields the document carries,
+//! reject vectors must fail with the single reason `error-codes.json` freezes,
+//! and the signature encoding rules in `trust-model.json` must hold even where
+//! the underlying ECDSA library is happy.
+
+use std::fs;
+use std::path::PathBuf;
+
+use base64::Engine as _;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
+use rustfs::connect::identity::DeviceIdentity;
+use rustfs::connect::offline::{EnrollmentError, OfflineEnrollment, VerifiedChallenge};
+use serde_json::Value;
+use sha2::{Digest as _, Sha256};
+
+/// DER prefix of a P-256 `SubjectPublicKeyInfo`, frozen by
+/// `trust-model.json` as `signature.subjectPublicKeyInfoDerPrefix`. The 65
+/// octet uncompressed point follows it, so a SEC1 point published in a fixture
+/// becomes a decodable public key by concatenation.
+const SPKI_PREFIX_HEX: &str = "3059301306072a8648ce3d020106082a8648ce3d030107034200";
+
+/// `clockSkew.toleranceSeconds` in `trust-model.json`.
+const SKEW_TOLERANCE_SECONDS: i64 = 300;
+
+// ---------------------------------------------------------------------------
+// Fixture access
+// ---------------------------------------------------------------------------
+
+fn fixture_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../protocol/agent/v1/fixtures/offline-enrollment")
+}
+
+fn sha256_hex(bytes: &[u8]) -> String {
+    Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+/// Read one fixture file and refuse it unless its bytes match the digest
+/// `MANIFEST.sha256` freezes.
+///
+/// Every vector in this suite arrives through here. A fixture edited on this
+/// side therefore fails the tests that depend on it instead of quietly
+/// redefining what conformance means, which is the failure mode a
+/// fixture-driven suite is otherwise blind to.
+fn read_fixture(name: &str) -> Vec<u8> {
+    let dir = fixture_dir();
+    let manifest = fs::read_to_string(dir.join("MANIFEST.sha256")).expect("read MANIFEST.sha256");
+
+    let expected = manifest
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .find_map(|line| {
+            let (digest, file) = line
+                .split_once("  ")
+                .unwrap_or_else(|| panic!("malformed manifest line: {line}"));
+            (file == name).then(|| digest.to_string())
+        })
+        .unwrap_or_else(|| panic!("{name} is not listed in MANIFEST.sha256"));
+
+    let bytes = fs::read(dir.join(name)).unwrap_or_else(|error| panic!("read {name}: {error}"));
+    assert_eq!(sha256_hex(&bytes), expected, "{name} does not match the digest MANIFEST.sha256 freezes");
+    bytes
+}
+
+fn fixture_json(name: &str) -> Value {
+    serde_json::from_slice(&read_fixture(name)).unwrap_or_else(|error| panic!("{name} parses: {error}"))
+}
+
+fn accept_vectors() -> Value {
+    fixture_json("accept-vectors.json")
+}
+
+fn reject_vectors() -> Value {
+    fixture_json("reject-vectors.json")
+}
+
+fn trust_model() -> Value {
+    fixture_json("trust-model.json")
+}
+
+fn vector_list(fixture: &Value) -> Vec<Value> {
+    fixture["vectors"].as_array().expect("fixture carries a vector list").clone()
+}
+
+fn field<'a>(value: &'a Value, key: &str) -> &'a str {
+    value[key]
+        .as_str()
+        .unwrap_or_else(|| panic!("expected a string at '{key}' in {value}"))
+}
+
+/// The octets an operator carries in on removable media.
+///
+/// The fixture's `document` object *is* the transmitted artifact: a padded
+/// base64 `bytes` field holding the raw signed octets, plus the detached
+/// signature over them. Only `bytes` is covered by the signature, so
+/// re-serialising the surrounding envelope here cannot change what a verifier
+/// checks.
+fn envelope(document: &Value) -> Vec<u8> {
+    serde_json::to_vec(document).expect("envelope serialises")
+}
+
+/// The raw octets the signature covers, exactly as transmitted.
+fn signed_octets(document: &Value) -> Vec<u8> {
+    BASE64_STANDARD
+        .decode(field(document, "bytes"))
+        .expect("document bytes are padded base64")
+}
+
+/// The parsed signed document. Parsing is a convenience for the assertions
+/// below; the implementation under test is required to verify before it parses.
+fn signed_document(document: &Value) -> Value {
+    serde_json::from_slice(&signed_octets(document)).expect("signed document parses")
+}
+
+fn unix(rfc3339: &str) -> i64 {
+    chrono::DateTime::parse_from_rfc3339(rfc3339)
+        .unwrap_or_else(|error| panic!("'{rfc3339}' is not RFC 3339: {error}"))
+        .timestamp()
+}
+
+fn hex_to_bytes(hex: &str) -> Vec<u8> {
+    (0..hex.len())
+        .step_by(2)
+        .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("valid hex"))
+        .collect()
+}
+
+/// Turn a fixture's unpadded-base64url SEC1 point into a usable verifying key.
+fn verifying_key(sec1_base64url: &str) -> p256::ecdsa::VerifyingKey {
+    let point = BASE64_URL_NO_PAD.decode(sec1_base64url).expect("public key is base64url");
+    assert_eq!(point.len(), 65, "the protocol freezes a 65 octet uncompressed SEC1 point");
+
+    let mut der = hex_to_bytes(SPKI_PREFIX_HEX);
+    der.extend_from_slice(&point);
+    <p256::ecdsa::VerifyingKey as p256::pkcs8::DecodePublicKey>::from_public_key_der(&der).expect("public key decodes")
+}
+
+fn published_key(role_or_name: &str) -> Value {
+    fixture_json("trust-chain.json")["keys"]
+        .as_array()
+        .expect("trust chain publishes keys")
+        .iter()
+        .find(|key| field(key, "name") == role_or_name)
+        .unwrap_or_else(|| panic!("trust-chain.json publishes no key named '{role_or_name}'"))
+        .clone()
+}
+
+/// `signatureInput = domainSeparationTag || 0x00 || the received octets`, the
+/// rule `trust-model.json` freezes under `domainSeparation`.
+fn signing_input(artifact_tag: &str, received: &[u8]) -> Vec<u8> {
+    let mut input = artifact_tag.as_bytes().to_vec();
+    input.push(0x00);
+    input.extend_from_slice(received);
+    input
+}
+
+fn domain_tag(artifact: &str) -> String {
+    let model = trust_model();
+    assert_eq!(
+        field(&model["domainSeparation"], "separatorByte"),
+        "0x00",
+        "the separator byte this suite encodes is the one the trust model freezes"
+    );
+    field(&model["domainSeparation"]["tags"], artifact).to_string()
+}
+
+/// Locate an accept vector by the name other vectors reference it by.
+fn accept_vector_named(name: &str) -> Value {
+    vector_list(&accept_vectors())
+        .into_iter()
+        .find(|vector| field(vector, "name") == name)
+        .unwrap_or_else(|| panic!("accept-vectors.json carries no vector named '{name}'"))
+}
+
+/// Verify the challenge a response vector answers, at that challenge's own
+/// evaluation time.
+fn answered_challenge(response_vector: &Value) -> (Value, VerifiedChallenge) {
+    let challenge_vector = accept_vector_named(field(response_vector, "answersChallenge"));
+    let now = unix(field(&challenge_vector, "evaluationTime"));
+    let verified = OfflineEnrollment::verify_challenge(&envelope(&challenge_vector["document"]), now)
+        .expect("the answered challenge is an accept vector and must verify");
+    (challenge_vector, verified)
+}
+
+fn device_nonce_of(document: &Value) -> [u8; 32] {
+    let raw = BASE64_URL_NO_PAD
+        .decode(field(&signed_document(document), "deviceNonce"))
+        .expect("deviceNonce is base64url");
+    raw.try_into().expect("replay.nonceLengthBytes freezes a 32 octet nonce")
+}
+
+// ---------------------------------------------------------------------------
+// Accept vectors
+// ---------------------------------------------------------------------------
+
+/// Every challenge accept vector must verify at its own evaluation time and
+/// expose exactly what the signed document says.
+///
+/// Two of these vectors sit on the skew boundary — 120 seconds before
+/// `issuedAt` and 300 seconds after `expiresAt` — so a verifier that compares
+/// against the raw window instead of the tolerated one fails here rather than
+/// in an air-gapped data centre. `challenge_proof` is pinned to the challenge's
+/// own detached signature value because that is what the response has to echo;
+/// deriving it from anything else would silently break the binding.
+#[test]
+fn every_challenge_accept_vector_verifies_and_exposes_the_signed_fields() {
+    let mut verified_count = 0usize;
+
+    for vector in vector_list(&accept_vectors()) {
+        if field(&vector, "artifact") != "challenge" {
+            continue;
+        }
+
+        let name = field(&vector, "name");
+        let document = &vector["document"];
+        let now = unix(field(&vector, "evaluationTime"));
+
+        let verified = OfflineEnrollment::verify_challenge(&envelope(document), now)
+            .unwrap_or_else(|error| panic!("accept vector '{name}' must verify: {}", error.reason()));
+
+        let signed = signed_document(document);
+        assert_eq!(verified.challenge_id, field(&signed, "challengeId"), "vector '{name}' challengeId");
+        assert_eq!(
+            verified.organization_name,
+            field(&signed, "organizationName"),
+            "vector '{name}' organizationName"
+        );
+        assert_eq!(verified.cluster_name, field(&signed, "clusterName"), "vector '{name}' clusterName");
+        assert_eq!(verified.nonce, field(&signed, "nonce"), "vector '{name}' nonce");
+        assert_eq!(verified.issued_at, field(&signed, "issuedAt"), "vector '{name}' issuedAt");
+        assert_eq!(verified.expires_at, field(&signed, "expiresAt"), "vector '{name}' expiresAt");
+        assert_eq!(verified.connect_key_id, field(&signed, "connectKeyId"), "vector '{name}' connectKeyId");
+        assert_eq!(
+            verified.challenge_proof,
+            field(&document["signature"], "value"),
+            "vector '{name}' must carry the challenge's own signature as the proof a response echoes"
+        );
+
+        verified_count += 1;
+    }
+
+    assert_eq!(
+        verified_count, 3,
+        "accept-vectors.json publishes three challenge vectors; a fourth is a protocol change"
+    );
+}
+
+/// Connect's own producer wrote the response accept vectors. Rebuilding them
+/// from the challenge they answer, with the device nonce and production time
+/// they used, must reproduce every field that does not depend on which device
+/// key signed — including the discarded-unknown-field vector, whose extra
+/// `telemetryHint` must not survive into anything this side produces.
+#[test]
+fn response_accept_vectors_are_reproduced_field_for_field_by_build_response() {
+    let key = DeviceIdentity::generate();
+    let mut reproduced = 0usize;
+
+    for vector in vector_list(&accept_vectors()) {
+        if field(&vector, "artifact") != "response" {
+            continue;
+        }
+
+        let name = field(&vector, "name");
+        let published = signed_document(&vector["document"]);
+        let (_, challenge) = answered_challenge(&vector);
+
+        let produced_at = unix(field(&published, "producedAt"));
+        let nonce = device_nonce_of(&vector["document"]);
+
+        let built_envelope: Value = serde_json::from_slice(
+            &OfflineEnrollment::build_response(&challenge, &key, &nonce, produced_at)
+                .unwrap_or_else(|error| panic!("vector '{name}' must be reproducible: {}", error.reason())),
+        )
+        .expect("the built response is JSON");
+        let built = signed_document(&built_envelope);
+
+        for shared in [
+            "formatVersion",
+            "protocolVersion",
+            "challengeId",
+            "organizationName",
+            "clusterName",
+            "challengeNonce",
+            "challengeProof",
+            "deviceNonce",
+        ] {
+            assert_eq!(
+                field(&built, shared),
+                field(&published, shared),
+                "vector '{name}' field {shared} must match the response Connect published"
+            );
+        }
+        assert_eq!(
+            unix(field(&built, "producedAt")),
+            produced_at,
+            "vector '{name}' producedAt must be the instant it was given"
+        );
+
+        // `versioning.additive` says an unknown optional field is discarded and
+        // never echoed back; a producer that copied the challenge or a previous
+        // response wholesale would carry it forward.
+        assert!(
+            built.get("telemetryHint").is_none(),
+            "vector '{name}' must not echo an unknown optional field"
+        );
+
+        reproduced += 1;
+    }
+
+    assert_eq!(
+        reproduced, 2,
+        "accept-vectors.json publishes two response vectors; a third is a protocol change"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Reject vectors
+// ---------------------------------------------------------------------------
+
+/// Every challenge reject vector must fail, and fail for the one reason
+/// `error-codes.json` freezes.
+///
+/// Asserting only that verification failed would pass for an implementation
+/// that rejects everything, and would let a tampered document be reported as an
+/// expiry — a rejection reason is what an operator acts on, so it is part of the
+/// contract rather than a diagnostic detail.
+#[test]
+fn every_challenge_reject_vector_fails_with_its_frozen_reason() {
+    let known_reasons: Vec<String> = fixture_json("error-codes.json")["reasons"]
+        .as_array()
+        .expect("error-codes.json carries reasons")
+        .iter()
+        .map(|entry| field(entry, "reason").to_string())
+        .collect();
+
+    let mut rejected = 0usize;
+
+    for vector in vector_list(&reject_vectors()) {
+        if field(&vector, "artifact") != "challenge" {
+            continue;
+        }
+
+        let name = field(&vector, "name");
+        let expected = field(&vector["expected"], "reason");
+        assert!(
+            known_reasons.iter().any(|reason| reason == expected),
+            "vector '{name}' names reason {expected}, which error-codes.json does not freeze"
+        );
+
+        let now = unix(field(&vector, "evaluationTime"));
+        let error = OfflineEnrollment::verify_challenge(&envelope(&vector["document"]), now)
+            .expect_err(&format!("reject vector '{name}' must not verify"));
+
+        assert_eq!(error.reason(), expected, "vector '{name}' must fail as {expected}");
+        rejected += 1;
+    }
+
+    assert_eq!(
+        rejected, 8,
+        "reject-vectors.json publishes eight challenge vectors; losing one silently narrows the suite"
+    );
+}
+
+/// The response reject vectors are artifacts Connect refuses. This side never
+/// verifies a response, so the device-side statement is the stronger one: given
+/// the challenge each vector answers, `build_response` must not be capable of
+/// emitting that artifact in the first place.
+///
+/// Each arm pins the specific field a compromised or careless producer would
+/// have to get wrong, so an implementation that copied values out of the wrong
+/// place — the response's own document, an operator-supplied argument, a
+/// previous exchange — fails here.
+#[test]
+fn response_reject_vectors_are_artifacts_build_response_cannot_emit() {
+    let key = DeviceIdentity::generate();
+    let mut covered = 0usize;
+
+    for vector in vector_list(&reject_vectors()) {
+        if field(&vector, "artifact") != "response" {
+            continue;
+        }
+
+        let name = field(&vector, "name");
+        let refused = signed_document(&vector["document"]);
+        let (_, challenge) = answered_challenge(&vector);
+        let produced_at = unix(field(&refused, "producedAt"));
+        let nonce = device_nonce_of(&vector["document"]);
+
+        let outcome = OfflineEnrollment::build_response(&challenge, &key, &nonce, produced_at);
+
+        match field(&vector["expected"], "reason") {
+            // `responseWindow` in trust-model.json: a device that emits a
+            // response outside the tolerated challenge window has produced an
+            // artifact Connect will refuse, so the refusal belongs here rather
+            // than at the far end of a courier run.
+            "CHALLENGE_EXPIRED" => {
+                let error = outcome.expect_err(&format!("vector '{name}': producing this response must be refused"));
+                assert_eq!(error.reason(), "CHALLENGE_EXPIRED", "vector '{name}' must refuse as CHALLENGE_EXPIRED");
+                covered += 1;
+                continue;
+            }
+            reason => {
+                let built_envelope: Value = serde_json::from_slice(
+                    &outcome.unwrap_or_else(|error| panic!("vector '{name}' baseline must build: {}", error.reason())),
+                )
+                .expect("the built response is JSON");
+                let built = signed_document(&built_envelope);
+
+                match reason {
+                    "ORGANIZATION_MISMATCH" => {
+                        assert_ne!(
+                            field(&refused, "organizationName"),
+                            challenge.organization_name,
+                            "vector '{name}' is only a mismatch if it names another organization"
+                        );
+                        assert_eq!(
+                            field(&built, "organizationName"),
+                            challenge.organization_name,
+                            "vector '{name}': the organization must come from the challenge, never from elsewhere"
+                        );
+                    }
+                    "CLUSTER_MISMATCH" => {
+                        assert_ne!(
+                            field(&refused, "clusterName"),
+                            challenge.cluster_name,
+                            "vector '{name}' is only a mismatch if it names another cluster"
+                        );
+                        assert_eq!(
+                            field(&built, "clusterName"),
+                            challenge.cluster_name,
+                            "vector '{name}': the cluster must come from the challenge, never from elsewhere"
+                        );
+                    }
+                    "CHALLENGE_PROOF_INVALID" => {
+                        // Two distinct vectors land here: a nonce the challenge
+                        // never carried, and a proof lifted from another
+                        // challenge. Both must be impossible to produce.
+                        assert_eq!(
+                            field(&built, "challengeNonce"),
+                            challenge.nonce,
+                            "vector '{name}': the echoed nonce must be the challenge's own"
+                        );
+                        assert_eq!(
+                            field(&built, "challengeProof"),
+                            challenge.challenge_proof,
+                            "vector '{name}': the proof must be the answered challenge's signature"
+                        );
+                        assert!(
+                            field(&refused, "challengeNonce") != challenge.nonce
+                                || field(&refused, "challengeProof") != challenge.challenge_proof,
+                            "vector '{name}' must differ from the challenge in nonce or proof to be rejectable"
+                        );
+                    }
+                    "DEVICE_PROOF_INVALID" => {
+                        // The refused vector presents one key and is signed by
+                        // another; hold the fixture to that claim, then require
+                        // the built response to be the opposite. Proof of
+                        // possession is the only thing that makes presenting a
+                        // key in an unauthenticated document safe.
+                        use p256::ecdsa::signature::Verifier as _;
+
+                        let presented = verifying_key(field(&refused, "devicePublicKey"));
+                        let raw = BASE64_URL_NO_PAD
+                            .decode(field(&vector["document"]["signature"], "value"))
+                            .expect("signature is base64url");
+                        let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
+                        assert!(
+                            presented
+                                .verify(
+                                    &signing_input(&domain_tag("enrollmentResponse"), &signed_octets(&vector["document"])),
+                                    &signature
+                                )
+                                .is_err(),
+                            "vector '{name}' is only a possession failure if it does not verify under the key it presents"
+                        );
+
+                        assert_response_proves_possession(&built_envelope, name);
+                    }
+                    "UNSUPPORTED_FORMAT" => {
+                        assert_ne!(
+                            field(&refused, "formatVersion"),
+                            field(&built, "formatVersion"),
+                            "vector '{name}' is only unsupported if it names another format version"
+                        );
+                        assert_eq!(
+                            field(&built, "formatVersion"),
+                            "rustfs.connect.offline.enrollmentResponse/1",
+                            "vector '{name}': the format version is frozen"
+                        );
+                    }
+                    "UNSUPPORTED_PROTOCOL" => {
+                        assert_ne!(
+                            field(&refused, "protocolVersion"),
+                            field(&built, "protocolVersion"),
+                            "vector '{name}' is only unsupported if it names another protocol major"
+                        );
+                        assert_eq!(field(&built, "protocolVersion"), "v1", "vector '{name}': the protocol major is frozen");
+                    }
+                    "ENROLLMENT_REPLAYED" => {
+                        // The vector claims to be a byte-identical replay of an
+                        // accepted response; hold it to that, because a replay
+                        // vector that is not byte identical proves nothing about
+                        // single use.
+                        let accepted = accept_vector_named("response binding the device public key and the challenge proof");
+                        assert_eq!(
+                            signed_octets(&vector["document"]),
+                            signed_octets(&accepted["document"]),
+                            "vector '{name}' must be the accepted response octet for octet"
+                        );
+                        assert_eq!(
+                            field(&vector["document"]["signature"], "value"),
+                            field(&accepted["document"]["signature"], "value"),
+                            "vector '{name}' must carry the accepted response's signature"
+                        );
+
+                        // A fresh device nonce is a different artifact, so a
+                        // second enrollment is never mistaken for a replay of
+                        // the first.
+                        let other = OfflineEnrollment::build_response(&challenge, &key, &[0x5a; 32], produced_at)
+                            .expect("a second response builds");
+                        assert_ne!(
+                            signed_octets(&built_envelope),
+                            signed_octets(&serde_json::from_slice::<Value>(&other).expect("JSON")),
+                            "vector '{name}': a different device nonce must yield a different artifact"
+                        );
+                    }
+                    other => panic!("vector '{name}' names an unhandled reason {other}; extend this test"),
+                }
+            }
+        }
+
+        covered += 1;
+    }
+
+    assert_eq!(
+        covered, 9,
+        "reject-vectors.json publishes nine response vectors; losing one silently narrows the suite"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Signature encoding
+// ---------------------------------------------------------------------------
+
+/// The high-S malleation is the rejection the whole encoding rule exists for.
+///
+/// `(r, n - s)` is a second valid signature over the same document under the
+/// same key. Every mainstream ECDSA library verifies it, so an implementation
+/// that hands the decoded octets straight to `p256` accepts a forged-looking
+/// duplicate of a genuine challenge — and because the 64 octets differ, that
+/// duplicate is a distinct artifact identity that slips past any deduplication
+/// keyed on the signature. This test proves the rejection came from the
+/// encoding rule and not from a failed verification: it first shows the
+/// malleated signature verifying mathematically, then requires
+/// `verify_challenge` to refuse it as SIGNATURE_NOT_CANONICAL.
+#[test]
+fn malleated_high_s_signature_is_refused_although_it_verifies_mathematically() {
+    use p256::ecdsa::signature::Verifier as _;
+
+    let model = trust_model();
+    let malleated = model["rejectedSignatureEncodings"]
+        .as_array()
+        .expect("trust-model.json publishes rejected encodings")
+        .iter()
+        .find(|entry| field(entry, "reason") == "SIGNATURE_NOT_CANONICAL")
+        .expect("trust-model.json publishes the high-S malleation")
+        .clone();
+    assert!(
+        malleated["acceptedByALenientVerifier"].as_bool() == Some(true),
+        "this vector is only interesting because a lenient verifier accepts it"
+    );
+
+    let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
+    let genuine_value = field(&vector["document"]["signature"], "value").to_string();
+    let malleated_value = field(&malleated, "value").to_string();
+    assert_ne!(genuine_value, malleated_value, "the malleation must be a different encoding");
+
+    let genuine = BASE64_URL_NO_PAD.decode(&genuine_value).expect("signature is base64url");
+    let raw = BASE64_URL_NO_PAD.decode(&malleated_value).expect("signature is base64url");
+    assert_eq!(raw.len(), 64, "the malleation is well formed at 64 octets");
+    assert_eq!(raw[..32], genuine[..32], "the malleation shares r with the genuine signature");
+    assert_ne!(raw[32..], genuine[32..], "the malleation replaces s with n - s");
+
+    // Step one: the malleated pair really does verify under the signing key, so
+    // a verifier cannot be excused for accepting it on mathematical grounds.
+    let signature = p256::ecdsa::Signature::from_slice(&raw).expect("the malleated signature parses");
+    assert!(signature.normalize_s().is_some(), "the malleated signature must be the high-S form");
+    let key = verifying_key(field(&published_key("signing"), "publicKey"));
+    let input = signing_input(&domain_tag("enrollmentChallenge"), &signed_octets(&vector["document"]));
+    key.verify(&input, &signature)
+        .expect("the malleated signature must verify mathematically, or this test proves nothing");
+
+    // Step two: the implementation must refuse it anyway, and say why.
+    let mut tampered = vector["document"].clone();
+    tampered["signature"]["value"] = Value::String(malleated_value);
+
+    let now = unix(field(&vector, "evaluationTime"));
+    let error = OfflineEnrollment::verify_challenge(&envelope(&tampered), now)
+        .expect_err("a high-S signature must be refused even though it verifies");
+    assert_eq!(
+        error.reason(),
+        "SIGNATURE_NOT_CANONICAL",
+        "a malleated signature is a canonicality failure, not a verification failure"
+    );
+}
+
+/// Every encoding `trust-model.json` names as rejected must fail with the
+/// reason it names — DER, padded base64url, truncation, and out-of-range
+/// scalars alongside the malleation. Three of the five are accepted by a
+/// lenient verifier, so a single blanket "signature did not verify" answer would
+/// be both wrong and undiagnosable.
+#[test]
+fn every_rejected_signature_encoding_fails_with_its_frozen_reason() {
+    let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
+    let now = unix(field(&vector, "evaluationTime"));
+    let model = trust_model();
+    let encodings = model["rejectedSignatureEncodings"]
+        .as_array()
+        .expect("trust-model.json publishes rejected encodings");
+
+    for entry in encodings {
+        let name = field(entry, "name");
+        let mut tampered = vector["document"].clone();
+        tampered["signature"]["value"] = Value::String(field(entry, "value").to_string());
+
+        let error: EnrollmentError = OfflineEnrollment::verify_challenge(&envelope(&tampered), now)
+            .err()
+            .unwrap_or_else(|| panic!("rejected encoding '{name}' must not verify"));
+
+        assert_eq!(error.reason(), field(entry, "reason"), "rejected encoding '{name}'");
+    }
+
+    assert_eq!(encodings.len(), 5, "trust-model.json freezes five rejected encodings");
+}
+
+// ---------------------------------------------------------------------------
+// Clock window
+// ---------------------------------------------------------------------------
+
+/// The tolerated window is `[issuedAt - 300, expiresAt + 300]`, inclusive at
+/// both ends. An air-gapped device has no synchronised clock, so an
+/// off-by-one here either strands a legitimate enrollment or widens the window
+/// a stolen challenge stays usable in. Both ends are checked at the exact bound
+/// and one second past it, and the reason distinguishes the two directions.
+#[test]
+fn challenge_is_accepted_at_the_exact_skew_bound_and_refused_one_second_past_it() {
+    let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
+    let document = envelope(&vector["document"]);
+    let signed = signed_document(&vector["document"]);
+
+    let issued_at = unix(field(&signed, "issuedAt"));
+    let expires_at = unix(field(&signed, "expiresAt"));
+
+    let earliest = issued_at - SKEW_TOLERANCE_SECONDS;
+    OfflineEnrollment::verify_challenge(&document, earliest).expect("the earliest tolerated instant is inside the window");
+    let error =
+        OfflineEnrollment::verify_challenge(&document, earliest - 1).expect_err("one second earlier is outside the window");
+    assert_eq!(error.reason(), "CHALLENGE_NOT_YET_VALID");
+
+    let latest = expires_at + SKEW_TOLERANCE_SECONDS;
+    OfflineEnrollment::verify_challenge(&document, latest).expect("the latest tolerated instant is inside the window");
+    let error = OfflineEnrollment::verify_challenge(&document, latest + 1).expect_err("one second later is outside the window");
+    assert_eq!(error.reason(), "CHALLENGE_EXPIRED");
+}
+
+// ---------------------------------------------------------------------------
+// Response production
+// ---------------------------------------------------------------------------
+
+/// Assert a built response proves possession of the key it presents: the
+/// fingerprint matches the presented key, and the detached signature is a
+/// canonical low-S ES256 signature that verifies under that key over the exact
+/// octets transmitted.
+fn assert_response_proves_possession(built_envelope: &Value, label: &str) {
+    use p256::ecdsa::signature::Verifier as _;
+
+    let raw = signed_octets(built_envelope);
+    let built = signed_document(built_envelope);
+    let signature_block = &built_envelope["signature"];
+
+    assert_eq!(field(signature_block, "algorithm"), "ES256", "{label}: the algorithm is frozen");
+
+    let value = field(signature_block, "value");
+    assert_eq!(value.len(), 86, "{label}: the transfer encoding is 86 unpadded base64url characters");
+    assert!(
+        value.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'),
+        "{label}: the signature must use the base64url alphabet with no padding"
+    );
+
+    let bytes = BASE64_URL_NO_PAD.decode(value).expect("signature is base64url");
+    assert_eq!(bytes.len(), 64, "{label}: the signature is a fixed-width r || s");
+    let signature = p256::ecdsa::Signature::from_slice(&bytes).expect("signature parses");
+    assert!(
+        signature.normalize_s().is_none(),
+        "{label}: this side must never emit the malleated high-S form it refuses to accept"
+    );
+
+    let presented = field(&built, "devicePublicKey");
+    let key = verifying_key(presented);
+    key.verify(&signing_input(&domain_tag("enrollmentResponse"), &raw), &signature)
+        .unwrap_or_else(|error| panic!("{label}: the response must verify under the key it presents: {error}"));
+
+    // `signature.keyIdAlgorithm`: the lowercase SHA-256 of the DER
+    // SubjectPublicKeyInfo, not of the bare point and not of the transfer
+    // encoding.
+    let mut spki = hex_to_bytes(SPKI_PREFIX_HEX);
+    spki.extend_from_slice(&BASE64_URL_NO_PAD.decode(presented).expect("public key is base64url"));
+    let fingerprint = sha256_hex(&spki);
+    assert_eq!(
+        field(&built, "deviceKeyId"),
+        fingerprint,
+        "{label}: deviceKeyId must be the fingerprint of the key the document presents"
+    );
+    assert_eq!(
+        field(signature_block, "keyId"),
+        fingerprint,
+        "{label}: the detached signature must name the same key"
+    );
+}
+
+/// A response is the only thing Connect will ever see from this device, so it
+/// has to carry the whole binding on its own: the challenge it answers, the
+/// proof that challenge was genuine, the key being enrolled, and possession of
+/// that key.
+#[test]
+fn built_response_binds_the_challenge_proof_and_proves_possession_of_the_device_key() {
+    let vector = accept_vector_named("response binding the device public key and the challenge proof");
+    let (challenge_vector, challenge) = answered_challenge(&vector);
+    let key = DeviceIdentity::generate();
+    let produced_at = unix(field(&signed_document(&vector["document"]), "producedAt"));
+
+    let bytes = OfflineEnrollment::build_response(&challenge, &key, &[0x11; 32], produced_at).expect("the response builds");
+    let built_envelope: Value = serde_json::from_slice(&bytes).expect("the response is JSON");
+    let built = signed_document(&built_envelope);
+
+    assert_response_proves_possession(&built_envelope, "built response");
+
+    // The proof is the challenge's own detached signature. A producer that
+    // echoed the nonce alone, or hashed something, would let a response be
+    // built from an unverified challenge.
+    assert_eq!(
+        field(&built, "challengeProof"),
+        field(&challenge_vector["document"]["signature"], "value"),
+        "the proof must be the signature of the challenge being answered"
+    );
+    assert_eq!(field(&built, "challengeNonce"), challenge.nonce);
+    assert_eq!(field(&built, "challengeId"), challenge.challenge_id);
+
+    assert_eq!(
+        field(&built, "devicePublicKey"),
+        BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]),
+        "the presented key must be the key that was passed in"
+    );
+    assert_eq!(
+        field(&built, "deviceNonce"),
+        BASE64_URL_NO_PAD.encode([0x11; 32]),
+        "the device nonce must be the one that was passed in"
+    );
+    assert!(field(&built, "producedAt").ends_with('Z'), "producedAt is a UTC RFC 3339 instant");
+}
+
+/// The response leaves the air gap on removable media and is read by anyone who
+/// handles it. A producer that serialised the key pair instead of the public
+/// key, or logged a debug rendering into the document, would put the enrolled
+/// private key on that medium — and the enrollment would still succeed, so
+/// nothing else in this suite would notice.
+#[test]
+fn built_response_carries_no_private_key_material() {
+    let vector = accept_vector_named("response binding the device public key and the challenge proof");
+    let (_, challenge) = answered_challenge(&vector);
+    let key = DeviceIdentity::generate();
+    let produced_at = unix(field(&signed_document(&vector["document"]), "producedAt"));
+
+    let response = OfflineEnrollment::build_response(&challenge, &key, &[0x22; 32], produced_at).expect("the response builds");
+
+    // The envelope carries the signed document base64-encoded, so a needle
+    // present in the document is not present in the envelope octets. Both
+    // layers are searched: an operator handling the medium can read either.
+    let envelope_value: Value = serde_json::from_slice(&response).expect("the response is JSON");
+    let mut haystack = response;
+    haystack.extend_from_slice(&signed_octets(&envelope_value));
+
+    let pkcs8 = key.to_pkcs8_der().expect("serialise the key");
+    let secret = <p256::SecretKey as p256::pkcs8::DecodePrivateKey>::from_pkcs8_der(&pkcs8).expect("the key parses");
+    let scalar = secret.to_bytes();
+
+    // Every spelling the scalar could plausibly reach a document in: raw, and
+    // the three encodings this protocol already uses elsewhere.
+    let scalar_hex: String = scalar.iter().map(|byte| format!("{byte:02x}")).collect();
+    for (description, needle) in [
+        ("the PKCS#8 encoding", pkcs8.to_vec()),
+        ("the raw private scalar", scalar.to_vec()),
+        ("the scalar in base64url", BASE64_URL_NO_PAD.encode(scalar).into_bytes()),
+        ("the scalar in standard base64", BASE64_STANDARD.encode(scalar).into_bytes()),
+        ("the scalar in hex", scalar_hex.into_bytes()),
+    ] {
+        assert!(
+            !haystack.windows(needle.len()).any(|window| window == needle.as_slice()),
+            "the response must not contain {description}"
+        );
+    }
+
+    // The public half must be there, so the absence above is a statement about
+    // what was excluded rather than about a haystack that would not have found
+    // the private half either.
+    let point = BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]);
+    assert!(
+        haystack.windows(point.len()).any(|window| window == point.as_bytes()),
+        "the response must still present the public key"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// The offline invariant
+// ---------------------------------------------------------------------------
+
+/// The whole surface exists because there is no network. This asserts that
+/// three different ways, because no single one of them is conclusive on its own.
+///
+/// 1. The process opens no descriptor across a full verify-and-respond cycle. A
+///    socket, a DNS resolver, a pooled HTTP client, or a revocation-list fetch
+///    all show up here — including one that is opened and cached rather than
+///    opened and closed, which is what a lazily built client does.
+/// 2. The cycle is a pure byte transform: the same inputs produce the same
+///    verified fields, and the evaluation instant is an argument rather than an
+///    ambient read, so nothing about the outcome can depend on reachability.
+/// 3. Repeating the cycle changes nothing observable, so a first call cannot be
+///    quietly initialising shared state that a later one reuses.
+#[cfg(unix)]
+#[test]
+fn enrollment_opens_no_descriptor_and_is_a_pure_byte_transform() {
+    let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root");
+    let document = envelope(&vector["document"]);
+    let now = unix(field(&vector, "evaluationTime"));
+    let key = DeviceIdentity::generate();
+
+    // Warm anything the test harness itself lazily opens before the baseline.
+    let _ = open_descriptors();
+    let baseline = open_descriptors();
+    assert!(
+        !baseline.is_empty(),
+        "the descriptor table must be readable for this test to mean anything"
+    );
+
+    let mut fields = Vec::new();
+    for _ in 0..2 {
+        let challenge = OfflineEnrollment::verify_challenge(&document, now).expect("the challenge verifies");
+        let response = OfflineEnrollment::build_response(&challenge, &key, &[0x33; 32], now).expect("the response builds");
+        fields.push((
+            challenge.challenge_id.clone(),
+            challenge.nonce.clone(),
+            challenge.challenge_proof.clone(),
+            signed_octets(&serde_json::from_slice::<Value>(&response).expect("JSON")),
+        ));
+    }
+
+    assert_eq!(
+        open_descriptors(),
+        baseline,
+        "the enrollment path must not open a descriptor: no socket, no resolver, no cached client"
+    );
+
+    let (first, second) = (&fields[0], &fields[1]);
+    assert_eq!(first.0, second.0, "verification must be deterministic");
+    assert_eq!(first.1, second.1, "verification must be deterministic");
+    assert_eq!(first.2, second.2, "verification must be deterministic");
+    assert_eq!(
+        first.3, second.3,
+        "the signed response octets are a function of the challenge, the key, the nonce, and the instant"
+    );
+}
+
+#[cfg(unix)]
+fn open_descriptors() -> Vec<String> {
+    // Linux publishes the table at /proc/self/fd; the BSDs and macOS at /dev/fd.
+    let path = if PathBuf::from("/proc/self/fd").is_dir() {
+        "/proc/self/fd"
+    } else {
+        "/dev/fd"
+    };
+
+    let mut entries: Vec<String> = fs::read_dir(path)
+        .unwrap_or_else(|error| panic!("read {path}: {error}"))
+        .map(|entry| entry.expect("read dir entry").file_name().to_string_lossy().into_owned())
+        .collect();
+    entries.sort();
+    entries
+}
+
+/// A descriptor count taken around a call cannot see a socket that was opened
+/// and closed inside it, so the invariant is also asserted where it can be
+/// stated absolutely: the implementation names no network API at all.
+///
+/// This is the shape the regression actually takes — someone adds a
+/// revocation-list fetch, a time-server check, or a "just confirm the challenge
+/// with Connect" call — and it is caught at the source rather than by observing
+/// its effects.
+#[test]
+fn enrollment_implementation_names_no_network_api() {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/connect/offline/enrollment.rs");
+    let source = fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display()));
+
+    // Prose is allowed to discuss the invariant it is documenting, so only code
+    // is scanned.
+    let code: String = source
+        .lines()
+        .filter(|line| !line.trim_start().starts_with("//"))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    for forbidden in [
+        "std::net",
+        "tokio::net",
+        "TcpStream",
+        "TcpListener",
+        "UdpSocket",
+        "UnixStream",
+        "ToSocketAddrs",
+        "reqwest",
+        "hyper",
+        "tonic",
+    ] {
+        assert!(
+            !code.contains(forbidden),
+            "offline enrollment must not reach the network, but the implementation names {forbidden}"
+        );
+    }
+}

rustfs/tests/inspect_cli.rs

@@ -25,7 +25,9 @@ fn decode_hex(source: &str) -> Vec<u8> {
         .collect::<String>();
     digits
         .as_bytes()
-        .chunks_exact(2)
+        .as_chunks::<2>()
+        .0
+        .iter()
         .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).expect("hex pair"), 16).expect("fixture hex byte"))
         .collect()
 }

scripts/prometheus_metrics_1649_smoke.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Read-only Prometheus smoke checks for backlog #1649 metric dimensions.
+
+The harness queries Prometheus' instant-query API. It never writes to RustFS,
+Prometheus, or the scrape targets. A check is ``metric|label=value,...``;
+``--require-labels`` accepts ``metric|label1,label2``. ``--retired`` checks
+that an exact label set is absent after the scheduler's retirement window.
+"""
+
+import argparse
+import base64
+import json
+import sys
+from dataclasses import dataclass
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode, urlparse
+from urllib.request import Request, urlopen
+
+
+@dataclass(frozen=True)
+class Series:
+    labels: dict[str, str]
+
+
+def query_url(value: str) -> str:
+    parsed = urlparse(value)
+    if parsed.path.rstrip("/").endswith("/api/v1/query"):
+        return value
+    return value.rstrip("/") + "/api/v1/query"
+
+
+def parse_spec(spec: str, separator: str = "|") -> tuple[str, dict[str, str]]:
+    metric, _, labels = spec.partition(separator)
+    if not metric or any(c in metric for c in "{} \t"):
+        raise ValueError(f"invalid metric check: {spec!r}")
+    expected: dict[str, str] = {}
+    if labels:
+        for pair in labels.split(","):
+            key, sep, value = pair.partition("=")
+            if not sep or not key or not value:
+                raise ValueError(f"invalid label selector in {spec!r}")
+            if key in expected:
+                raise ValueError(f"duplicate label {key!r} in {spec!r}")
+            expected[key] = value
+    return metric, expected
+
+
+def parse_label_names(spec: str) -> tuple[str, list[str]]:
+    metric, separator, labels = spec.partition("|")
+    names = [item for item in labels.split(",") if item] if separator else []
+    if not metric or any(c in metric for c in "{} \t") or not names or any(
+        "=" in item or not item.replace("_", "a").isalnum() for item in names
+    ):
+        raise ValueError(f"invalid label-name check: {spec!r}")
+    return metric, names
+
+
+def fetch_series(endpoint: str, metric: str, headers: dict[str, str], timeout: float) -> list[Series]:
+    query = f"{metric}{{}}" if "{" not in metric else metric
+    request = Request(f"{endpoint}?{urlencode({'query': query})}", headers=headers)
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            payload = json.load(response)
+    except (HTTPError, URLError, TimeoutError) as error:
+        raise RuntimeError(f"Prometheus query failed for {query!r}: {error}") from error
+    if payload.get("status") != "success":
+        raise RuntimeError(f"Prometheus returned non-success for {query!r}: {payload}")
+    data = payload.get("data", {})
+    if data.get("resultType") != "vector":
+        raise RuntimeError(f"Prometheus query did not return an instant vector: {query!r}")
+    return [Series(dict(item.get("metric", {}))) for item in data.get("result", [])]
+
+
+def has_labels(series: Iterable[Series], expected: dict[str, str]) -> bool:
+    return any(all(item.labels.get(key) == value for key, value in expected.items()) for item in series)
+
+
+def run(args: argparse.Namespace) -> int:
+    headers = {"Accept": "application/json"}
+    if args.bearer:
+        headers["Authorization"] = f"Bearer {args.bearer}"
+    if args.basic:
+        headers["Authorization"] = "Basic " + base64.b64encode(args.basic.encode()).decode()
+    endpoint = query_url(args.query_url)
+    required = list(args.require)
+    labels = list(args.require_labels)
+    retired = list(args.retired)
+    if args.profile == "backlog-1649":
+        required += [
+            "rustfs_system_drive_total_bytes",
+            "rustfs_system_drive_writes_total",
+            "rustfs_system_drive_deletes_total",
+            "rustfs_scanner_source_work_total",
+            "rustfs_scanner_active_bucket_drive_scans",
+            "rustfs_scanner_bucket_drive_result_total",
+            "rustfs_ilm_action_tasks",
+            "rustfs_ilm_tasks",
+            "rustfs_ilm_task_events_total",
+            "rustfs_ilm_queue_backpressure_total",
+            "rustfs_ilm_versions_scanned_by_server",
+            "rustfs_notification_current_send_in_progress_by_server",
+            "rustfs_notification_events_errors_total_by_server",
+            "rustfs_notification_events_sent_total_by_server",
+            "rustfs_notification_events_skipped_total_by_server",
+            "rustfs_audit_failed_messages_by_server",
+            "rustfs_audit_target_queue_length_by_server",
+            "rustfs_audit_total_messages_by_server",
+            "rustfs_notification_events_errors_total",
+            "rustfs_notification_events_sent_total",
+            "rustfs_notification_events_skipped_total",
+            "rustfs_audit_failed_messages",
+            "rustfs_audit_target_queue_length",
+            "rustfs_audit_total_messages",
+        ]
+        labels += [
+            "rustfs_system_drive_total_bytes|server,drive",
+            "rustfs_system_drive_writes_total|server,drive",
+            "rustfs_system_drive_deletes_total|server,drive",
+            "rustfs_scanner_source_work_total|server,source,state",
+            "rustfs_scanner_active_bucket_drive_scans|server,source,bucket,drive",
+            "rustfs_scanner_bucket_drive_result_total|server,bucket,drive,result",
+            "rustfs_ilm_action_tasks|server,action,state",
+            "rustfs_ilm_tasks|server,action,queue_state",
+            "rustfs_ilm_task_events_total|server,action,result",
+            "rustfs_ilm_queue_backpressure_total|server,action,reason",
+            "rustfs_ilm_versions_scanned_by_server|server,source",
+            "rustfs_notification_current_send_in_progress_by_server|server",
+            "rustfs_notification_events_errors_total_by_server|server",
+            "rustfs_notification_events_sent_total_by_server|server",
+            "rustfs_notification_events_skipped_total_by_server|server",
+            "rustfs_audit_failed_messages_by_server|server,target_id",
+            "rustfs_audit_target_queue_length_by_server|server,target_id",
+            "rustfs_audit_total_messages_by_server|server,target_id",
+            "rustfs_audit_failed_messages|target_id",
+            "rustfs_audit_target_queue_length|target_id",
+            "rustfs_audit_total_messages|target_id",
+        ]
+    if not required and not labels and not retired:
+        raise ValueError("provide --profile backlog-1649 or at least one check")
+    failures: list[str] = []
+    cache: dict[str, list[Series]] = {}
+
+    def get(metric: str) -> list[Series]:
+        if metric not in cache:
+            cache[metric] = fetch_series(endpoint, metric, headers, args.timeout)
+        return cache[metric]
+
+    def missing_servers(series: list[Series]) -> list[str]:
+        if not args.server or not any("server" in item.labels for item in series):
+            return []
+        observed = {item.labels["server"] for item in series if "server" in item.labels}
+        return sorted(set(args.server) - observed)
+
+    for spec in required:
+        metric, expected = parse_spec(spec)
+        series = get(metric)
+        if not series:
+            failures.append(f"{metric}: no series returned")
+        elif expected and not has_labels(series, expected):
+            failures.append(f"{metric}: no series has labels {expected}; observed {len(series)} series")
+        elif missing_servers(series):
+            failures.append(f"{metric}: missing requested server series {missing_servers(series)}")
+    for spec in labels:
+        metric, required_labels = parse_label_names(spec)
+        series = get(metric)
+        if not series:
+            failures.append(f"{metric}: aggregate series absent")
+        elif any(not all(label in item.labels for label in required_labels) for item in series):
+            failures.append(f"{metric}: at least one series is missing labels {required_labels}")
+        elif missing_servers(series):
+            failures.append(f"{metric}: missing requested server series {missing_servers(series)}")
+    for spec in retired:
+        metric, expected = parse_spec(spec)
+        if has_labels(get(metric), expected):
+            failures.append(f"{metric}: retired series still present with labels {expected}")
+    if failures:
+        for failure in failures:
+            print(f"FAIL: {failure}", file=sys.stderr)
+        return 1
+    print(f"PASS: {len(required)} required, {len(labels)} label, {len(retired)} retirement checks")
+    return 0
+
+
+def self_test() -> None:
+    assert parse_spec("metric|server=node1,drive=d1") == ("metric", {"server": "node1", "drive": "d1"})
+    assert parse_label_names("metric|server,drive") == ("metric", ["server", "drive"])
+    assert has_labels([Series({"server": "node1", "drive": "d1"})], {"server": "node1"})
+    assert not has_labels([Series({"server": "node1"})], {"server": "node2"})
+    try:
+        parse_spec("metric|server")
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("malformed value selector accepted")
+    print("PASS: self-test")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--query-url", help="Prometheus base URL or /api/v1/query endpoint")
+    parser.add_argument("--profile", choices=["backlog-1649"])
+    parser.add_argument("--server", action="append", default=[], help="server label value required in every server-scoped check")
+    parser.add_argument("--require", action="append", default=[], metavar="METRIC|k=v,...")
+    parser.add_argument("--require-labels", action="append", default=[], metavar="METRIC|k1,k2")
+    parser.add_argument("--retired", action="append", default=[], metavar="METRIC|k=v,...")
+    parser.add_argument("--bearer")
+    parser.add_argument("--basic", help="username:password; prefer --bearer in shared shells")
+    parser.add_argument("--timeout", type=float, default=10.0)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+    if args.self_test:
+        self_test()
+        return 0
+    if not args.query_url:
+        parser.error("--query-url is required unless --self-test is used")
+    try:
+        return run(args)
+    except (RuntimeError, ValueError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())