fix(scanner): defer usage publication during pool recovery (#6333)

* fix(scanner): defer usage publication during pool recovery

* fix(scanner): preserve metrics when publication is deferred

* fix(scanner): route test types through storage boundary

* fix(scanner): keep cache floor deferred during movement

.github/workflows/package.yml

@@ -189,6 +189,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - arch: x86_64
@@ -510,15 +511,13 @@ jobs:
 
           CHECKSUM_DIR="$(mktemp -d)"
           gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
-            -D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
+            -D "$CHECKSUM_DIR" --clobber
 
           for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
             asset="${spec%%:*}"
             checksum_cmd="${spec##*:}"
             checksum_file="${CHECKSUM_DIR}/${asset}"
 
-            touch "$checksum_file"
-
             for f in "$DEB_FILE" "$RPM_FILE"; do
               if [[ -n "$f" && -f "$f" ]]; then
                 base="$(basename "$f")"
@@ -531,7 +530,8 @@ jobs:
                 grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
                 grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
                 mv "${checksum_file}.tmp2" "$checksum_file"
-                (cd "$(dirname "$f")" && "$checksum_cmd" -- "$github_base") >> "$checksum_file"
+                digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
+                printf '%s  %s\n' "$digest" "$github_base" >> "$checksum_file"
               fi
             done
 

AGENTS.md

@@ -127,8 +127,9 @@ the broadest gate. Inspect only the final task-owned diff, classify it by
 behavioral impact rather than line count or path alone, and run the smallest
 set of checks that provides meaningful coverage. Do not let unrelated
 worktree changes or a generic contributor checklist expand the scope.
-Non-exempt changes must also pass Adversarial Validation (next section) before
+For non-exempt changes, complete the applicable multi-role adversarial review
-the checks below count as completion.
+before running `make pre-pr` (or an equivalent full gate). Resolve or rebut
+every finding first, then run the gate against the reviewed final diff.
 
 ### Validation floor
 
@@ -166,8 +167,9 @@ the checks below count as completion.
    dependency set is identifiable, validate those packages and known
    dependents instead of the whole workspace. Use `make pre-commit` only when
    a repository-wide fast gate adds useful confidence beyond those checks.
-4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage
+4. **Broad or high-risk change:** After the applicable adversarial review has
-   cannot bound the impact, including:
+   completed, run `make pre-pr` only when targeted coverage cannot bound the
+   impact, including:
    - dependency, feature, build-script, procedural-macro, code-generation,
      toolchain, or CI changes that alter compilation or the test matrix;
    - cross-crate public APIs, shared foundational code, or broad refactors with
@@ -287,8 +289,9 @@ High risk: all seven roles.
 - Every testable behavior change has a focused regression check. Exceptions
   follow the validation floor and state why a check is impractical and what
   risk remains.
-- The Verification Before PR gates pass — adversarial review supplements
+- After the applicable adversarial review has completed, the Verification
-  those gates, never replaces them.
+  Before PR gates pass; adversarial review supplements those gates, never
+  replaces them.
 - High risk only: record a one-line verdict per role in the PR description.
 
 ## Git and PR Baseline

CONTRIBUTING.md

@@ -91,8 +91,9 @@ A green `make pre-commit` is not enough to open a pull request.
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
-tests). Run `make pre-pr` before opening or updating a pull request — this is
+tests). Complete the applicable multi-role adversarial review described in
-what CI enforces.
+`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
+updating a pull request. This is what CI enforces.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -150,8 +151,9 @@ Example output when formatting fails:
 2. **Format your code**: `make fmt` or `cargo fmt --all`
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
-5. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
-6. **Push to your branch**: `git push`
+6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration
 

Cargo.lock

@@ -4757,9 +4757,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.17"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f877e75f39e9827ec50a572dd592684ac28c029578726c85f1b2aa6ab807449"
+checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228"
 dependencies = [
  "atomic-waker",
  "bytes",

crates/common/src/metrics.rs

@@ -729,7 +729,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
         return 0;
     }
 
-    u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
+    u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
 }
 
 #[derive(Clone, Copy, Debug, Default)]
@@ -781,6 +781,19 @@ struct ScannerBucketDriveResultValue {
     last_seen: u64,
 }
 
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+struct ScannerActiveBucketDriveKey {
+    source: String,
+    bucket: String,
+    drive: String,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct ScannerActiveBucketDriveValue {
+    count: u64,
+    started_at: Timestamp,
+}
+
 // ---------------------------------------------------------------------------
 // Metrics
 // ---------------------------------------------------------------------------
@@ -813,6 +826,7 @@ pub struct Metrics {
     scanner_set_scans_active: AtomicU64,
     scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
     scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
+    scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
     scanner_bucket_drive_result_clock: AtomicU64,
     current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
     last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
@@ -1045,6 +1059,15 @@ pub struct ScannerBucketDriveResultSnapshot {
     pub count: u64,
 }
 
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScannerActiveBucketDriveSnapshot {
+    pub source: String,
+    pub bucket: String,
+    pub drive: String,
+    pub count: u64,
+    pub age_seconds: u64,
+}
+
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct ScannerReplicationRepairSnapshot {
     pub source: String,
@@ -1387,6 +1410,8 @@ pub struct ScannerRuntimeDetailsReport {
     pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
     #[serde(default)]
     pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
+    #[serde(default)]
+    pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
 }
 
 impl CurrentCycle {
@@ -1746,7 +1771,7 @@ pub fn emit_scan_cycle_deferred(duration: Duration) {
     metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
 }
 
-pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
+pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
     let result = if success { "success" } else { "error" };
     global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
     metrics::counter!(
@@ -1764,7 +1789,7 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
     .record(duration.as_secs_f64());
 }
 
-pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
+pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
     global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
     metrics::counter!(
         OTEL_SCANNER_BUCKETS_SCANNED,
@@ -1817,6 +1842,7 @@ impl Metrics {
             scanner_set_scans_active: AtomicU64::new(0),
             scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
             scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
+            scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
             scanner_bucket_drive_result_clock: AtomicU64::new(0),
             current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
             last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
@@ -2308,8 +2334,45 @@ impl Metrics {
         }
     }
 
-    pub fn record_scan_bucket_drive_start(&self) {
+    pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
         self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
+        if bucket.is_empty() || drive.is_empty() {
+            return;
+        }
+        let key = ScannerActiveBucketDriveKey {
+            source: source.as_str().to_string(),
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        };
+        let mut active = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        active
+            .entry(key)
+            .and_modify(|value| value.count = value.count.saturating_add(1))
+            .or_insert(ScannerActiveBucketDriveValue {
+                count: 1,
+                started_at: Timestamp::now(),
+            });
+    }
+
+    pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
+        let key = ScannerActiveBucketDriveKey {
+            source: source.as_str().to_string(),
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        };
+        let mut active = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(value) = active.get_mut(&key) {
+            value.count = value.count.saturating_sub(1);
+            if value.count == 0 {
+                active.remove(&key);
+            }
+        }
     }
 
     pub fn record_scan_bucket_drive_failure(&self) {
@@ -2782,6 +2845,26 @@ impl Metrics {
         } else {
             Vec::new()
         };
+        let now = Timestamp::now();
+        let mut active_bucket_drive_scans = self
+            .scanner_active_bucket_drive_scans
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .iter()
+            .map(|(key, value)| ScannerActiveBucketDriveSnapshot {
+                source: key.source.clone(),
+                bucket: key.bucket.clone(),
+                drive: key.drive.clone(),
+                count: value.count,
+                age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
+            })
+            .collect::<Vec<_>>();
+        active_bucket_drive_scans.sort_by(|left, right| {
+            left.source
+                .cmp(&right.source)
+                .then_with(|| left.bucket.cmp(&right.bucket))
+                .then_with(|| left.drive.cmp(&right.drive))
+        });
         ScannerRuntimeDetailsReport {
             disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
             bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
@@ -2791,6 +2874,7 @@ impl Metrics {
                 .lock()
                 .unwrap_or_else(|poisoned| poisoned.into_inner())
                 .clone(),
+            active_bucket_drive_scans,
         }
     }
 
@@ -4371,7 +4455,7 @@ mod tests {
     #[tokio::test]
     async fn report_includes_bucket_drive_scan_starts() {
         let metrics = Metrics::new();
-        metrics.record_scan_bucket_drive_start();
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
         metrics.record_scan_bucket_drive_failure();
 
         let report = metrics.report().await;
@@ -4380,6 +4464,27 @@ mod tests {
         assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
     }
 
+    #[tokio::test]
+    async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
+        let metrics = Metrics::new();
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
+        assert_eq!(active.len(), 1);
+        assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
+        assert_eq!(active[0].bucket, "bucket-a");
+        assert_eq!(active[0].drive, "/mnt/data/1");
+        assert_eq!(active[0].count, 2);
+
+        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
+        metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
+        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
+
+        metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
+        assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
+    }
+
     #[tokio::test]
     async fn report_includes_structured_bucket_drive_results() {
         let metrics = Metrics::new();

crates/config/README.md

@@ -115,6 +115,15 @@ Current guidance:
   - enables KMS readiness enforcement for `/health/ready`.
   - default is `false`.
 
+## Object lock admission environment variables
+
+- `RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS`
+  - experimental same-object PUT commit namespace-lock admission budget.
+  - default is `0`, which disables this override and keeps `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` behavior.
+  - when set, only `put_object_commit` write-lock acquisition is bounded by this millisecond budget; other namespace lock users keep the global object-lock timeout.
+  - timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
+  - this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
+
 ## Drive timeout environment variables
 
 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`

crates/config/src/constants/object.rs

@@ -427,6 +427,19 @@ pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TI
 /// Default lock acquisition timeout: 5 seconds.
 pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
 
+/// Environment variable for the experimental PUT commit namespace lock acquire timeout in milliseconds.
+///
+/// A value of `0` disables the experiment and keeps
+/// `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` as the timeout. This only bounds the
+/// `put_object_commit` namespace write-lock wait and is intended for #925
+/// tail-drain admission experiments.
+///
+/// Default: 0 milliseconds (disabled).
+pub const ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: &str = "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS";
+
+/// Default: PUT commit namespace lock acquire timeout override is disabled.
+pub const DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: u64 = 0;
+
 /// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
 ///
 /// This timeout bounds the internode RPC call itself. It is intentionally

crates/e2e_test/src/common.rs

@@ -53,7 +53,8 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
 const RUSTFS_FULL_FEATURE: &str = "full";
 const TEST_PORT_MIN: u16 = 20_000;
-const TEST_PORT_RANGE: u16 = 40_000;
+// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
+const TEST_PORT_RANGE: u16 = 10_000;
 const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
 const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
 const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -1028,20 +1028,6 @@ impl<'a> ReaderPathExpectation<'a> {
         }
     }
 
-    fn with_size_bucket(
-        object: ReaderObject<'a>,
-        expected_path: &'a str,
-        object_class: &'a str,
-        expected_size_bucket: &'a str,
-    ) -> Self {
-        Self {
-            object,
-            expected_path,
-            object_class,
-            expected_size_bucket: Some(expected_size_bucket),
-        }
-    }
-
     fn with_any_size_bucket(object: ReaderObject<'a>, expected_path: &'a str, object_class: &'a str) -> Self {
         Self {
             object,
@@ -1909,12 +1895,7 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
     assert_reader_path(
         &collector,
         &client,
-        ReaderPathExpectation::with_size_bucket(
+        ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
-            ReaderObject::new(bucket, key, &body, put.e_tag(), None),
-            LEGACY_DUPLEX,
-            COMPRESSED,
-            size_bucket(4 * KIB),
-        ),
     )
     .await?;
 
@@ -2274,6 +2255,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
     hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
     hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
     hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1");
+    hot.set_env("RUSTFS_TRANSITION_QUEUE_SEND_TIMEOUT_MS", "1");
     hot.start().await?;
 
     let hot_client = hot.create_s3_client(0)?;
@@ -2290,7 +2272,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
             .put_object()
             .bucket(&bucket)
             .key(key)
-            .body(ByteStream::from(payload(64 * KIB, index)))
+            .body(ByteStream::from(payload(1024 * KIB, index)))
             .send()
             .await?;
     }

crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs

@@ -39,6 +39,7 @@ use std::time::Duration;
 use tracing::info;
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+type S3OperationResult<T> = Result<T, Box<aws_sdk_s3::Error>>;
 
 const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
 const OTHER_KEY: &str = "kms-matrix-other-key";
@@ -130,7 +131,7 @@ fn policy_document(statements: Vec<serde_json::Value>) -> String {
     serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
 }
 
-async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
+async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> S3OperationResult<()> {
     client
         .put_object()
         .bucket(BUCKET)
@@ -141,16 +142,23 @@ async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(),
         .send()
         .await
         .map(|_| ())
-        .map_err(aws_sdk_s3::Error::from)
+        .map_err(|error| Box::new(aws_sdk_s3::Error::from(error)))
 }
 
 /// Assert the operation failed with `AccessDenied` rather than any other error.
 ///
 /// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
 /// would hide both a leak of key state and an outage masquerading as a denial.
-fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
+fn assert_access_denied<T: std::fmt::Debug, E: std::fmt::Debug + std::borrow::Borrow<aws_sdk_s3::Error>>(
+    result: Result<T, E>,
+    what: &str,
+) {
     let error = result.expect_err(&format!("{what} must be denied"));
-    assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
+    assert_eq!(
+        error.borrow().code(),
+        Some("AccessDenied"),
+        "{what} must fail with AccessDenied: {error:?}"
+    );
 }
 
 /// Retry an SSE-KMS write until the identity's policy has reached the request path.
@@ -296,7 +304,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
             .send()
             .await
             .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
+            .map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
         "SSE-KMS read by an identity holding no kms grant",
     );
 
@@ -310,7 +318,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
             .send()
             .await
             .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
+            .map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
         "SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
     );
 

crates/e2e_test/src/protocols/webdav_core.rs

@@ -233,6 +233,111 @@ pub async fn test_webdav_core_operations() -> Result<()> {
         );
         info!("PASS: PUT file '{}' successful", filename);
 
+        // Regression for #6260: a bucket-scoped policy must be able to discover its bucket at the
+        // WebDAV root without the unrelated global ListAllMyBuckets permission.
+        let scoped_bucket = "webdav-scoped-bucket";
+        let scoped_file = "visible.txt";
+        let scoped_user = "webdav-scoped-user";
+        let scoped_secret = "webdav-scoped-secret";
+        let scoped_policy_name = "webdav-scoped-policy";
+
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"MKCOL").unwrap(), format!("{}/{}", base_url, scoped_bucket))
+            .header("Authorization", &auth_header)
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 201, "scoped test bucket should be created");
+
+        let resp = client
+            .put(format!("{}/{}/{}", base_url, scoped_bucket, scoped_file))
+            .header("Authorization", &auth_header)
+            .body("visible to the scoped principal")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 201, "scoped test object should be created");
+
+        admin_create_user(&admin_base_url, scoped_user, scoped_secret).await?;
+        admin_add_canned_policy(
+            &admin_base_url,
+            scoped_policy_name,
+            &serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ]
+                    },
+                    {
+                        "Effect": "Deny",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ],
+                        "Condition": { "Bool": { "aws:SecureTransport": "true" } }
+                    },
+                    {
+                        "Effect": "Deny",
+                        "Action": ["s3:*"],
+                        "Resource": [
+                            format!("arn:aws:s3:::{}", scoped_bucket),
+                            format!("arn:aws:s3:::{}/*", scoped_bucket)
+                        ],
+                        "Condition": { "StringEquals": { "s3:signatureversion": "AWS4-HMAC-SHA256" } }
+                    }
+                ]
+            }),
+        )
+        .await?;
+        admin_attach_policy_to_user(&admin_base_url, scoped_policy_name, scoped_user).await?;
+
+        let scoped_auth = basic_auth_header_for(scoped_user, scoped_secret);
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", &scoped_auth)
+            .header("Depth", "1")
+            .header("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 207, "bucket-scoped root PROPFIND should succeed");
+        let root_listing = resp.text().await?;
+        assert!(root_listing.contains(scoped_bucket), "the authorized bucket should be listed");
+        assert!(!root_listing.contains(bucket_name), "an unauthorized bucket must not be listed");
+
+        let resp = client
+            .request(
+                reqwest::Method::from_bytes(b"PROPFIND").unwrap(),
+                format!("{}/{}", base_url, scoped_bucket),
+            )
+            .header("Authorization", &scoped_auth)
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert_eq!(resp.status().as_u16(), 207, "authorized bucket PROPFIND should succeed");
+        assert!(resp.text().await?.contains(scoped_file), "the authorized object should be listed");
+
+        let denied_user = "webdav-no-buckets-user";
+        let denied_secret = "webdav-no-buckets-secret";
+        admin_create_user(&admin_base_url, denied_user, denied_secret).await?;
+        let resp = client
+            .request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
+            .header("Authorization", basic_auth_header_for(denied_user, denied_secret))
+            .header("Depth", "1")
+            .send()
+            .await?;
+        assert_eq!(
+            resp.status().as_u16(),
+            207,
+            "PROPFIND keeps the root resource visible when the directory listing is forbidden"
+        );
+        let denied_body = resp.text().await?;
+        assert!(!denied_body.contains(scoped_bucket), "a denied response must not leak the scoped bucket");
+        assert!(!denied_body.contains(bucket_name), "a denied response must not leak the admin bucket");
+
         // Test GET (download file)
         info!("Testing WebDAV: GET (download file '{}')", filename);
         let resp = client

crates/e2e_test/src/quota_test.rs

@@ -169,6 +169,42 @@ impl QuotaTestEnv {
         bucket: &str,
         quota_bytes: u64,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.wait_for_quota_usage_for(bucket).await?;
+
+        let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
+        let quota_config = serde_json::json!({
+            "quota": quota_bytes,
+            "quota_type": "HARD"
+        })
+        .to_string();
+        let readiness = async {
+            loop {
+                let (status, response) = admin_request(
+                    &self.env.url,
+                    Method::PUT,
+                    &quota_path,
+                    Some(quota_config.clone()),
+                    &self.env.access_key,
+                    &self.env.secret_key,
+                )
+                .await?;
+                if status.is_success() {
+                    return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
+                }
+                if status != StatusCode::SERVICE_UNAVAILABLE {
+                    return Err(format!("failed to set quota for {bucket}: {status} {response}").into());
+                }
+
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        match timeout(Duration::from_secs(30), readiness).await {
+            Ok(result) => result,
+            Err(_) => Err(format!("quota readiness did not converge for {bucket} within 30 seconds").into()),
+        }
+    }
+
+    pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
         let readiness = async {
             loop {
@@ -181,28 +217,12 @@ impl QuotaTestEnv {
                 if status != StatusCode::SERVICE_UNAVAILABLE {
                     return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
                 }
-
                 sleep(Duration::from_secs(1)).await;
             }
         };
         match timeout(Duration::from_secs(30), readiness).await {
-            Ok(result) => result?,
+            Ok(result) => result,
-            Err(_) => {
+            Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
-                return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
-            }
-        }
-
-        let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
-        let quota_config = serde_json::json!({
-            "quota": quota_bytes,
-            "quota_type": "HARD"
-        });
-
-        let response = awscurl_put(&url, &quota_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
-        if response.contains("error") {
-            Err(format!("Failed to set quota: {}", response).into())
-        } else {
-            Ok(())
         }
     }
 
@@ -614,6 +634,7 @@ mod integration_tests {
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
+        env.wait_for_quota_usage_for(&env.bucket_name).await?;
 
         // Test 1: GET quota for bucket without quota config
         let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
@@ -621,12 +642,7 @@ mod integration_tests {
         assert!(response.contains("quota") && response.contains("null"));
 
         // Test 2: PUT quota - valid config
-        let quota_config = serde_json::json!({
+        env.set_bucket_quota(1048576).await?;
-            "quota": 1048576,
-            "quota_type": "HARD"
-        });
-        let response = awscurl_put(&url, &quota_config.to_string(), &env.env.access_key, &env.env.secret_key).await?;
-        assert!(response.contains("success") || !response.contains("error"));
 
         // Test 3: GET quota after setting
         let response = awscurl_get(&url, &env.env.access_key, &env.env.secret_key).await?;

crates/e2e_test/src/reliant/tiering.rs

@@ -110,6 +110,7 @@ const USER_META_KEY: &str = "ilm7-origin";
 const USER_META_VAL: &str = "hermetic-transition";
 const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
 const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
+const TIER_MUTATION_RECOVERY_CHANGED: &str = "Remote tier mutation recovery changed before publish";
 
 /// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
 /// internal part boundary sits at this offset.
@@ -183,6 +184,17 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
     })
     .to_string();
 
+    let verify_path = format!("/rustfs/admin/v3/tier/{TIER_NAME}");
+    let deadline = Instant::now() + StdDuration::from_secs(30);
+    let mut recovery_changed = false;
+    loop {
+        if recovery_changed {
+            let (status, _) =
+                signed_admin_request(&hot.url, Method::GET, &verify_path, None, &hot.access_key, &hot.secret_key).await?;
+            if status.is_success() {
+                return Ok(());
+            }
+        }
         let (status, resp) = signed_admin_request(
             &hot.url,
             Method::PUT,
@@ -192,10 +204,19 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
             &hot.secret_key,
         )
         .await?;
-    if !status.is_success() {
+        if status.is_success() {
+            return Ok(());
+        }
+        if resp.contains(TIER_MUTATION_RECOVERY_CHANGED) {
+            recovery_changed = true;
+        } else if !recovery_changed || !resp.contains("TierNameAlreadyExist") {
             return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
         }
-    Ok(())
+        if Instant::now() >= deadline {
+            return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
+        }
+        tokio::time::sleep(StdDuration::from_millis(100)).await;
+    }
 }
 
 async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
@@ -207,10 +228,12 @@ async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
         if status.is_success() {
             return Ok(());
         }
-        if !resp.contains("TierNameBackendInUse") || Instant::now() >= deadline {
+        if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
+            || Instant::now() >= deadline
+        {
             return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
         }
-        // AddTier cleanup is asynchronous; wait until its committed mutation fence clears.
+        // Tier mutation cleanup and startup recovery are asynchronous.
         tokio::time::sleep(StdDuration::from_millis(100)).await;
     }
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -90,6 +90,12 @@ use uuid::Uuid;
 const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
 const REDACTED_CREDENTIAL: &str = "<redacted>";
 
+pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
+pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
+pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
+pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
+pub type DeleteObjectTaggingSdkError = Box<SdkError<DeleteObjectTaggingError>>;
+
 pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
 
 fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
@@ -1968,7 +1974,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    ) -> Result<HeadObjectOutput, HeadObjectSdkError> {
         // Announce the replication check so a RustFS target returns SSE-C
         // object metadata (etag/size) without the customer key the replication
         // worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
@@ -1981,8 +1987,7 @@ impl TargetClient {
         // object with an identical ETag, and the worker concludes the object
         // already converged — so it never actually replicates it.
         insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
-        match self
+        self.client
-            .client
             .head_object()
             .bucket(bucket)
             .key(object)
@@ -1999,10 +2004,7 @@ impl TargetClient {
             })
             .send()
             .await
-        {
+            .map_err(Box::new)
-            Ok(res) => Ok(res),
-            Err(e) => Err(e),
-        }
     }
 
     /// HEAD used by the read-proxy path (GET/HEAD of an object not yet
@@ -2023,7 +2025,7 @@ impl TargetClient {
         range: Option<String>,
         part_number: Option<i32>,
         extra_headers: HeaderMap,
-    ) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    ) -> Result<HeadObjectOutput, HeadObjectSdkError> {
         let headers = proxy_outbound_headers(extra_headers);
         self.client
             .head_object()
@@ -2036,6 +2038,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
@@ -2051,7 +2054,7 @@ impl TargetClient {
         range: Option<String>,
         part_number: Option<i32>,
         extra_headers: HeaderMap,
-    ) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
+    ) -> Result<GetObjectOutput, GetObjectSdkError> {
         let headers = proxy_outbound_headers(extra_headers);
         self.client
             .get_object()
@@ -2064,6 +2067,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// GetObjectTagging for the tagging read-proxy path
@@ -2073,7 +2077,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
+    ) -> Result<GetObjectTaggingOutput, GetObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .get_object_tagging()
@@ -2084,6 +2088,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// PutObjectTagging for the tagging proxy path
@@ -2094,7 +2099,7 @@ impl TargetClient {
         object: &str,
         version_id: Option<String>,
         tagging: SdkTagging,
-    ) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
+    ) -> Result<PutObjectTaggingOutput, PutObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .put_object_tagging()
@@ -2106,6 +2111,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// DeleteObjectTagging for the tagging proxy path
@@ -2115,7 +2121,7 @@ impl TargetClient {
         bucket: &str,
         object: &str,
         version_id: Option<String>,
-    ) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
+    ) -> Result<DeleteObjectTaggingOutput, DeleteObjectTaggingSdkError> {
         let headers = proxy_outbound_headers(HeaderMap::new());
         self.client
             .delete_object_tagging()
@@ -2126,6 +2132,7 @@ impl TargetClient {
             .map_request(move |req| apply_extra_headers(req, &headers))
             .send()
             .await
+            .map_err(Box::new)
     }
 
     /// On success returns the version id the target assigned (from

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -2180,7 +2180,7 @@ pub async fn recover_manual_transition_jobs_once(
     if limit == 0 {
         return Err(Error::other("manual transition job recovery limit must be greater than zero"));
     }
-    let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
+    let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
     let page = api
         .clone()
         .list_objects_v2(
@@ -2386,7 +2386,7 @@ async fn replay_manual_transition_pending_tasks(
             version_id: task.version_id,
             etag: task.etag,
             mod_time,
-            size: task.size.map_or(0, |size| size),
+            size: task.size.unwrap_or(0),
             is_latest: task.is_latest.unwrap_or(false),
             ..Default::default()
         };

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -1016,7 +1016,7 @@ pub async fn recover_transition_transaction_records(
         return Err(Error::other("transition transaction recovery limit must be greater than zero"));
     }
 
-    let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
+    let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
     let list = api
         .clone()
         .list_objects_v2(

crates/ecstore/src/bucket/migration.rs

@@ -41,6 +41,7 @@ const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
 const IAM_USERS_PREFIX: &str = "config/iam/users/";
 const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
 const IAM_STS_PREFIX: &str = "config/iam/sts/";
+const MINIO_GO_ZERO_TIME: OffsetDateTime = time::macros::datetime!(0001-01-01 00:00 UTC);
 const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
 const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
 const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
@@ -120,6 +121,15 @@ fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Opt
     if is_identity_path(path) {
         let mut identity: UserIdentity =
             serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
+        if (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX))
+            && identity
+                .credentials
+                .expiration
+                .as_ref()
+                .is_some_and(|expiration| *expiration == MINIO_GO_ZERO_TIME || *expiration == OffsetDateTime::UNIX_EPOCH)
+        {
+            identity.credentials.expiration = None;
+        }
         if identity.update_at.is_none() {
             identity.update_at = Some(OffsetDateTime::now_utc());
         }
@@ -441,7 +451,10 @@ mod tests {
     use crate::bucket::replication::{
         BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
     };
+    use rustfs_policy::auth::UserIdentity;
     use std::collections::HashMap;
+    use time::OffsetDateTime;
+    use time::format_description::well_known::Rfc3339;
 
     #[test]
     fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
@@ -493,6 +506,54 @@ mod tests {
         assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
     }
 
+    #[test]
+    fn test_normalize_minio_permanent_credential_expiration() {
+        let cases = [
+            ("config/iam/users/alice/identity.json", "0001-01-01T00:00:00Z", true),
+            ("config/iam/users/alice/identity.json", "1970-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "0001-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00Z", true),
+            ("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00.000000001Z", false),
+            ("config/iam/sts/temp/identity.json", "0001-01-01T00:00:00Z", false),
+            ("config/iam/sts/temp/identity.json", "1970-01-01T00:00:00Z", false),
+            ("config/iam/users/alice/identity.json", "1969-12-31T23:59:59Z", false),
+            ("config/iam/users/alice/identity.json", "1970-01-01T00:00:00.000000001Z", false),
+            ("config/iam/users/alice/identity.json", "0001-01-01T00:00:00.000000001Z", false),
+            ("config/iam/users/alice/identity.json", "2030-01-01T00:00:00Z", false),
+        ];
+
+        for (path, expiration, should_clear) in cases {
+            let input = serde_json::json!({
+                "version": 1,
+                "credentials": {
+                    "accessKey": "test-access",
+                    "secretKey": "test-secret",
+                    "sessionToken": "test-session-token",
+                    "parentUser": "test-parent",
+                    "expiration": expiration,
+                }
+            });
+            let output = normalize_iam_config_blob(path, &serde_json::to_vec(&input).expect("serialize identity fixture"))
+                .expect("normalize should succeed")
+                .expect("identity path should be supported");
+            let identity: UserIdentity = serde_json::from_slice(&output).expect("deserialize normalized identity");
+
+            assert_eq!(identity.credentials.access_key, "test-access");
+            assert_eq!(identity.credentials.secret_key, "test-secret");
+            assert_eq!(identity.credentials.session_token, "test-session-token");
+            assert_eq!(identity.credentials.parent_user, "test-parent");
+            if should_clear {
+                assert_eq!(identity.credentials.expiration, None, "path: {path}, expiration: {expiration}");
+            } else {
+                assert_eq!(
+                    identity.credentials.expiration,
+                    Some(OffsetDateTime::parse(expiration, &Rfc3339).expect("parse expected expiration")),
+                    "path: {path}, expiration: {expiration}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn test_normalize_bucket_meta_blob_resync_reencode() {
         let path = ".buckets/test/.replication/resync.bin";

crates/ecstore/src/bucket/quota/checker.rs

@@ -76,7 +76,12 @@ impl QuotaChecker {
 
         let current_usage = self.get_real_time_usage(bucket).await?;
 
-        let admission_size = if uses_durable_reservations { 0 } else { operation_size };
+        // The reporting path projects this operation; storage mutations reserve it at commit.
+        let admission_size = if uses_durable_reservations && !force_usage_calculation {
+            0
+        } else {
+            operation_size
+        };
         let expected_usage = match operation {
             QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
                 current_usage.saturating_add(admission_size)

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -52,8 +52,8 @@ use super::replication_storage_boundary::{
     ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
 };
 use super::replication_target_boundary::{
-    ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore,
+    ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
-    SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
+    ReplicationTargetStore, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
     replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
     replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
     replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
@@ -214,7 +214,7 @@ async fn head_object_for_worker(
     target_bucket: &str,
     object: &str,
     version_id: Option<String>,
-) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+) -> std::result::Result<HeadObjectOutput, HeadObjectSdkError> {
     target_client.head_object(target_bucket, object, version_id).await
 }
 
@@ -233,7 +233,7 @@ async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetCli
 async fn head_object_fallback(
     tgt_client: &TargetClient,
     object: &str,
-) -> std::result::Result<Option<HeadObjectOutput>, SdkError<HeadObjectError>> {
+) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
     match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
         Ok(oi) => Ok(Some(oi)),
         Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
@@ -1152,11 +1152,11 @@ fn spawn_resync_walk_task<S: ReplicationStorage>(
 /// updating the per-object status counters and returning the accounted size
 /// together with any verification error.
 async fn verify_resync_head_result(
-    head_result: std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>>,
+    head_result: std::result::Result<HeadObjectOutput, HeadObjectSdkError>,
     roi: &ReplicateObjectInfo,
     st: &mut TargetReplicationResyncStatus,
     target_client: &Arc<TargetClient>,
-) -> (i64, Option<SdkError<HeadObjectError>>) {
+) -> (i64, Option<HeadObjectSdkError>) {
     match head_result {
         Ok(_) => {
             st.replicated_count += 1;
@@ -1275,7 +1275,7 @@ async fn resync_worker_process_object<S: ReplicationStorage>(
             "Processed resync object"
         );
     }
-    st.error = err.as_ref().and_then(resync_target_error_detail);
+    st.error = err.as_ref().and_then(|err| resync_target_error_detail(err.as_ref()));
 
     st
 }
@@ -2467,7 +2467,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             Ok(_) => {}
             Err(e) => {
                 let non_retryable = matches!(
-                    &e,
+                    e.as_ref(),
                     SdkError::ServiceError(service_err)
                         if is_retryable_delete_replication_head_error(
                             service_err.err().is_not_found(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -36,7 +36,8 @@ use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 
 pub(crate) use crate::bucket::bucket_target_sys::{
-    AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
+    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
+    resolve_read_api_version_id,
 };
 #[cfg(test)]
 pub(crate) use crate::bucket::target::BucketTarget;

crates/ecstore/src/core/pools.rs

@@ -94,6 +94,9 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
 const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
 const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
 const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
+/// Background decommission walks must tolerate slow object migrations; the
+/// stall timeout is the drive-health bound, not the total listing duration.
+const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
 
 pub const POOL_META_NAME: &str = "pool.bin";
 pub const POOL_META_FORMAT: u16 = 1;
@@ -5047,6 +5050,8 @@ impl SetDisks {
                 path: bucket_info.prefix.clone(),
                 recursive: true,
                 min_disks: listing_quorum,
+                skip_walkdir_total_timeout: true,
+                walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
                 agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
                 partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
                     let resolver = resolver.clone();

crates/ecstore/src/disk/os.rs

@@ -23,7 +23,7 @@ use std::{
     io,
     path::{Component, Path, PathBuf},
     sync::{Arc, LazyLock, Weak},
-    time::Instant,
+    time::{Duration, Instant},
 };
 use tokio::fs;
 use tokio::sync::{
@@ -328,6 +328,9 @@ const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR
 const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
 const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
 const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
+const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
+const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
+const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
 #[cfg(not(test))]
 const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
 #[cfg(test)]
@@ -354,6 +357,16 @@ static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
 static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
     rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
 });
+fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
+    Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
+}
+
+static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
+    file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
+        ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
+        DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
+    ))
+});
 
 #[cfg(test)]
 mod dst_dir_fsync_group_commit_override {
@@ -402,6 +415,7 @@ mod file_fdatasync_group_commit_override {
     use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
 
     static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
+    static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
     static SERIAL: Mutex<()> = Mutex::new(());
 
     pub(crate) fn get() -> Option<bool> {
@@ -415,6 +429,7 @@ mod file_fdatasync_group_commit_override {
     impl Drop for OverrideGuard {
         fn drop(&mut self) {
             *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
+            *WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
         }
     }
 
@@ -423,6 +438,14 @@ mod file_fdatasync_group_commit_override {
         *OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
         OverrideGuard { _serial: serial }
     }
+
+    pub(crate) fn set_wait_micros(wait_micros: u64) {
+        *WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = Some(wait_micros);
+    }
+
+    pub(crate) fn wait_micros() -> Option<u64> {
+        *WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
+    }
 }
 
 #[cfg(test)]
@@ -430,6 +453,11 @@ pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fd
     file_fdatasync_group_commit_override::set(enabled)
 }
 
+#[cfg(test)]
+fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
+    file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
+}
+
 fn file_fdatasync_group_commit_enabled() -> bool {
     #[cfg(test)]
     if let Some(enabled) = file_fdatasync_group_commit_override::get() {
@@ -439,6 +467,15 @@ fn file_fdatasync_group_commit_enabled() -> bool {
     *FILE_FDATASYNC_GROUP_COMMIT_ENABLED
 }
 
+fn file_fdatasync_group_commit_wait() -> Duration {
+    #[cfg(test)]
+    if let Some(wait_micros) = file_fdatasync_group_commit_override::wait_micros() {
+        return file_fdatasync_group_commit_wait_duration(wait_micros);
+    }
+
+    *FILE_FDATASYNC_GROUP_COMMIT_WAIT
+}
+
 #[derive(Clone, Eq, Hash, PartialEq)]
 struct DstDirFsyncGroupKey {
     canonical_path: PathBuf,
@@ -934,6 +971,10 @@ async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
         #[cfg(test)]
         file_sync_probe::run_before_group_batch();
         tokio::task::yield_now().await;
+        let wait = file_fdatasync_group_commit_wait();
+        if !wait.is_zero() {
+            tokio::time::sleep(wait).await;
+        }
         let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
             let mut group_state = group.inner.lock();
             let batch_file_count = group_state.pending_files;
@@ -6075,6 +6116,7 @@ mod tests {
         use std::sync::mpsc;
 
         let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        set_file_fdatasync_group_commit_wait_for_test(0);
         clear_file_fdatasync_group_commit_for_test();
         let temp_dir = tempdir().expect("create temp dir");
         let first_dir = temp_dir.path().join("first");
@@ -6141,12 +6183,105 @@ mod tests {
         assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
     }
 
+    #[test]
+    fn file_fdatasync_group_commit_wait_duration_uses_default_and_cap() {
+        assert_eq!(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS, 0);
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
+            Duration::ZERO
+        );
+        assert_eq!(file_fdatasync_group_commit_wait_duration(250), Duration::from_micros(250));
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
+            Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
+        );
+        assert_eq!(
+            file_fdatasync_group_commit_wait_duration(u64::MAX),
+            Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial(file_sync_probe)]
+    async fn file_fdatasync_group_commit_wait_budget_batches_late_follower() {
+        use std::sync::mpsc;
+
+        let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        let wait_budget_micros = 1_000;
+        let wait_budget = file_fdatasync_group_commit_wait_duration(wait_budget_micros);
+        set_file_fdatasync_group_commit_wait_for_test(wait_budget_micros);
+        clear_file_fdatasync_group_commit_for_test();
+        let temp_dir = tempdir().expect("create temp dir");
+        let first_dir = temp_dir.path().join("first");
+        let second_dir = temp_dir.path().join("second");
+        std::fs::create_dir(&first_dir).expect("create first dir");
+        std::fs::create_dir(&second_dir).expect("create second dir");
+        std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
+        std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
+        let _probe = file_sync_probe::set_blocking(temp_dir.path());
+        let (entered_tx, entered_rx) = mpsc::channel();
+        file_sync_probe::set_before_group_batch(move || {
+            entered_tx.send(()).expect("signal first file fdatasync group worker");
+        });
+
+        let limiter = file_sync_limiter();
+        let first_limiter = limiter.clone();
+        let first_path = first_dir.clone();
+        let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
+        tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
+            .await
+            .expect("group worker hook waiter should run")
+            .expect("first file fdatasync group worker should start");
+
+        let second_limiter = limiter.clone();
+        let second_path = second_dir.clone();
+        let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
+        tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                if file_fdatasync_group_commit_counts_for_test().1 == 2 {
+                    return;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("second waiter should enqueue during the configured wait budget");
+        tokio::time::advance(wait_budget).await;
+        tokio::task::yield_now().await;
+        file_sync_probe::wait_for_active(1).await;
+
+        assert_eq!(
+            file_sync_probe::group_batches(),
+            vec![2],
+            "configured wait budget should let a follower join the leader's batch"
+        );
+        file_sync_probe::release();
+        first
+            .await
+            .expect("join first wait-budget file sync")
+            .expect("first wait-budget file sync must succeed");
+        second
+            .await
+            .expect("join second wait-budget file sync")
+            .expect("second wait-budget file sync must succeed");
+        assert!(
+            fsync_dir_recorder::was_fsynced(&first_dir),
+            "first source directory must still be fsynced"
+        );
+        assert!(
+            fsync_dir_recorder::was_fsynced(&second_dir),
+            "second source directory must still be fsynced"
+        );
+        assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial_test::serial(file_sync_probe)]
     async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
         use std::sync::mpsc;
 
         let _group_commit = set_file_fdatasync_group_commit_for_test(true);
+        set_file_fdatasync_group_commit_wait_for_test(0);
         clear_file_fdatasync_group_commit_for_test();
         let temp_dir = tempdir().expect("create temp dir");
         let first_dir = temp_dir.path().join("first");

crates/ecstore/src/object_api/types.rs

@@ -719,14 +719,23 @@ impl ObjectInfo {
     }
 
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
-        let name = decode_dir_object(object);
-
         let mut version_id = fi.version_id;
 
         if versioned && version_id.is_none() {
             version_id = Some(Uuid::nil())
         }
 
+        Self::from_file_info_with_version_id(fi, bucket, object, version_id)
+    }
+
+    pub(crate) fn from_file_info_with_version_id(
+        fi: &FileInfo,
+        bucket: &str,
+        object: &str,
+        version_id: Option<Uuid>,
+    ) -> ObjectInfo {
+        let name = decode_dir_object(object);
+
         // etag
         let (content_type, content_encoding, etag) = {
             let content_type = fi.metadata.get("content-type").cloned();
@@ -1640,6 +1649,18 @@ mod tests {
         assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
     }
 
+    #[test]
+    fn from_file_info_with_version_id_keeps_normalized_absent_version() {
+        let fi = FileInfo {
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        let info = ObjectInfo::from_file_info_with_version_id(&fi, "bucket", "object", None);
+
+        assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
+    }
+
     #[test]
     fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
         for legacy_label in [

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -657,7 +657,7 @@ where
             prefix,
             marker,
             None,
-            i32::try_from(limit).map_or(i32::MAX, |value| value),
+            i32::try_from(limit).unwrap_or(i32::MAX),
             false,
             None,
             false,

crates/ecstore/src/set_disk/mod.rs

@@ -1501,6 +1501,102 @@ pub fn get_lock_acquire_timeout() -> Duration {
     }
 }
 
+fn get_put_object_commit_lock_acquire_timeout_override_ms() -> u64 {
+    #[cfg(test)]
+    {
+        rustfs_utils::get_env_u64(
+            rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+            rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+        )
+    }
+    #[cfg(not(test))]
+    {
+        static CACHED: OnceLock<u64> = OnceLock::new();
+        *CACHED.get_or_init(|| {
+            rustfs_utils::get_env_u64(
+                rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+                rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
+            )
+        })
+    }
+}
+
+fn get_put_object_commit_lock_acquire_timeout(op: &'static str) -> Duration {
+    let default_timeout = get_lock_acquire_timeout();
+    if op != "put_object_commit" {
+        return default_timeout;
+    }
+
+    let timeout_ms = get_put_object_commit_lock_acquire_timeout_override_ms();
+    if timeout_ms == 0 {
+        default_timeout
+    } else {
+        Duration::from_millis(timeout_ms)
+    }
+}
+
+fn put_object_commit_lock_timeout_override_enabled(op: &'static str) -> bool {
+    op == "put_object_commit" && get_put_object_commit_lock_acquire_timeout_override_ms() != 0
+}
+
+fn put_object_commit_lock_admission_budget_label() -> &'static str {
+    match get_put_object_commit_lock_acquire_timeout_override_ms() {
+        0 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+        1..=250 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+        251..=500 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+        501..=1000 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
+        _ => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
+    }
+}
+
+fn record_put_object_commit_lock_admission(op: &'static str, outcome: &'static str) {
+    if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
+        return;
+    }
+    rustfs_io_metrics::record_put_object_commit_lock_admission(put_object_commit_lock_admission_budget_label(), outcome);
+}
+
+fn put_object_commit_lock_acquire_error_outcome(op: &'static str, err: &rustfs_lock::error::LockError) -> &'static str {
+    if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
+        rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
+    } else {
+        rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+    }
+}
+
+fn resolve_put_object_commit_lock_acquire_result(
+    set: &SetDisks,
+    op: &'static str,
+    bucket: &str,
+    object: &str,
+    result: std::result::Result<rustfs_lock::namespace::NamespaceLockGuard, rustfs_lock::error::LockError>,
+) -> Result<rustfs_lock::namespace::NamespaceLockGuard> {
+    match result {
+        Ok(guard) => {
+            record_put_object_commit_lock_admission(op, rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED);
+            Ok(guard)
+        }
+        Err(err) => {
+            record_put_object_commit_lock_admission(op, put_object_commit_lock_acquire_error_outcome(op, &err));
+            Err(map_put_object_commit_lock_acquire_error(set, op, bucket, object, err))
+        }
+    }
+}
+
+fn map_put_object_commit_lock_acquire_error(
+    set: &SetDisks,
+    op: &'static str,
+    bucket: &str,
+    object: &str,
+    err: rustfs_lock::error::LockError,
+) -> StorageError {
+    if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
+        StorageError::SlowDown
+    } else {
+        set.map_namespace_lock_error(bucket, object, "write", err)
+    }
+}
+
 pub fn is_object_lock_diag_enabled() -> bool {
     *OBJECT_LOCK_DIAG_ENABLED.get_or_init(|| {
         let enabled = rustfs_utils::get_env_bool(
@@ -3302,10 +3398,14 @@ impl SetDisks {
         let diag_enabled = is_object_lock_diag_enabled();
         let ns_lock = self.new_ns_lock(bucket, object).await?;
         let acquire_start = Instant::now();
-        let guard = ns_lock
+        let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
-            .get_write_lock(get_lock_acquire_timeout())
+        let guard = resolve_put_object_commit_lock_acquire_result(
-            .await
+            self,
-            .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
+            op,
+            bucket,
+            object,
+            ns_lock.get_write_lock(acquire_timeout).await,
+        )?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
         let owner = diag_enabled.then(|| ns_lock.owner().to_string());
         self.log_object_lock_acquire_if_slow(
@@ -3340,10 +3440,16 @@ impl SetDisks {
         let diag_enabled = is_object_lock_diag_enabled();
         let ns_lock = self.new_ns_lock(bucket, object).await?;
         let acquire_start = Instant::now();
-        let acquire = ns_lock.get_write_lock(get_lock_acquire_timeout());
+        let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
+        let acquire = ns_lock.get_write_lock(acquire_timeout);
         tokio::pin!(acquire);
         let mut on_pending = Some(on_pending);
-        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+        let guard = resolve_put_object_commit_lock_acquire_result(
+            self,
+            op,
+            bucket,
+            object,
+            futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
                 std::task::Poll::Pending => {
                     if let Some(on_pending) = on_pending.take() {
                         on_pending();
@@ -3352,8 +3458,8 @@ impl SetDisks {
                 }
                 std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
             })
-        .await
+            .await,
-        .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
+        )?;
         Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
         let owner = diag_enabled.then(|| ns_lock.owner().to_string());
         self.log_object_lock_acquire_if_slow(
@@ -5717,8 +5823,8 @@ mod tests {
             .filter(|(composite, _, _, _)| {
                 composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
                     && composite.key().labels().any(|label| {
-                        label.key().to_string() == "stage"
+                        label.key() == "stage"
-                            && label.value().to_string() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
+                            && label.value() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
                     })
             })
             .map(|(_, _, _, value)| match value {
@@ -5728,6 +5834,81 @@ mod tests {
             .sum()
     }
 
+    fn put_object_commit_lock_admission_count(
+        rows: &[(
+            metrics_util::CompositeKey,
+            Option<metrics::Unit>,
+            Option<metrics::SharedString>,
+            DebugValue,
+        )],
+        budget: &'static str,
+        outcome: &'static str,
+    ) -> u64 {
+        rows.iter()
+            .filter(|(composite, _, _, _)| {
+                composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "budget" && label.value() == budget)
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "outcome" && label.value() == outcome)
+            })
+            .map(|(_, _, _, value)| match value {
+                DebugValue::Counter(count) => *count,
+                _ => 0,
+            })
+            .sum()
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_budget_labels_are_bounded() {
+        let cases = [
+            ("0", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED),
+            ("250", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS),
+            ("251", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
+            ("500", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
+            ("501", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
+            ("1000", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
+            ("1001", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS),
+        ];
+        for (timeout_ms, expected) in cases {
+            temp_env::with_vars(
+                [(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some(timeout_ms))],
+                || {
+                    assert_eq!(put_object_commit_lock_admission_budget_label(), expected);
+                },
+            );
+        }
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_error_outcomes_are_bounded() {
+        let timeout = LockError::timeout("bucket/object", Duration::from_millis(1));
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("put_object_commit", &timeout),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
+            );
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("complete_multipart_upload_commit", &timeout),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+            );
+        });
+
+        let internal = LockError::internal("simulated lock manager error");
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            assert_eq!(
+                put_object_commit_lock_acquire_error_outcome("put_object_commit", &internal),
+                rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
+            );
+        });
+    }
+
     #[test]
     #[serial]
     fn put_object_commit_namespace_lock_wait_metric_is_wired_to_both_write_lock_paths() {
@@ -5793,6 +5974,289 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_timeout_override_only_applies_to_put_commit() {
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("17"))], || {
+            assert_eq!(get_put_object_commit_lock_acquire_timeout("put_object_commit"), Duration::from_millis(17));
+            assert_eq!(
+                get_put_object_commit_lock_acquire_timeout("complete_multipart_upload_commit"),
+                get_lock_acquire_timeout()
+            );
+        });
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
+            assert_eq!(
+                get_put_object_commit_lock_acquire_timeout("put_object_commit"),
+                get_lock_acquire_timeout()
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_timeout_override_bounds_contention_wait() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            runtime.block_on(async {
+                let ctx = Arc::new(InstanceContext::new());
+                ctx.update_erasure_type(SetupType::Erasure).await;
+                let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                let bucket = "bucket";
+                let object = "object";
+
+                let held_guard = set
+                    .acquire_write_lock_diag("put_object_commit", bucket, object)
+                    .await
+                    .expect("holder acquire should succeed");
+                let started = Instant::now();
+                let err = match set.acquire_write_lock_diag("put_object_commit", bucket, object).await {
+                    Ok(_) => panic!("contended PUT commit lock should honor the short timeout"),
+                    Err(err) => err,
+                };
+                assert!(
+                    started.elapsed() < Duration::from_secs(1),
+                    "short PUT commit lock timeout should not wait for the global timeout"
+                );
+                assert!(matches!(err, StorageError::SlowDown));
+
+                drop(held_guard);
+                set.acquire_write_lock_diag("put_object_commit", bucket, object)
+                    .await
+                    .expect("permit should not leak after timeout");
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_acquired_and_timeout() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let held_guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("holder acquire should succeed");
+                    let err = match set.acquire_write_lock_diag("put_object_commit", "bucket", "object").await {
+                        Ok(_) => panic!("contended PUT commit acquire should return SlowDown"),
+                        Err(err) => err,
+                    };
+                    assert!(matches!(err, StorageError::SlowDown));
+                    drop(held_guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                1
+            );
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+                ),
+                1
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_disabled_budget_acquired() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("PUT commit acquire should succeed with default timeout");
+                    drop(guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                1
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_skips_non_put_commit_ops() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let guard = set
+                        .acquire_write_lock_diag("complete_multipart_upload_commit", "bucket", "object")
+                        .await
+                        .expect("non-PUT commit acquire should succeed");
+                    drop(guard);
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                rows.iter()
+                    .filter(|(composite, _, _, _)| {
+                        composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+                    })
+                    .count(),
+                0
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_lock_error() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let healthy: Arc<dyn LockClient> =
+                        Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+                    let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::DistErasure).await;
+                    let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
+                    assert!(
+                        set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                            .await
+                            .is_err(),
+                        "one healthy locker must not satisfy the PUT commit write quorum"
+                    );
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
+                ),
+                1
+            );
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+                ),
+                0
+            );
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn put_object_commit_lock_admission_records_pending_hook_acquired() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should start");
+
+        temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("500"))], || {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                rustfs_io_metrics::set_put_stage_metrics_enabled(true);
+                runtime.block_on(async {
+                    let ctx = Arc::new(InstanceContext::new());
+                    ctx.update_erasure_type(SetupType::Erasure).await;
+                    let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
+                    let held_guard = set
+                        .acquire_write_lock_diag("put_object_commit", "bucket", "object")
+                        .await
+                        .expect("holder acquire should succeed");
+                    let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
+                    let pending_acquire =
+                        set.acquire_write_lock_diag_with_pending_hook("put_object_commit", "bucket", "object", move || {
+                            let _ = pending_tx.send(());
+                        });
+                    let release_holder = async {
+                        pending_rx.await.expect("pending hook should fire");
+                        drop(held_guard);
+                    };
+                    let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
+                    drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
+                    rustfs_io_metrics::set_put_stage_metrics_enabled(false);
+                });
+            });
+
+            let rows = snapshotter.snapshot().into_vec();
+            assert_eq!(
+                put_object_commit_lock_admission_count(
+                    &rows,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+                    rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+                ),
+                2
+            );
+        });
+    }
+
     #[tokio::test]
     async fn new_ns_lock_shares_clients_without_changing_quorum() {
         let healthy: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));

crates/ecstore/src/set_disk/ops/heal_walk.rs

@@ -124,14 +124,7 @@ impl HealWalkCollector {
         for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
             let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
             let lifecycle_object_info = if self.include_lifecycle_object_info {
-                let mut lifecycle_fi = fi.clone();
+                Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
-                lifecycle_fi.version_id = version_uuid;
-                Some(ObjectInfo::from_file_info(
-                    &lifecycle_fi,
-                    &self.bucket,
-                    &entry.name,
-                    version_uuid.is_some(),
-                ))
             } else {
                 None
             };
@@ -198,14 +191,7 @@ impl HealWalkCollector {
                 let vid = version_uuid.map(|u| u.to_string());
                 if seen.insert(vid.clone()) {
                     let lifecycle_object_info = if self.include_lifecycle_object_info {
-                        let mut lifecycle_fi = fi.clone();
+                        Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
-                        lifecycle_fi.version_id = version_uuid;
-                        Some(ObjectInfo::from_file_info(
-                            &lifecycle_fi,
-                            &self.bucket,
-                            &entry.name,
-                            version_uuid.is_some(),
-                        ))
                     } else {
                         None
                     };

crates/ecstore/src/set_disk/ops/object.rs

@@ -1322,7 +1322,12 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let object_info = prepared_object_info
             .unwrap_or_else(|| build_get_object_info(fi, bucket, object, opts.versioned || opts.version_suspended));
         let object_class = classify_get_codec_streaming_object_class(&range, &object_info, fi);
-        let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size);
+        let metrics_size = if stage_metrics_enabled {
+            object_info.get_actual_size().unwrap_or(object_info.size)
+        } else {
+            object_info.size
+        };
+        let size_bucket = rustfs_io_metrics::get_object_size_bucket(metrics_size);
         record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start);
         let metadata_elapsed = metadata_stage_start.elapsed().as_secs_f64();
         rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_elapsed);
@@ -3766,7 +3771,7 @@ pub(crate) async fn complete_transition_upload<Remote, Producer>(
     producer: Producer,
     expected_size: u64,
     consumed: Arc<AtomicU64>,
-) -> std::result::Result<TransitionUploadCompletion, TransitionUploadFailure>
+) -> std::result::Result<TransitionUploadCompletion, Box<TransitionUploadFailure>>
 where
     Remote: Future<Output = std::result::Result<String, std::io::Error>>,
     Producer: Future<Output = Result<u64>>,
@@ -3784,23 +3789,23 @@ where
                 Err(_) => StorageError::Unexpected,
                 Ok(Ok(_)) => StorageError::Io(remote_error),
             };
-            return Err(TransitionUploadFailure { error, candidate: None });
+            return Err(Box::new(TransitionUploadFailure { error, candidate: None }));
         }
     };
     let candidate = TransitionUploadCandidate::from_put_response(remote_version);
     let produced = match producer_result {
         Ok(Ok(produced)) => produced,
         Ok(Err(error)) => {
-            return Err(TransitionUploadFailure {
+            return Err(Box::new(TransitionUploadFailure {
                 error,
                 candidate: Some(candidate),
-            });
+            }));
         }
         Err(_) => {
-            return Err(TransitionUploadFailure {
+            return Err(Box::new(TransitionUploadFailure {
                 error: StorageError::Unexpected,
                 candidate: Some(candidate),
-            });
+            }));
         }
     };
     let consumed = consumed.load(Ordering::Acquire);
@@ -3810,10 +3815,10 @@ where
         } else {
             StorageError::MoreData
         };
-        return Err(TransitionUploadFailure {
+        return Err(Box::new(TransitionUploadFailure {
             error,
             candidate: Some(candidate),
-        });
+        }));
     }
     Ok(TransitionUploadCompletion {
         candidate,
@@ -7284,7 +7289,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
             let gr = gr?;
             let reader = BufReader::new(gr.stream);
-            let hash_reader = HashReader::from_stream(reader, gr.object_info.size, gr.object_info.size, None, None, false)?;
+            let hash_reader = HashReader::from_stream(reader, gr.object_info.size, oi.get_actual_size()?, None, None, false)?;
             let mut p_reader = PutObjReader::new(hash_reader);
             return match self_.clone().put_object(bucket, object, &mut p_reader, &ropts).await {
                 Ok(restored_info) => {
@@ -8826,7 +8831,7 @@ mod transition_commit_failure_tests {
     use s3s::dto::RestoreRequest;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-    fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
+    pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
         let mut metadata = HashMap::new();
         rustfs_utils::http::metadata_compat::insert_str(
             &mut metadata,
@@ -8836,7 +8841,7 @@ mod transition_commit_failure_tests {
         metadata
     }
 
-    fn restore_metadata(operation_id: Uuid, ongoing: bool) -> HashMap<String, String> {
+    pub(super) fn restore_metadata(operation_id: Uuid, ongoing: bool) -> HashMap<String, String> {
         let mut metadata = restore_operation_id_metadata(operation_id);
         metadata.insert(s3s::header::X_AMZ_RESTORE.as_str().to_string(), format!("ongoing-request=\"{ongoing}\""));
         metadata
@@ -10097,6 +10102,51 @@ mod transition_commit_failure_tests {
             .await
             .expect("operation B should replace operation A before final commit");
 
+        let mismatch = set_disks
+            .finalize_restore_metadata(
+                bucket,
+                object,
+                &set_disks
+                    .get_object_info(bucket, object, &ObjectOptions::default())
+                    .await
+                    .expect("operation B metadata should be readable"),
+                &ObjectOptions {
+                    user_defined: restore_operation_id_metadata(operation_a),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("operation A must not finalize operation B metadata");
+        assert!(matches!(
+            mismatch,
+            Error::Io(ref error)
+                if error.kind() == std::io::ErrorKind::Other
+                    && error.to_string() == "restore operation id changed before metadata finalization"
+        ));
+        let current = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("operation B metadata should remain after mismatched finalization");
+        assert_eq!(
+            rustfs_utils::http::metadata_compat::get_consistent_str(
+                current.user_defined.as_ref(),
+                rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
+            ),
+            Some(operation_b.to_string().as_str()),
+            "mismatched finalization must not remove operation B"
+        );
+        assert!(
+            parse_restore_obj_status(
+                current
+                    .user_defined
+                    .get(s3s::header::X_AMZ_RESTORE.as_str())
+                    .expect("operation B restore header should remain pending"),
+            )
+            .expect("operation B restore header should parse")
+            .on_going(),
+            "mismatched finalization must not publish restore completion"
+        );
+
         let mut stale_restore_reader = PutObjReader::from_vec(b"stale A restored body".repeat(1024));
         let result = set_disks
             .put_object(
@@ -10126,18 +10176,37 @@ mod transition_commit_failure_tests {
 
         let mut matching_restore_reader = PutObjReader::from_vec(b"matching B restored body".repeat(1024));
         let operation_b_restore_metadata = restore_metadata(operation_b, false);
-        set_disks
+        let restored = set_disks
             .put_object(
                 bucket,
                 object,
                 &mut matching_restore_reader,
                 &ObjectOptions {
-                    user_defined: operation_b_restore_metadata,
+                    user_defined: operation_b_restore_metadata.clone(),
                     ..Default::default()
                 },
             )
             .await
             .expect("matching operation B should be allowed to commit");
+        set_disks
+            .finalize_restore_metadata(
+                bucket,
+                object,
+                &restored,
+                &ObjectOptions {
+                    user_defined: restore_operation_id_metadata(operation_b),
+                    transition: TransitionOptions {
+                        restore_request: RestoreRequest {
+                            days: Some(1),
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("matching operation B should finalize after its commit consumes the operation id");
         let restored = set_disks
             .get_object_info(bucket, object, &ObjectOptions::default())
             .await
@@ -10503,13 +10572,16 @@ mod transition_commit_failure_tests {
 #[cfg(all(test, feature = "test-util"))]
 mod transition_upload_integrity_tests {
     use super::hermetic_set_disks_support::{hermetic_set_disks, hermetic_set_disks_with_lockers};
+    use super::transition_commit_failure_tests::{restore_metadata, restore_operation_id_metadata};
     use super::*;
     use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
     use crate::disk::DiskAPI as _;
     use crate::layout::endpoints::SetupType;
     use crate::services::tier::test_util::register_mock_tier;
+    use crate::set_disk::replication::RestoreFinalizeBarrier;
     use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
     use http::HeaderMap;
+    use rustfs_filemeta::RestoreStatusOps as _;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats};
     use std::collections::HashSet;
@@ -10655,6 +10727,162 @@ mod transition_upload_integrity_tests {
         }
     }
 
+    async fn write_committed_restore(
+        set_disks: &Arc<SetDisks>,
+        disk_stores: &[DiskStore],
+        bucket: &str,
+        object: &str,
+        operation_id: Uuid,
+    ) -> ObjectInfo {
+        for disk in disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+        let mut source = PutObjReader::from_vec(b"restore source body".repeat(1024));
+        set_disks
+            .put_object(bucket, object, &mut source, &ObjectOptions::default())
+            .await
+            .expect("source object should be written");
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    eval_metadata: Some(restore_metadata(operation_id, true)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("pending restore metadata should be installed");
+
+        let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(1024));
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut restored_reader,
+                &ObjectOptions {
+                    user_defined: restore_metadata(operation_id, true),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("matching restore commit should consume its operation id")
+    }
+
+    fn restore_finalize_options(operation_id: Uuid) -> ObjectOptions {
+        ObjectOptions {
+            user_defined: restore_operation_id_metadata(operation_id),
+            transition: TransitionOptions {
+                restore_request: s3s::dto::RestoreRequest {
+                    days: Some(1),
+                    ..Default::default()
+                },
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
+        let current = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("pending restore metadata should remain readable");
+        assert!(
+            restore_operation_id_from_metadata(current.user_defined.as_ref())
+                .expect("operation id metadata should parse")
+                .is_none(),
+            "successful restore commit must have consumed the operation id"
+        );
+        assert!(
+            rustfs_filemeta::parse_restore_obj_status(
+                current
+                    .user_defined
+                    .get(s3s::header::X_AMZ_RESTORE.as_str())
+                    .expect("pending restore header should remain"),
+            )
+            .expect("restore header should parse")
+            .on_going(),
+            "failed finalization must not publish completion metadata"
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial]
+    async fn restore_finalize_rejects_acquired_lock_loss_after_commit() {
+        let refresh_calls = Arc::new(AtomicUsize::new(0));
+        let lockers: Vec<Arc<dyn LockClient>> = (0..4)
+            .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
+            .collect();
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+        let bucket = "restore-finalize-acquired-lock-lost-bucket";
+        let object = "object.bin";
+        let operation_id = Uuid::new_v4();
+        let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+        let barrier = RestoreFinalizeBarrier::install(bucket, object);
+        let finalize_set = Arc::clone(&set_disks);
+        let finalize = tokio::spawn(async move {
+            finalize_set
+                .finalize_restore_metadata(bucket, object, &restored, &restore_finalize_options(operation_id))
+                .await
+        });
+        barrier.wait_until_paused().await;
+        tokio::time::advance(Duration::from_secs(11)).await;
+        tokio::task::yield_now().await;
+        assert!(refresh_calls.load(Ordering::SeqCst) > 0, "restore finalization lock must attempt renewal");
+        barrier.release();
+
+        let error = finalize
+            .await
+            .expect("restore finalization task should join")
+            .expect_err("lost acquired lock must reject restore finalization");
+        assert!(matches!(
+            error,
+            Error::Io(ref error)
+                if error.kind() == std::io::ErrorKind::Other
+                    && error.to_string() == "restore finalization lock lost before metadata update"
+        ));
+        assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn restore_finalize_rejects_outer_fence_loss_after_metadata_read() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "restore-finalize-outer-fence-lost-bucket";
+        let object = "object.bin";
+        let operation_id = Uuid::new_v4();
+        let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
+        let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
+        let barrier = RestoreFinalizeBarrier::install(bucket, object);
+        let finalize_set = Arc::clone(&set_disks);
+        let finalize = tokio::spawn(async move {
+            let mut opts = restore_finalize_options(operation_id);
+            opts.no_lock = true;
+            opts.namespace_lock_fence = Some(fence);
+            finalize_set.finalize_restore_metadata(bucket, object, &restored, &opts).await
+        });
+        barrier.wait_until_paused().await;
+        loss_handle.store(true, std::sync::atomic::Ordering::Release);
+        barrier.release();
+
+        let error = finalize
+            .await
+            .expect("restore finalization task should join")
+            .expect_err("lost outer fence must reject restore finalization");
+        assert!(matches!(
+            error,
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "restore_finalize_metadata",
+                required: 1,
+                achieved: 0,
+                ..
+            }
+        ));
+        assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
+    }
+
     async fn assert_local_source_intact(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, payload: &[u8]) {
         let mut restored = Vec::new();
         set_disks

crates/ecstore/src/set_disk/replication.rs

@@ -18,6 +18,78 @@ use rustfs_filemeta::RestoreStatusOps;
 use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
 use s3s::dto::{RestoreStatus, Timestamp};
 
+#[cfg(all(test, feature = "test-util"))]
+struct RestoreFinalizeBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+static RESTORE_FINALIZE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<RestoreFinalizeBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(all(test, feature = "test-util"))]
+pub(in crate::set_disk) struct RestoreFinalizeBarrier {
+    state: Arc<RestoreFinalizeBarrierState>,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl RestoreFinalizeBarrier {
+    pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(RestoreFinalizeBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        assert!(slot.is_none(), "restore finalize barrier must be installed by one test at a time");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(in crate::set_disk) async fn wait_until_paused(&self) {
+        self.state.arrived.notified().await;
+    }
+
+    pub(in crate::set_disk) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl Drop for RestoreFinalizeBarrier {
+    fn drop(&mut self) {
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
+    let barrier = RESTORE_FINALIZE_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("restore finalize barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 struct RestoreCleanupIdentity {
     version_id: Option<Uuid>,
@@ -80,7 +152,7 @@ impl SetDisks {
             .clone()
             .unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
         let version_id = expected.version_id.map(|v| v.to_string());
-        let _lock_guard = if !opts.no_lock {
+        let lock_guard = if !opts.no_lock {
             Some(
                 self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
                     .await?,
@@ -99,13 +171,16 @@ impl SetDisks {
             .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
             .await?
             .into_owned();
-        if let Some(expected_operation_id) = expected_operation_id {
+        if let Some(expected_operation_id) = expected_operation_id
-            require_restore_operation_id(&fi.metadata, expected_operation_id)?;
+            && restore_operation_id_from_metadata(&fi.metadata)?.is_some_and(|actual| actual != expected_operation_id)
+        {
+            return Err(Error::other("restore operation id changed before metadata finalization"));
         }
         if !expected.matches_file_info(&fi, &expected_etag) {
             return Err(Error::other("restored object changed before restore metadata finalization"));
         }
-        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        #[cfg(all(test, feature = "test-util"))]
+        maybe_pause_restore_finalize(bucket, object).await;
         let restore_expiry =
             lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
         fi.metadata.insert(
@@ -117,6 +192,10 @@ impl SetDisks {
             .to_string(),
         );
         self.invalidate_get_object_metadata_cache(bucket, object).await;
+        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(Error::other("restore finalization lock lost before metadata update"));
+        }
         self.update_object_meta_with_opts(
             bucket,
             object,

crates/ecstore/src/store/mod.rs

@@ -343,6 +343,23 @@ impl ECStore {
         let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
         decommission || rebalance
     }
+
+    /// Returns whether scanner metadata may still be hidden by a local
+    /// data-movement state. Terminal failed/canceled decommission entries
+    /// remain suspended until an operator clears or retries them, so they are
+    /// a publication barrier even after the worker has stopped.
+    pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
+        if self.scanner_data_movement_active().await {
+            return true;
+        }
+
+        let pool_meta = self.pool_meta.read().await;
+        pool_meta.pools.iter().any(|pool| {
+            pool.decommission
+                .as_ref()
+                .is_some_and(|info| !info.queued && (info.failed || info.canceled))
+        })
+    }
 }
 
 // impl Clone for ECStore {
@@ -875,6 +892,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
     use crate::runtime::global::reset_local_disk_test_state;
     use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
@@ -911,6 +929,72 @@ mod tests {
         })
     }
 
+    #[tokio::test]
+    async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let cases = [
+            (
+                "active",
+                PoolDecommissionInfo {
+                    start_time: Some(OffsetDateTime::now_utc()),
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "failed",
+                PoolDecommissionInfo {
+                    failed: true,
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "canceled",
+                PoolDecommissionInfo {
+                    canceled: true,
+                    ..Default::default()
+                },
+                true,
+            ),
+            (
+                "queued_failed",
+                PoolDecommissionInfo {
+                    failed: true,
+                    queued: true,
+                    ..Default::default()
+                },
+                false,
+            ),
+            (
+                "complete",
+                PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                },
+                false,
+            ),
+            ("idle", PoolDecommissionInfo::default(), false),
+        ];
+
+        for (name, decommission, expected) in cases {
+            *store.pool_meta.write().await = PoolMeta {
+                pools: vec![PoolStatus {
+                    id: 0,
+                    cmd_line: format!("scanner-publication-{name}"),
+                    last_update: OffsetDateTime::now_utc(),
+                    decommission: Some(decommission),
+                }],
+                ..Default::default()
+            };
+            assert_eq!(
+                store.scanner_data_usage_publication_blocked().await,
+                expected,
+                "unexpected scanner publication barrier state for {name}"
+            );
+        }
+    }
+
     // The object graph is the isolation carrier: two ECStore instances holding
     // distinct contexts report independent erasure state through their real
     // `&self` accessors — no cross-contamination.

crates/heal/src/heal/manager/scheduler.rs

@@ -567,35 +567,17 @@ impl HealManager {
 pub(super) fn heal_request_set_key(request: &HealRequest) -> Option<String> {
     match &request.heal_type {
         HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
-        HealType::Object { .. } => heal_options_set_key(&request.options),
+        HealType::Object { .. } => request.options.set_key(),
-        _ => None,
-    }
-}
-
-pub(super) fn heal_options_set_key(options: &HealOptions) -> Option<String> {
-    match (options.pool_index, options.set_index) {
-        (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
         _ => None,
     }
 }
 
 pub(super) fn heal_request_type_label(request: &HealRequest) -> &'static str {
-    match &request.heal_type {
+    request.heal_type.kind_label()
-        HealType::Cluster => "cluster",
-        HealType::Object { .. } => "object",
-        HealType::Bucket { .. } => "bucket",
-        HealType::Prefix { .. } => "prefix",
-        HealType::ErasureSet { .. } => "erasure_set",
-        HealType::Metadata { .. } => "metadata",
-        HealType::ECDecode { .. } => "ec_decode",
-    }
 }
 
 pub(super) fn heal_request_set_metric_label(request: &HealRequest) -> String {
-    heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) {
+    heal_request_set_key(request).unwrap_or_else(|| request.options.set_metric_label())
-        (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
-        _ => "global".to_string(),
-    })
 }
 
 pub(super) fn record_scheduler_skip(set_label: &str) {
@@ -673,7 +655,7 @@ fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
 pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
     match &task.heal_type {
         HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
-        HealType::Object { .. } => heal_options_set_key(&task.options),
+        HealType::Object { .. } => task.options.set_key(),
         _ => None,
     }
 }

crates/heal/src/heal/manager/tests.rs

@@ -744,10 +744,8 @@ fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() {
     let mut running = HashMap::new();
     running.insert("pool_0_set_1".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
+    let (popped, skipped_sets) =
-        |request| can_schedule_request(request, &running, 1),
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
-        |request| heal_request_set_key(request),
-    );
     let popped = popped.expect("should find runnable request");
 
     assert_eq!(skipped_sets, vec!["pool_0_set_1".to_string()]);
@@ -788,10 +786,8 @@ fn test_priority_queue_pop_runnable_restores_all_blocked_items() {
     running.insert("pool_0_set_2".to_string(), 1);
     running.insert("pool_0_set_3".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
+    let (popped, skipped_sets) =
-        |request| can_schedule_request(request, &running, 1),
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
-        |request| heal_request_set_key(request),
-    );
 
     assert!(popped.is_none());
     assert_eq!(
@@ -843,10 +839,8 @@ fn test_priority_queue_pop_runnable_restores_deferred_with_tail() {
     running.insert("pool_0_set_1".to_string(), 1);
     running.insert("pool_0_set_2".to_string(), 1);
 
-    let (popped, skipped_sets) = queue.pop_runnable_with_skips(
+    let (popped, skipped_sets) =
-        |request| can_schedule_request(request, &running, 1),
+        queue.pop_runnable_with_skips(|request| can_schedule_request(request, &running, 1), heal_request_set_key);
-        |request| heal_request_set_key(request),
-    );
 
     assert_eq!(skipped_sets, vec!["pool_0_set_1".to_string(), "pool_0_set_2".to_string()]);
     assert!(matches!(
@@ -904,6 +898,31 @@ fn test_can_schedule_scoped_object_request_respects_per_set_limit() {
     assert!(can_schedule_request(&request, &running, 2));
 }
 
+#[test]
+fn test_heal_request_and_task_metric_labels_match() {
+    let request = HealRequest::new(
+        HealType::Object {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+        },
+        HealOptions {
+            pool_index: Some(0),
+            set_index: Some(1),
+            ..Default::default()
+        },
+        HealPriority::Normal,
+    );
+
+    assert_eq!(heal_request_type_label(&request), "object");
+    assert_eq!(heal_request_set_key(&request), Some("pool_0_set_1".to_string()));
+    assert_eq!(heal_request_set_metric_label(&request), "pool_0_set_1");
+
+    let task = HealTask::from_request(request, Arc::new(MockStorage));
+    assert_eq!(task.metric_type_label(), "object");
+    assert_eq!(task.metric_set_label(), "pool_0_set_1");
+}
+
 #[tokio::test]
 async fn test_submit_heal_request_returns_merged_for_duplicate() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/heal/src/heal/progress.rs

@@ -218,15 +218,6 @@ impl HealStatistics {
         self.total_bytes_healed += bytes;
         self.last_update_time = SystemTime::now();
     }
-
-    pub fn get_success_rate(&self) -> f64 {
-        let total = self.successful_tasks + self.failed_tasks;
-        if total > 0 {
-            (self.successful_tasks as f64 / total as f64) * 100.0
-        } else {
-            0.0
-        }
-    }
 }
 
 #[cfg(test)]
@@ -539,38 +530,4 @@ mod tests {
         assert_eq!(stats.total_objects_healed, 8);
         assert_eq!(stats.total_bytes_healed, 8192);
     }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 8;
-        stats.failed_tasks = 2;
-
-        // success_rate = 8 / (8 + 2) * 100 = 80%
-        assert!((stats.get_success_rate() - 80.0).abs() < 0.001);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_zero_total() {
-        let stats = HealStatistics::new();
-        assert_eq!(stats.get_success_rate(), 0.0);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_all_success() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 10;
-        stats.failed_tasks = 0;
-
-        assert!((stats.get_success_rate() - 100.0).abs() < 0.001);
-    }
-
-    #[test]
-    fn test_heal_statistics_get_success_rate_all_failure() {
-        let mut stats = HealStatistics::new();
-        stats.successful_tasks = 0;
-        stats.failed_tasks = 5;
-
-        assert_eq!(stats.get_success_rate(), 0.0);
-    }
 }

crates/heal/src/heal/storage.rs

@@ -1202,14 +1202,23 @@ impl HealStorageAPI for ECStoreHealStorage {
                 let version_id = obj.version_id.map(|u| u.to_string());
                 let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
                 let is_delete_marker = obj.delete_marker;
-                let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
+                if include_lifecycle_object_info {
+                    HealListItem {
+                        name: obj.name.clone(),
+                        version_id,
+                        mod_time_unix_nanos,
+                        lifecycle_object_info: Some(obj),
+                        is_delete_marker,
+                    }
+                } else {
                     HealListItem {
                         name: obj.name,
                         version_id,
                         mod_time_unix_nanos,
-                    lifecycle_object_info,
+                        lifecycle_object_info: None,
                         is_delete_marker,
                     }
+                }
             })
             .collect();
         let page_count = page_objects.len();

crates/heal/src/heal/task.rs

@@ -109,7 +109,7 @@ pub enum HealType {
 }
 
 impl HealType {
-    fn log_kind(&self) -> &'static str {
+    pub(crate) fn kind_label(&self) -> &'static str {
         match self {
             Self::Cluster => "cluster",
             Self::Object { .. } => "object",
@@ -227,6 +227,19 @@ impl Default for HealOptions {
     }
 }
 
+impl HealOptions {
+    pub(crate) fn set_key(&self) -> Option<String> {
+        match (self.pool_index, self.set_index) {
+            (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
+            _ => None,
+        }
+    }
+
+    pub(crate) fn set_metric_label(&self) -> String {
+        self.set_key().unwrap_or_else(|| "global".to_string())
+    }
+}
+
 /// Heal task status
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub enum HealTaskStatus {
@@ -491,15 +504,7 @@ impl HealTask {
     }
 
     pub fn metric_type_label(&self) -> &'static str {
-        match &self.heal_type {
+        self.heal_type.kind_label()
-            HealType::Cluster => "cluster",
-            HealType::Object { .. } => "object",
-            HealType::Bucket { .. } => "bucket",
-            HealType::Prefix { .. } => "prefix",
-            HealType::ErasureSet { .. } => "erasure_set",
-            HealType::Metadata { .. } => "metadata",
-            HealType::ECDecode { .. } => "ec_decode",
-        }
     }
 
     pub(crate) fn has_batch_failure(&self) -> bool {
@@ -520,10 +525,7 @@ impl HealTask {
     pub fn metric_set_label(&self) -> String {
         match &self.heal_type {
             HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
-            _ => match (self.options.pool_index, self.options.set_index) {
+            _ => self.options.set_metric_label(),
-                (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
-                _ => "global".to_string(),
-            },
         }
     }
 
@@ -532,7 +534,7 @@ impl HealTask {
             let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
                 .with_duration(duration)
                 .with_attr("task_id", self.id.as_str())
-                .with_attr("heal_type", self.heal_type.log_kind())
+                .with_attr("heal_type", self.heal_type.kind_label())
                 .with_attr("state", state)
                 .with_attr("source", self.source.as_str())
                 .with_attr("priority", self.priority.as_str())
@@ -795,7 +797,7 @@ impl HealTask {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_TASK,
             task_id = %self.id,
-            heal_type = self.heal_type.log_kind(),
+            heal_type = self.heal_type.kind_label(),
             state = "started",
             queue_delay = ?queue_delay,
             "Heal task started"
@@ -836,7 +838,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "completed",
                     "Heal task completed"
                 });
@@ -850,7 +852,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "cancelled",
                     "Heal task cancelled"
                 );
@@ -863,7 +865,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "timed_out",
                     "Heal task timed out"
                 });
@@ -880,7 +882,7 @@ impl HealTask {
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
                     task_id = %self.id,
-                    heal_type = self.heal_type.log_kind(),
+                    heal_type = self.heal_type.kind_label(),
                     state = "failed",
                     error = %e,
                     "Heal task failed"
@@ -909,7 +911,7 @@ impl HealTask {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_TASK,
             task_id = %self.id,
-            heal_type = self.heal_type.log_kind(),
+            heal_type = self.heal_type.kind_label(),
             state = "cancelled",
             source = "manual",
             "Heal task cancellation requested"

crates/heal/src/heal/task/tests.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk};
+use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
 use super::*;
 use crate::heal::storage::{HealListItem, HealObjectInfo};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};

crates/iam/tests/ecstore_test_compat/mod.rs

@@ -17,11 +17,11 @@
 //! All direct `rustfs_ecstore` facade imports used by tests in this crate
 //! must go through this module (architecture migration rule:
 //! `check_architecture_migration_rules.sh`). Keep the surface minimal —
-//! only what the tests actually need to build a temp-disk ECStore fixture
+//! only what the tests actually need to run storage-backed IAM scenarios.
-//! and to flip the erasure setup type for lock-quorum fault injection.
 
 #[allow(unused_imports)]
 pub(crate) mod fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::migration::try_migrate_iam_config;
     pub(crate) use rustfs_ecstore::api::layout::SetupType;
 
     // `update_erasure_type` is a write-side global facade entry. Its use is

crates/iam/tests/minio_iam_migration_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod ecstore_test_compat;
+
+use ecstore_test_compat::fixture::try_migrate_iam_config;
+use rustfs_credentials::{get_global_action_cred, init_global_action_credentials};
+use rustfs_iam::store::object::{
+    IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX, IAM_CONFIG_POLICY_DB_USERS_PREFIX, IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX,
+    IAM_CONFIG_USERS_PREFIX, ObjectStore,
+};
+use rustfs_iam::store::{Store, UserType};
+use rustfs_iam::utils::generate_jwt;
+use rustfs_policy::auth::UserIdentity;
+use serde_json::{Value, json};
+use std::collections::HashMap;
+
+const LEGACY_META_BUCKET: &str = ".minio.sys";
+const REGULAR_USER: &str = "minio-user";
+const SERVICE_ACCOUNT: &str = "minio-service-account";
+
+async fn seed_legacy_iam_object(env: &rustfs_test_utils::TestECStoreEnv, path: &str, value: &Value) {
+    env.put_object_bytes(
+        LEGACY_META_BUCKET,
+        path,
+        serde_json::to_vec(value).expect("legacy IAM object must serialize"),
+    )
+    .await;
+}
+
+fn assert_identity_fields(actual: &UserIdentity, expected: &Value) {
+    assert_eq!(
+        serde_json::to_value(actual).expect("loaded identity must serialize"),
+        *expected,
+        "migration must preserve every credential field except expiration",
+    );
+}
+
+async fn assert_identity_survives(
+    store: &ObjectStore,
+    identity_path: &str,
+    name: &str,
+    user_type: UserType,
+    source: &Value,
+    expected_policy: &Value,
+) {
+    let mut expected = source.clone();
+    expected["credentials"]["expiration"] = Value::Null;
+
+    let persisted: UserIdentity = store
+        .load_iam_config(identity_path)
+        .await
+        .expect("migrated identity must be persisted");
+    assert_identity_fields(&persisted, &expected);
+
+    for _ in 0..2 {
+        let actual = store
+            .load_user_identity(name, user_type)
+            .await
+            .expect("migrated permanent identity must remain loadable");
+        assert_identity_fields(&actual, &expected);
+    }
+
+    let mut mappings = HashMap::new();
+    store
+        .load_mapped_policy(name, user_type, false, &mut mappings)
+        .await
+        .expect("loading the identity must not delete its policy mapping");
+    let actual_policy = mappings.get(name).expect("migrated policy mapping must exist");
+    assert_eq!(
+        serde_json::to_value(actual_policy).expect("loaded policy mapping must serialize"),
+        *expected_policy,
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn minio_permanent_identities_survive_migration_and_repeated_iam_loads() {
+    if get_global_action_cred().is_none() {
+        init_global_action_credentials(Some("MINIOMIGRATIONROOT".to_string()), Some("minio-migration-root-secret".to_string()))
+            .expect("root credentials must initialize for JWT validation");
+    }
+
+    let temp_dir = tempfile::TempDir::with_prefix("rustfs_minio_iam_migration_").expect("temp directory must be created");
+    let env = rustfs_test_utils::TestECStoreEnv::builder()
+        .base_dir(temp_dir.path())
+        .init_bucket_metadata(false)
+        .build()
+        .await;
+    for disk_path in &env.disk_paths {
+        tokio::fs::create_dir_all(disk_path.join(LEGACY_META_BUCKET))
+            .await
+            .expect("legacy metadata volume must be created");
+    }
+
+    let regular_source = json!({
+        "version": 1,
+        "credentials": {
+            "accessKey": REGULAR_USER,
+            "secretKey": "regular-user-secret",
+            "sessionToken": "",
+            "expiration": "0001-01-01T00:00:00Z",
+            "status": "on",
+            "parentUser": "regular-parent",
+            "groups": ["engineering", "operations"],
+            "claims": {"tenant": "alpha"},
+            "name": "MinIO regular user",
+            "description": "migrated regular identity"
+        },
+        "updatedAt": "2025-03-07T12:00:00Z"
+    });
+    let service_claims = json!({"sa-policy": "inherited-policy", "tenant": "alpha"});
+    let service_secret = "service-account-secret";
+    let service_source = json!({
+        "version": 1,
+        "credentials": {
+            "accessKey": SERVICE_ACCOUNT,
+            "secretKey": service_secret,
+            "sessionToken": generate_jwt(&service_claims, service_secret).expect("service-account JWT must be generated"),
+            "expiration": "1970-01-01T00:00:00Z",
+            "status": "on",
+            "parentUser": REGULAR_USER,
+            "groups": ["service-accounts"],
+            "claims": service_claims,
+            "name": "MinIO service account",
+            "description": "migrated service identity"
+        },
+        "updatedAt": "2025-03-07T12:00:00Z"
+    });
+    let regular_policy_source = json!({"version": 1, "policy": "readwrite", "updatedAt": "2025-03-07T12:00:00Z"});
+    let service_policy_source = json!({"version": 1, "policy": "readonly", "updatedAt": "2025-03-07T12:00:00Z"});
+
+    let regular_identity_path = format!("{}{REGULAR_USER}/identity.json", IAM_CONFIG_USERS_PREFIX.as_str());
+    let service_identity_path = format!("{}{SERVICE_ACCOUNT}/identity.json", IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX.as_str());
+
+    seed_legacy_iam_object(&env, &regular_identity_path, &regular_source).await;
+    seed_legacy_iam_object(&env, &service_identity_path, &service_source).await;
+    seed_legacy_iam_object(
+        &env,
+        &format!("{}{REGULAR_USER}.json", IAM_CONFIG_POLICY_DB_USERS_PREFIX.as_str()),
+        &regular_policy_source,
+    )
+    .await;
+    seed_legacy_iam_object(
+        &env,
+        &format!("{}{SERVICE_ACCOUNT}.json", IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX.as_str()),
+        &service_policy_source,
+    )
+    .await;
+
+    try_migrate_iam_config(env.ecstore.clone(), None).await;
+
+    let store = ObjectStore::new(env.ecstore);
+    assert_identity_survives(
+        &store,
+        &regular_identity_path,
+        REGULAR_USER,
+        UserType::Reg,
+        &regular_source,
+        &regular_policy_source,
+    )
+    .await;
+    assert_identity_survives(
+        &store,
+        &service_identity_path,
+        SERVICE_ACCOUNT,
+        UserType::Svc,
+        &service_source,
+        &service_policy_source,
+    )
+    .await;
+}

crates/io-metrics/src/internode_metrics.rs

@@ -211,6 +211,146 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
 
 static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
 
+#[cfg(not(test))]
+struct InternodeServerMetricHandles {
+    sent_bytes: metrics::Counter,
+    recv_bytes: metrics::Counter,
+    outgoing_requests: metrics::Counter,
+    incoming_requests: metrics::Counter,
+    errors: metrics::Counter,
+}
+
+#[cfg(not(test))]
+impl InternodeServerMetricHandles {
+    fn new(server: &'static str) -> Self {
+        Self {
+            sent_bytes: counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => server),
+            recv_bytes: counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => server),
+            outgoing_requests: counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => server),
+            incoming_requests: counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => server),
+            errors: counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => server),
+        }
+    }
+}
+
+#[cfg(not(test))]
+static INTERNODE_SERVER_METRIC_HANDLES: LazyLock<InternodeServerMetricHandles> =
+    LazyLock::new(|| InternodeServerMetricHandles::new(current_server_label()));
+
+#[cfg(not(test))]
+struct GrpcReadVersionMetricHandles {
+    sent_bytes: metrics::Counter,
+    recv_bytes: metrics::Counter,
+    outgoing_requests: metrics::Counter,
+    incoming_requests: metrics::Counter,
+    errors: metrics::Counter,
+    duration: metrics::Histogram,
+    request_encode: metrics::Histogram,
+    request_decode: metrics::Histogram,
+    disk_read: metrics::Histogram,
+    response_json_encode: metrics::Histogram,
+    response_msgpack_encode: metrics::Histogram,
+    rpc_roundtrip: metrics::Histogram,
+    response_decode: metrics::Histogram,
+}
+
+#[cfg(not(test))]
+impl GrpcReadVersionMetricHandles {
+    fn new(server: &'static str) -> Self {
+        Self {
+            sent_bytes: counter!(
+                INTERNODE_OPERATION_SENT_BYTES_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            recv_bytes: counter!(
+                INTERNODE_OPERATION_RECV_BYTES_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            outgoing_requests: counter!(
+                INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            incoming_requests: counter!(
+                INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            errors: counter!(
+                INTERNODE_OPERATION_ERRORS_TOTAL,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            duration: metrics::histogram!(
+                INTERNODE_OPERATION_DURATION_MS,
+                SERVER_LABEL => server,
+                OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+                BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
+            ),
+            request_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE),
+            request_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE),
+            disk_read: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_DISK_READ),
+            response_json_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE),
+            response_msgpack_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE),
+            rpc_roundtrip: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
+            response_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE),
+        }
+    }
+
+    fn stage_duration(server: &'static str, stage: &'static str) -> metrics::Histogram {
+        metrics::histogram!(
+            INTERNODE_OPERATION_STAGE_DURATION_MS,
+            SERVER_LABEL => server,
+            OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
+            BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC,
+            STAGE_LABEL => stage
+        )
+    }
+
+    fn stage_duration_for(&self, stage: &'static str) -> Option<&metrics::Histogram> {
+        match stage {
+            INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE => Some(&self.request_encode),
+            INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE => Some(&self.request_decode),
+            INTERNODE_STAGE_READ_VERSION_DISK_READ => Some(&self.disk_read),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE => Some(&self.response_json_encode),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE => Some(&self.response_msgpack_encode),
+            INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP => Some(&self.rpc_roundtrip),
+            INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE => Some(&self.response_decode),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(not(test))]
+static GRPC_READ_VERSION_METRIC_HANDLES: LazyLock<GrpcReadVersionMetricHandles> =
+    LazyLock::new(|| GrpcReadVersionMetricHandles::new(current_server_label()));
+
+#[cfg(not(test))]
+fn server_metric_handles_if_ready() -> Option<&'static InternodeServerMetricHandles> {
+    STABLE_SERVER_LABEL.get()?;
+    Some(&INTERNODE_SERVER_METRIC_HANDLES)
+}
+
+#[cfg(not(test))]
+fn grpc_read_version_metric_handles_if_ready(
+    operation: &'static str,
+    backend: &'static str,
+) -> Option<&'static GrpcReadVersionMetricHandles> {
+    STABLE_SERVER_LABEL.get()?;
+    if operation == INTERNODE_OPERATION_GRPC_READ_VERSION && backend == INTERNODE_TRANSPORT_BACKEND_GRPC {
+        Some(&GRPC_READ_VERSION_METRIC_HANDLES)
+    } else {
+        None
+    }
+}
+
 /// Injects the stable server label (node name or address) stamped on
 /// internode metrics. The runtime calls this when the local node name is
 /// published (see ecstore's `set_local_node_name`); the first write wins.
@@ -284,6 +424,11 @@ impl InternodeMetrics {
             return;
         }
         self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.sent_bytes.increment(bytes);
+            return;
+        }
         counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
     }
 
@@ -298,6 +443,11 @@ impl InternodeMetrics {
         if bytes == 0 {
             return;
         }
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.sent_bytes.increment(bytes);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_SENT_BYTES_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -313,6 +463,11 @@ impl InternodeMetrics {
             return;
         }
         self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.recv_bytes.increment(bytes);
+            return;
+        }
         counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
     }
 
@@ -327,6 +482,11 @@ impl InternodeMetrics {
         if bytes == 0 {
             return;
         }
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.recv_bytes.increment(bytes);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_RECV_BYTES_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -338,6 +498,11 @@ impl InternodeMetrics {
 
     pub fn record_outgoing_request(&self) {
         self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.outgoing_requests.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -347,6 +512,11 @@ impl InternodeMetrics {
 
     pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_outgoing_request();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.outgoing_requests.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -358,6 +528,11 @@ impl InternodeMetrics {
 
     pub fn record_incoming_request(&self) {
         self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.incoming_requests.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -367,6 +542,11 @@ impl InternodeMetrics {
 
     pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_incoming_request();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.incoming_requests.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -378,6 +558,11 @@ impl InternodeMetrics {
 
     pub fn record_error(&self) {
         self.errors_total.fetch_add(1, Ordering::Relaxed);
+        #[cfg(not(test))]
+        if let Some(handles) = server_metric_handles_if_ready() {
+            handles.errors.increment(1);
+            return;
+        }
         counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => current_server_label()).increment(1);
     }
 
@@ -387,6 +572,11 @@ impl InternodeMetrics {
 
     pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_error();
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.errors.increment(1);
+            return;
+        }
         counter!(
             INTERNODE_OPERATION_ERRORS_TOTAL,
             SERVER_LABEL => current_server_label(),
@@ -398,6 +588,11 @@ impl InternodeMetrics {
 
     pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
         let duration_ms = duration.as_secs_f64() * 1000.0;
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
+            handles.duration.record(duration_ms);
+            return;
+        }
         metrics::histogram!(
             INTERNODE_OPERATION_DURATION_MS,
             SERVER_LABEL => current_server_label(),
@@ -415,6 +610,13 @@ impl InternodeMetrics {
         duration: Duration,
     ) {
         let duration_ms = duration.as_secs_f64() * 1000.0;
+        #[cfg(not(test))]
+        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend)
+            && let Some(histogram) = handles.stage_duration_for(stage)
+        {
+            histogram.record(duration_ms);
+            return;
+        }
         metrics::histogram!(
             INTERNODE_OPERATION_STAGE_DURATION_MS,
             SERVER_LABEL => current_server_label(),

crates/io-metrics/src/lib.rs

@@ -121,6 +121,16 @@ pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_ba
 pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync";
 pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall";
 
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED: &str = "disabled";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS: &str = "le_250ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS: &str = "le_500ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS: &str = "le_1000ms";
+pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS: &str = "gt_1000ms";
+
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED: &str = "acquired";
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN: &str = "timeout_slowdown";
+pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR: &str = "lock_error";
+
 pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial";
 pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel";
 pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER: &str = "leader";
@@ -2060,6 +2070,14 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
     }
 }
 
+#[inline(always)]
+pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
+    if !put_stage_metrics_enabled() {
+        return;
+    }
+    counter!("rustfs_s3_put_object_commit_namespace_lock_admission_total", "budget" => budget, "outcome" => outcome).increment(1);
+}
+
 #[inline(always)]
 fn put_stage_count_value(value: usize) -> f64 {
     match u32::try_from(value) {
@@ -3204,6 +3222,83 @@ mod tests {
         assert!(stages.iter().all(|stage| recorded.contains(*stage)));
     }
 
+    #[test]
+    fn put_commit_lock_admission_labels_are_static_and_gated() {
+        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        let budgets = [
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
+            PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
+        ];
+        let outcomes = [
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
+        ];
+        assert_eq!(budgets.iter().copied().collect::<HashSet<_>>().len(), budgets.len());
+        assert_eq!(outcomes.iter().copied().collect::<HashSet<_>>().len(), outcomes.len());
+        assert!(budgets.iter().chain(outcomes.iter()).all(|label| {
+            !label.contains('/')
+                && !label.contains('{')
+                && !label.contains('}')
+                && !label.contains(' ')
+                && label
+                    .chars()
+                    .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_')
+        }));
+
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            set_put_stage_metrics_enabled(false);
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            );
+
+            set_put_stage_metrics_enabled(true);
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
+            );
+            record_put_object_commit_lock_admission(
+                PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
+                PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
+            );
+            set_put_stage_metrics_enabled(false);
+        });
+
+        let rows = snapshotter.snapshot().into_vec();
+        assert_eq!(
+            counter_total(&rows, "rustfs_s3_put_object_commit_namespace_lock_admission_total"),
+            Some(2)
+        );
+        let label_sets = rows
+            .iter()
+            .filter(|(composite, _, _, _)| {
+                composite.kind() == MetricKind::Counter
+                    && composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
+            })
+            .map(|(composite, _, _, _)| {
+                composite
+                    .key()
+                    .labels()
+                    .map(|label| (label.key().to_string(), label.value().to_string()))
+                    .collect::<HashSet<_>>()
+            })
+            .collect::<Vec<_>>();
+        assert!(label_sets.contains(&HashSet::from([
+            ("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS.to_string()),
+            ("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN.to_string(),),
+        ])));
+        assert!(label_sets.contains(&HashSet::from([
+            ("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS.to_string()),
+            ("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED.to_string()),
+        ])));
+    }
+
     #[test]
     fn put_rename_code_level_metrics_are_static_and_gated() {
         let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());

crates/io-metrics/tests/internode_cached_handles.rs

@@ -0,0 +1,216 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::with_local_recorder;
+use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+    INTERNODE_TRANSPORT_BACKEND_GRPC, InternodeMetrics, set_internode_server_label,
+};
+use std::time::Duration;
+
+type MetricRow = (
+    metrics_util::CompositeKey,
+    Option<metrics::Unit>,
+    Option<metrics::SharedString>,
+    DebugValue,
+);
+
+const SERVER_LABEL: &str = "server";
+const OPERATION_LABEL: &str = "operation";
+const BACKEND_LABEL: &str = "backend";
+const STAGE_LABEL: &str = "stage";
+const SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_sent_bytes_total";
+const RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_recv_bytes_total";
+const REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_requests_outgoing_total";
+const REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_requests_incoming_total";
+const ERRORS_TOTAL: &str = "rustfs_system_network_internode_errors_total";
+const OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
+const OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
+const OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
+const OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
+const OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
+const OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
+const OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
+
+#[test]
+fn cached_grpc_read_version_metric_handles_preserve_labels_and_values() {
+    set_internode_server_label("cached-grpc-read-version-test");
+
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    let metrics = InternodeMetrics::default();
+
+    with_local_recorder(&recorder, || {
+        metrics.record_sent_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            17,
+        );
+        metrics.record_recv_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            23,
+        );
+        metrics.record_outgoing_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_incoming_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+        metrics.record_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            Duration::from_micros(250),
+        );
+        metrics.record_stage_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+            Duration::from_micros(125),
+        );
+        metrics.record_stage_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            INTERNODE_STAGE_READ_VERSION_DISK_READ,
+            Duration::from_micros(75),
+        );
+    });
+
+    let rows = snapshotter.snapshot().into_vec();
+    assert_counter(&rows, SENT_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 17);
+    assert_counter(&rows, RECV_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 23);
+    assert_counter(&rows, REQUESTS_OUTGOING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(&rows, REQUESTS_INCOMING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(&rows, ERRORS_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
+    assert_counter(
+        &rows,
+        OPERATION_SENT_BYTES_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        17,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_RECV_BYTES_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        23,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_REQUESTS_OUTGOING_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_REQUESTS_INCOMING_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_counter(
+        &rows,
+        OPERATION_ERRORS_TOTAL,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        1,
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+        ],
+        &[0.25],
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_STAGE_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+            (STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
+        ],
+        &[0.125],
+    );
+    assert_histogram(
+        &rows,
+        OPERATION_STAGE_DURATION_MS,
+        &[
+            (SERVER_LABEL, "cached-grpc-read-version-test"),
+            (OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
+            (BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
+            (STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_DISK_READ),
+        ],
+        &[0.075],
+    );
+}
+
+fn assert_counter(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: u64) {
+    match metric_value(rows, name, labels) {
+        DebugValue::Counter(value) => assert_eq!(*value, expected),
+        other => panic!("{name} should be a counter, got {other:?}"),
+    }
+}
+
+fn assert_histogram(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: &[f64]) {
+    match metric_value(rows, name, labels) {
+        DebugValue::Histogram(samples) => {
+            let actual: Vec<_> = samples.iter().map(|sample| sample.0).collect();
+            assert_eq!(actual, expected);
+        }
+        other => panic!("{name} should be a histogram, got {other:?}"),
+    }
+}
+
+fn metric_value<'a>(rows: &'a [MetricRow], name: &str, labels: &[(&str, &str)]) -> &'a DebugValue {
+    let mut matches = rows.iter().filter(|(composite, _, _, _)| {
+        composite.key().name() == name
+            && labels.iter().all(|(key, value)| {
+                composite
+                    .key()
+                    .labels()
+                    .any(|label| label.key() == *key && label.value() == *value)
+            })
+    });
+    let Some((_, _, _, value)) = matches.next() else {
+        panic!("{name} with labels {labels:?} was not recorded; rows={rows:?}");
+    };
+    assert!(matches.next().is_none(), "{name} with labels {labels:?} must be unique; rows={rows:?}");
+    value
+}

crates/lifecycle/src/evaluator.rs

@@ -66,9 +66,10 @@ impl Evaluator {
     }
 
     /// IsObjectLocked checks if it is appropriate to remove an
-    /// object according to its persisted object-lock metadata.
+    /// object according to its persisted object-lock metadata and the bucket
+    /// default retention.
     pub fn is_object_locked(&self, obj: &ObjectOpts) -> bool {
-        object_lock::is_object_locked_by_metadata(&obj.user_defined, obj.delete_marker)
+        object_lock::is_object_locked(&obj.user_defined, obj.delete_marker, self.lock_retention.as_deref(), obj.mod_time)
     }
 
     /// eval will return a lifecycle event for each object in objs for a given time.
@@ -198,8 +199,9 @@ mod tests {
 
     use rustfs_common::metrics::IlmAction;
     use s3s::dto::{
-        BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, ObjectLockConfiguration,
+        BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
-        ObjectLockEnabled, Transition, TransitionStorageClass,
+        NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
+        Transition, TransitionStorageClass,
     };
     use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
     use time::OffsetDateTime;
@@ -300,6 +302,40 @@ mod tests {
         })
     }
 
+    fn lock_enabled_with_default_retention(days: i32) -> Arc<ObjectLockConfiguration> {
+        Arc::new(ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: Some(days),
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: None,
+                }),
+            }),
+        })
+    }
+
+    fn noncurrent_expiration_lifecycle() -> Arc<BucketLifecycleConfiguration> {
+        Arc::new(BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("expire-noncurrent".to_string()),
+                noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: None,
+                }),
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        })
+    }
+
     fn object_opts(replication_status: ReplicationStatusType, version_purge_status: VersionPurgeStatusType) -> ObjectOpts {
         ObjectOpts {
             name: "logs/object".to_string(),
@@ -459,6 +495,52 @@ mod tests {
         assert_eq!(events[0].action, IlmAction::NoneAction);
     }
 
+    #[tokio::test]
+    async fn evaluator_skips_noncurrent_expiration_during_default_retention() {
+        let evaluator =
+            Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(30)));
+        let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
+        let noncurrent = ObjectOpts {
+            name: "logs/object".to_string(),
+            mod_time: Some(successor_time - time::Duration::days(1)),
+            successor_mod_time: Some(successor_time),
+            version_id: Some(Uuid::new_v4()),
+            is_latest: false,
+            num_versions: 1,
+            ..Default::default()
+        };
+
+        let events = evaluator
+            .eval(&[noncurrent])
+            .await
+            .expect("lifecycle evaluation should succeed");
+
+        assert_eq!(events[0].action, IlmAction::NoneAction);
+    }
+
+    #[tokio::test]
+    async fn evaluator_allows_noncurrent_expiration_after_default_retention() {
+        let evaluator =
+            Evaluator::new(noncurrent_expiration_lifecycle()).with_lock_retention(Some(lock_enabled_with_default_retention(1)));
+        let successor_time = OffsetDateTime::now_utc() - time::Duration::days(2);
+        let noncurrent = ObjectOpts {
+            name: "logs/object".to_string(),
+            mod_time: Some(successor_time - time::Duration::days(1)),
+            successor_mod_time: Some(successor_time),
+            version_id: Some(Uuid::new_v4()),
+            is_latest: false,
+            num_versions: 1,
+            ..Default::default()
+        };
+
+        let events = evaluator
+            .eval(&[noncurrent])
+            .await
+            .expect("lifecycle evaluation should succeed");
+
+        assert_eq!(events[0].action, IlmAction::DeleteVersionAction);
+    }
+
     #[tokio::test]
     async fn evaluator_skips_transition_while_replication_pending() {
         let evaluator = Evaluator::new(latest_transition_lifecycle());

crates/lifecycle/src/object_lock.rs

@@ -14,7 +14,7 @@
 
 use std::collections::HashMap;
 
-use s3s::dto::ObjectLockRetentionMode;
+use s3s::dto::{ObjectLockConfiguration, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use time::{OffsetDateTime, format_description};
 
@@ -43,6 +43,90 @@ pub fn is_object_locked_by_metadata(user_defined: &HashMap<String, String>, is_d
         .is_some_and(|retain_until| retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp())
 }
 
+/// Check persisted object-lock metadata and the bucket default retention.
+///
+/// A configured default retention with missing or malformed input is treated
+/// as locked so a lifecycle worker cannot turn incomplete metadata into an
+/// unsafe delete.
+pub fn is_object_locked(
+    user_defined: &HashMap<String, String>,
+    is_delete_marker: bool,
+    config: Option<&ObjectLockConfiguration>,
+    mod_time: Option<OffsetDateTime>,
+) -> bool {
+    if is_delete_marker {
+        return false;
+    }
+    if is_object_locked_by_metadata(user_defined, false) {
+        return true;
+    }
+    if has_explicit_lock_metadata(user_defined) {
+        return !explicit_lock_metadata_is_well_formed(user_defined);
+    }
+
+    let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
+        return false;
+    };
+    let Some(mode) = default_retention.mode.as_ref() else {
+        return true;
+    };
+    if !is_retention_mode(mode.as_str()) {
+        return true;
+    }
+
+    let Some(mod_time) = mod_time else {
+        return true;
+    };
+    let Some(retain_until) = default_retention_until(mod_time, default_retention) else {
+        return true;
+    };
+
+    retain_until.unix_timestamp() > OffsetDateTime::now_utc().unix_timestamp()
+}
+
+fn has_explicit_lock_metadata(user_defined: &HashMap<String, String>) -> bool {
+    user_defined.contains_key(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
+        || user_defined.contains_key(X_AMZ_OBJECT_LOCK_MODE.as_str())
+        || user_defined.contains_key(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str())
+}
+
+fn explicit_lock_metadata_is_well_formed(user_defined: &HashMap<String, String>) -> bool {
+    if user_defined
+        .get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str())
+        .is_some_and(|value| !value.eq_ignore_ascii_case("ON") && !value.eq_ignore_ascii_case("OFF"))
+    {
+        return false;
+    }
+
+    match (
+        user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()),
+        user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()),
+    ) {
+        (None, None) => true,
+        (Some(mode), Some(retain_until)) => {
+            is_retention_mode(mode)
+                && OffsetDateTime::parse(retain_until, &format_description::well_known::Iso8601::DEFAULT).is_ok()
+        }
+        _ => false,
+    }
+}
+
+fn default_retention_until(mod_time: OffsetDateTime, retention: &s3s::dto::DefaultRetention) -> Option<OffsetDateTime> {
+    match (retention.days, retention.years) {
+        (Some(days), None) if days > 0 => Some(mod_time.saturating_add(time::Duration::days(i64::from(days)))),
+        (None, Some(years)) if years > 0 => add_years(mod_time, years),
+        _ => None,
+    }
+}
+
+fn add_years(mod_time: OffsetDateTime, years: i32) -> Option<OffsetDateTime> {
+    let target_year = mod_time.year().checked_add(years)?;
+    mod_time
+        .replace_year(target_year)
+        .or_else(|_| mod_time.replace_day(28).and_then(|date| date.replace_year(target_year)))
+        .ok()
+}
+
 fn is_retention_mode(mode: &str) -> bool {
     mode.eq_ignore_ascii_case(ObjectLockRetentionMode::COMPLIANCE)
         || mode.eq_ignore_ascii_case(ObjectLockRetentionMode::GOVERNANCE)
@@ -52,6 +136,9 @@ fn is_retention_mode(mode: &str) -> bool {
 mod tests {
     use super::*;
 
+    use s3s::dto::{DefaultRetention, ObjectLockEnabled, ObjectLockRule};
+    use time::Duration;
+
     #[test]
     fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
         let mut user_defined = HashMap::new();
@@ -60,4 +147,120 @@ mod tests {
         assert!(is_object_locked_by_metadata(&user_defined, false));
         assert!(!is_object_locked_by_metadata(&user_defined, true));
     }
+
+    fn default_retention_config(days: i32) -> ObjectLockConfiguration {
+        ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: Some(days),
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: None,
+                }),
+            }),
+        }
+    }
+
+    #[test]
+    fn default_retention_blocks_lifecycle_delete_until_expired() {
+        let config = default_retention_config(30);
+        let created = OffsetDateTime::now_utc() - Duration::days(1);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn expired_default_retention_allows_lifecycle_delete() {
+        let config = default_retention_config(1);
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(!is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn missing_mod_time_blocks_default_retention_delete() {
+        let config = default_retention_config(30);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), None));
+    }
+
+    #[test]
+    fn zero_default_retention_days_fail_closed() {
+        let config = default_retention_config(0);
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn zero_default_retention_years_fail_closed() {
+        let config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: None,
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
+                    years: Some(0),
+                }),
+            }),
+        };
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn default_retention_years_block_lifecycle_delete_until_expired() {
+        let config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            rule: Some(ObjectLockRule {
+                default_retention: Some(DefaultRetention {
+                    days: None,
+                    mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
+                    years: Some(1),
+                }),
+            }),
+        };
+        let created = OffsetDateTime::now_utc() - Duration::days(1);
+
+        assert!(is_object_locked(&HashMap::new(), false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn expired_explicit_retention_does_not_reapply_default_retention() {
+        let config = default_retention_config(30);
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            (OffsetDateTime::now_utc() - Duration::days(1))
+                .format(&format_description::well_known::Iso8601::DEFAULT)
+                .expect("expired retention date should format"),
+        );
+
+        assert!(!is_object_locked(&user_defined, false, Some(&config), Some(OffsetDateTime::now_utc())));
+    }
+
+    #[test]
+    fn malformed_explicit_retention_fails_closed() {
+        let config = default_retention_config(1);
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        let created = OffsetDateTime::now_utc() - Duration::days(2);
+
+        assert!(is_object_locked(&user_defined, false, Some(&config), Some(created)));
+    }
+
+    #[test]
+    fn delete_markers_are_not_locked_by_default_retention() {
+        let config = default_retention_config(30);
+
+        assert!(!is_object_locked(&HashMap::new(), true, Some(&config), None));
+    }
 }

crates/obs/src/metrics/collectors/ilm.rs

@@ -54,12 +54,37 @@ pub(crate) struct IlmActionTaskStats {
     pub(crate) value: u64,
 }
 
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmQueueTaskStats {
+    pub(crate) action: String,
+    pub(crate) state: String,
+    pub(crate) value: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmTaskEventStats {
+    pub(crate) action: String,
+    pub(crate) result: String,
+    pub(crate) value: u64,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct IlmBackpressureStats {
+    pub(crate) action: String,
+    pub(crate) reason: String,
+    pub(crate) value: u64,
+}
+
 /// ILM statistics with runtime-local node identity and bounded action/state details.
 #[derive(Debug, Clone, Default)]
 pub(crate) struct IlmRuntimeStats {
     pub(crate) server: String,
     pub(crate) stats: IlmStats,
     pub(crate) action_tasks: Vec<IlmActionTaskStats>,
+    pub(crate) queue_tasks: Vec<IlmQueueTaskStats>,
+    pub(crate) task_events: Vec<IlmTaskEventStats>,
+    pub(crate) backpressure: Vec<IlmBackpressureStats>,
+    pub(crate) versions_scanned: u64,
 }
 
 fn is_live_action_task_state(state: &str) -> bool {
@@ -112,6 +137,30 @@ pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<Promet
             }),
     );
 
+    metrics.extend(stats.queue_tasks.iter().map(|task| {
+        PrometheusMetric::from_descriptor(&ILM_TASKS_MD, task.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, task.action.clone())
+            .with_label_owned(QUEUE_STATE_LABEL, task.state.clone())
+    }));
+    metrics.extend(stats.task_events.iter().map(|event| {
+        PrometheusMetric::from_descriptor(&ILM_TASK_EVENTS_MD, event.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, event.action.clone())
+            .with_label_owned(RESULT_LABEL, event.result.clone())
+    }));
+    metrics.extend(stats.backpressure.iter().map(|event| {
+        PrometheusMetric::from_descriptor(&ILM_QUEUE_BACKPRESSURE_MD, event.value as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(ACTION_LABEL, event.action.clone())
+            .with_label_owned(REASON_LABEL, event.reason.clone())
+    }));
+    metrics.push(
+        PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_BY_SERVER_MD, stats.versions_scanned as f64)
+            .with_label_owned(SERVER_LABEL, stats.server.clone())
+            .with_label_owned(SOURCE_LABEL, "lifecycle".to_string()),
+    );
+
     metrics
 }
 
@@ -135,6 +184,22 @@ mod tests {
         let runtime_stats = IlmRuntimeStats {
             server: "node1:9000".to_string(),
             stats,
+            queue_tasks: vec![IlmQueueTaskStats {
+                action: "transition".to_string(),
+                state: "pending".to_string(),
+                value: 8,
+            }],
+            task_events: vec![IlmTaskEventStats {
+                action: "transition".to_string(),
+                result: "completed".to_string(),
+                value: 7,
+            }],
+            backpressure: vec![IlmBackpressureStats {
+                action: "transition".to_string(),
+                reason: "queue_full".to_string(),
+                value: 2,
+            }],
+            versions_scanned: 1000000,
             action_tasks: vec![
                 IlmActionTaskStats {
                     action: "expiry".to_string(),
@@ -156,7 +221,7 @@ mod tests {
 
         let metrics = collect_ilm_runtime_metrics(&runtime_stats);
 
-        assert_eq!(metrics.len(), 11);
+        assert_eq!(metrics.len(), 15);
 
         let pending = metrics.iter().find(|m| m.value == 100.0);
         assert!(pending.is_some());
@@ -178,6 +243,44 @@ mod tests {
         });
         assert!(transition_timeout.is_none());
 
+        let transition_queue = metrics.iter().find(|m| {
+            m.name == ILM_TASKS_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == QUEUE_STATE_LABEL && value.as_ref() == "pending")
+        });
+        assert_eq!(transition_queue.map(|metric| metric.value), Some(8.0));
+
+        let completed = metrics.iter().find(|m| {
+            m.name == ILM_TASK_EVENTS_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == RESULT_LABEL && value.as_ref() == "completed")
+        });
+        assert_eq!(completed.map(|metric| metric.value), Some(7.0));
+
+        let backpressure = metrics.iter().find(|m| {
+            m.name == ILM_QUEUE_BACKPRESSURE_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == REASON_LABEL && value.as_ref() == "queue_full")
+        });
+        assert_eq!(backpressure.map(|metric| metric.value), Some(2.0));
+
+        let version_detail = metrics.iter().find(|m| {
+            m.name == ILM_VERSIONS_SCANNED_BY_SERVER_MD.get_full_metric_name()
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
+                && m.labels
+                    .iter()
+                    .any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
+        });
+        assert_eq!(version_detail.map(|metric| metric.value), Some(1000000.0));
+
         let transition_active = metrics.iter().find(|m| {
             m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
                 && m.labels

crates/obs/src/metrics/collectors/mod.rs

@@ -59,9 +59,12 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
 pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
 pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
 pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
-pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
+pub(crate) use ilm::{
+    IlmActionTaskStats, IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmTaskEventStats, collect_ilm_runtime_metrics,
+};
 pub use ilm::{IlmStats, collect_ilm_metrics};
 pub use node::{DiskStats, collect_node_metrics};
+pub(crate) use notification::collect_notification_runtime_metrics;
 pub use notification::{NotificationStats, collect_notification_metrics};
 pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
 pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};

crates/obs/src/metrics/collectors/notification.rs

@@ -19,9 +19,12 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::cluster_notification::{
-    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
+    NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD, NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD,
-    NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD,
+    NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_ERRORS_TOTAL_MD,
+    NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_SENT_TOTAL_MD,
+    NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD, NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD, SERVER,
 };
+use std::borrow::Cow;
 
 /// Notification statistics.
 #[derive(Debug, Clone, Default)]
@@ -49,6 +52,30 @@ pub fn collect_notification_metrics(stats: &NotificationStats) -> Vec<Prometheus
     ]
 }
 
+/// Collects the legacy aggregate metrics and node-local runtime siblings.
+pub(crate) fn collect_notification_runtime_metrics(stats: &NotificationStats, server: &str) -> Vec<PrometheusMetric> {
+    let mut metrics = collect_notification_metrics(stats);
+    if server.is_empty() {
+        return metrics;
+    }
+
+    let server_label: Cow<'static, str> = Cow::Owned(server.to_string());
+    metrics.extend([
+        PrometheusMetric::from_descriptor(
+            &NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD,
+            stats.current_send_in_progress as f64,
+        )
+        .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD, stats.events_errors_total as f64)
+            .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD, stats.events_sent_total as f64)
+            .with_label(SERVER, server_label.clone()),
+        PrometheusMetric::from_descriptor(&NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD, stats.events_skipped_total as f64)
+            .with_label(SERVER, server_label),
+    ]);
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -86,4 +113,32 @@ mod tests {
             assert!(metric.labels.is_empty());
         }
     }
+
+    #[test]
+    fn runtime_metrics_keep_aggregate_and_add_server_siblings() {
+        let stats = NotificationStats {
+            current_send_in_progress: 5,
+            events_errors_total: 10,
+            events_sent_total: 100,
+            events_skipped_total: 2,
+        };
+
+        let metrics = collect_notification_runtime_metrics(&stats, "node1:9000");
+        assert_eq!(metrics.len(), 8);
+        assert_eq!(metrics.iter().filter(|metric| metric.labels.is_empty()).count(), 4);
+        assert_eq!(metrics.iter().filter(|metric| metric.labels.len() == 1).count(), 4);
+        assert!(metrics.iter().filter(|metric| metric.labels.len() == 1).all(|metric| {
+            metric
+                .labels
+                .iter()
+                .any(|(name, value)| *name == SERVER && value == "node1:9000")
+        }));
+    }
+
+    #[test]
+    fn runtime_metrics_do_not_publish_empty_server_series() {
+        let metrics = collect_notification_runtime_metrics(&NotificationStats::default(), "");
+        assert_eq!(metrics.len(), 4);
+        assert!(metrics.iter().all(|metric| metric.labels.is_empty()));
+    }
 }

crates/obs/src/metrics/collectors/scanner.rs

@@ -184,6 +184,15 @@ pub struct ScannerBucketDriveResultStats {
     pub count: u64,
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct ScannerActiveBucketDriveStats {
+    pub source: String,
+    pub bucket: String,
+    pub drive: String,
+    pub count: u64,
+    pub age_seconds: u64,
+}
+
 /// Scanner statistics with runtime-local node identity and bounded source/result details.
 #[derive(Debug, Clone, Default)]
 pub(crate) struct ScannerRuntimeStats {
@@ -195,6 +204,7 @@ pub(crate) struct ScannerRuntimeStats {
     pub(crate) bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
     pub(crate) current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
     pub(crate) last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
+    pub(crate) active_bucket_drive_scans: Vec<ScannerActiveBucketDriveStats>,
 }
 
 /// Collects scanner metrics from the given stats.
@@ -452,6 +462,23 @@ fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&S
             &runtime.last_cycle_bucket_drive_results,
             Some("last"),
         );
+        for active in &runtime.active_bucket_drive_scans {
+            let labels = |metric: PrometheusMetric| {
+                metric
+                    .with_label_owned(SERVER_LABEL, runtime.server.clone())
+                    .with_label_owned(SOURCE_LABEL, active.source.clone())
+                    .with_label_owned(BUCKET_LABEL, active.bucket.clone())
+                    .with_label_owned(DRIVE_LABEL, active.drive.clone())
+            };
+            metrics.push(labels(PrometheusMetric::from_descriptor(
+                &SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD,
+                active.count as f64,
+            )));
+            metrics.push(labels(PrometheusMetric::from_descriptor(
+                &SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD,
+                active.age_seconds as f64,
+            )));
+        }
     }
 
     metrics
@@ -566,6 +593,13 @@ mod tests {
                 result: "error".to_string(),
                 count: 2,
             }],
+            active_bucket_drive_scans: vec![ScannerActiveBucketDriveStats {
+                source: "usage".to_string(),
+                bucket: "photos".to_string(),
+                drive: "/data1".to_string(),
+                count: 2,
+                age_seconds: 7,
+            }],
             stats: ScannerStats {
                 bucket_scans_finished: 100,
                 bucket_scans_started: 100,
@@ -642,7 +676,7 @@ mod tests {
         let metrics = collect_scanner_runtime_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 90);
+        assert_eq!(metrics.len(), 92);
 
         let objects = metrics.iter().find(|m| m.value == 1000000.0);
         assert!(objects.is_some());
@@ -656,6 +690,35 @@ mod tests {
         assert_eq!(active_paths.map(|m| m.value), Some(4.0));
         assert_eq!(active_paths.map(|m| m.labels.len()), Some(0));
 
+        let active_bucket_drive = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name())
+            .expect("active bucket-drive metric");
+        assert_eq!(active_bucket_drive.value, 2.0);
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == SOURCE_LABEL && value == "usage")
+        );
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == BUCKET_LABEL && value == "photos")
+        );
+        assert!(
+            active_bucket_drive
+                .labels
+                .iter()
+                .any(|(name, value)| *name == DRIVE_LABEL && value == "/data1")
+        );
+        let active_age = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD.get_full_metric_name())
+            .expect("active bucket-drive age metric");
+        assert_eq!(active_age.value, 7.0);
+
         let bucket_drive_result = metrics
             .iter()
             .find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name());

crates/obs/src/metrics/collectors/system_drive.rs

@@ -60,6 +60,10 @@ pub struct DriveDetailedStats {
     pub api_latency_micros: Option<u64>,
     /// Health status (1=healthy, 0=unhealthy)
     pub health: u8,
+    /// Total successful write operations when backed by a real disk metric.
+    pub writes_total: Option<u64>,
+    /// Total successful delete operations when backed by a real disk metric.
+    pub deletes_total: Option<u64>,
     /// Reads per second when backed by a real iostat sample
     pub reads_per_sec: Option<f64>,
     /// Kilobytes read per second when backed by a real iostat sample
@@ -282,6 +286,12 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail
         if let Some(value) = stat.stats.perc_util {
             push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, value, server_label, drive_label);
         }
+        if let Some(value) = stat.stats.writes_total {
+            push_drive_metric(&mut metrics, &DRIVE_WRITES_TOTAL_MD, value as f64, server_label, drive_label);
+        }
+        if let Some(value) = stat.stats.deletes_total {
+            push_drive_metric(&mut metrics, &DRIVE_DELETES_TOTAL_MD, value as f64, server_label, drive_label);
+        }
         if let Some(labels) = &topology_labels {
             if let Some(disk_id) = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty()) {
                 metrics.push(
@@ -449,6 +459,8 @@ mod tests {
                 waiting_io: Some(3),
                 api_latency_micros: Some(1500),
                 health: 1,
+                writes_total: Some(11),
+                deletes_total: Some(4),
                 reads_per_sec: Some(100.0),
                 reads_kb_per_sec: Some(1024.0),
                 reads_await: Some(5.5),
@@ -462,7 +474,7 @@ mod tests {
         let metrics = collect_drive_runtime_detailed_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 34);
+        assert_eq!(metrics.len(), 36);
 
         // Verify total bytes metric
         let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
@@ -503,6 +515,8 @@ mod tests {
                 API_LABEL,
             ],
         );
+        assert_metric_label_keys(&metrics, &DRIVE_WRITES_TOTAL_MD, 11.0, &[SERVER_LABEL, DRIVE_LABEL]);
+        assert_metric_label_keys(&metrics, &DRIVE_DELETES_TOTAL_MD, 4.0, &[SERVER_LABEL, DRIVE_LABEL]);
     }
 
     #[test]
@@ -524,6 +538,8 @@ mod tests {
             waiting_io: None,
             api_latency_micros: None,
             health: 1,
+            writes_total: None,
+            deletes_total: None,
             reads_per_sec: None,
             reads_kb_per_sec: None,
             reads_await: None,

crates/obs/src/metrics/scheduler.rs

@@ -62,7 +62,7 @@ use crate::metrics::collectors::{
     collect_memory_metrics,
     collect_network_metrics,
     collect_node_metrics,
-    collect_notification_metrics,
+    collect_notification_runtime_metrics,
     collect_notification_target_runtime_metrics,
     collect_process_attributes,
     collect_process_cpu_metrics,
@@ -120,12 +120,13 @@ use crate::metrics::schema::notification_target::{
 };
 use crate::metrics::schema::scanner::{
     BUCKET_LABEL as SCANNER_BUCKET_LABEL, CYCLE_SCOPE_LABEL as SCANNER_CYCLE_SCOPE_LABEL, DRIVE_LABEL as SCANNER_DRIVE_LABEL,
-    RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
+    RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD, SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD,
+    SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, SOURCE_LABEL as SCANNER_SOURCE_LABEL,
 };
 use crate::metrics::schema::system_drive::{
-    API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_HEALING_MD,
+    API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_DELETES_TOTAL_MD,
-    DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD, DRIVE_SCANNING_MD,
+    DRIVE_HEALING_MD, DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD,
-    POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL,
+    DRIVE_SCANNING_MD, DRIVE_WRITES_TOTAL_MD, POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL,
 };
 use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
 use crate::metrics::stats_collector::{
@@ -303,15 +304,33 @@ type AuditTargetKey = (String, String); // (server, target_id)
 type NotificationLegacyTargetKey = (String, String); // (target_id, target_type)
 type NotificationTargetKey = (String, String, String); // (server, target_id, target_type)
 type DriveTopologyKey = (String, String, String, String, String); // (server, drive, pool, set, drive_index)
+type DriveBasicKey = (String, String); // (server, drive)
 type DriveTopologyApiKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, api)
 type DriveInfoKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, disk_id)
 type ScannerCycleBucketDriveResultKey = (String, String, String, String, String); // (server, cycle_scope, bucket, drive, result)
 type ScannerBucketDriveResultKey = (String, String, String, String); // (server, bucket, drive, result)
+type ScannerActiveBucketDriveKey = (String, String, String, String); // (server, source, bucket, drive)
 
 fn drive_info_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveInfoKey> {
     stats.iter().filter_map(drive_info_key).collect()
 }
 
+fn drive_basic_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveBasicKey> {
+    stats
+        .iter()
+        .map(|stat| (stat.stats.server.clone(), stat.stats.drive.clone()))
+        .collect()
+}
+
+fn retire_drive_basic_metric_series(key: &DriveBasicKey) -> usize {
+    let labels = [
+        (SERVER_LABEL, Cow::Owned(key.0.clone())),
+        (DRIVE_LABEL, Cow::Owned(key.1.clone())),
+    ];
+    retire_metric_series(&DRIVE_WRITES_TOTAL_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&DRIVE_DELETES_TOTAL_MD.get_full_metric_name(), &labels)
+}
+
 fn drive_topology_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet<DriveTopologyKey> {
     stats.iter().filter_map(drive_topology_key).collect()
 }
@@ -469,6 +488,25 @@ fn retire_scanner_bucket_drive_result_metric_series(key: &ScannerBucketDriveResu
     retire_metric_series(&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name(), &labels)
 }
 
+fn scanner_active_bucket_drive_live_keys(stats: &ScannerRuntimeStats) -> HashSet<ScannerActiveBucketDriveKey> {
+    stats
+        .active_bucket_drive_scans
+        .iter()
+        .map(|active| (stats.server.clone(), active.source.clone(), active.bucket.clone(), active.drive.clone()))
+        .collect()
+}
+
+fn retire_scanner_active_bucket_drive_metric_series(key: &ScannerActiveBucketDriveKey) -> usize {
+    let labels = [
+        (SERVER_LABEL, Cow::Owned(key.0.clone())),
+        (SCANNER_SOURCE_LABEL, Cow::Owned(key.1.clone())),
+        (SCANNER_BUCKET_LABEL, Cow::Owned(key.2.clone())),
+        (SCANNER_DRIVE_LABEL, Cow::Owned(key.3.clone())),
+    ];
+    retire_metric_series(&SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD.get_full_metric_name(), &labels)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
 pub struct MetricsRuntimeCollectorHealthSnapshot {
     pub healthy_collectors: u8,
@@ -1841,6 +1879,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
     let token_clone = token.clone();
     tokio::spawn(async move {
         let mut interval = metrics_interval(node_interval, Duration::ZERO);
+        let mut prev_drive_basic_keys: HashSet<DriveBasicKey> = HashSet::new();
         let mut prev_drive_info_keys: HashSet<DriveInfoKey> = HashSet::new();
         let mut prev_drive_topology_keys: HashSet<DriveTopologyKey> = HashSet::new();
         let mut prev_drive_topology_api_keys: HashSet<DriveTopologyApiKey> = HashSet::new();
@@ -1851,6 +1890,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                     run_metrics_collector_tick(health, MetricsCollectorTaskId::NodeDiskStats, "node_disk_stats", async {
                         let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_runtime_stats().await;
                         let current_drive_info_keys = drive_info_live_keys(&drive_stats);
+                        let current_drive_basic_keys = drive_basic_live_keys(&drive_stats);
                         let current_drive_topology_keys = drive_topology_live_keys(&drive_stats);
                         let current_drive_topology_api_keys = drive_topology_api_live_keys(&drive_stats);
                         let retire_drive_info_keys = if has_seen_drive_info_snapshot {
@@ -1858,6 +1898,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                         } else {
                             Vec::new()
                         };
+                        let retire_drive_basic_keys = if has_seen_drive_info_snapshot {
+                            prev_drive_basic_keys.difference(&current_drive_basic_keys).cloned().collect::<Vec<_>>()
+                        } else {
+                            Vec::new()
+                        };
                         let retire_drive_topology_keys = if has_seen_drive_info_snapshot {
                             prev_drive_topology_keys.difference(&current_drive_topology_keys).cloned().collect::<Vec<_>>()
                         } else {
@@ -1872,6 +1917,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             Vec::new()
                         };
                         prev_drive_info_keys = current_drive_info_keys;
+                        prev_drive_basic_keys = current_drive_basic_keys;
                         prev_drive_topology_keys = current_drive_topology_keys;
                         prev_drive_topology_api_keys = current_drive_topology_api_keys;
                         has_seen_drive_info_snapshot = true;
@@ -1882,6 +1928,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                         for key in retire_drive_info_keys {
                             let _ = retire_drive_info_metric_series(&key);
                         }
+                        for key in retire_drive_basic_keys {
+                            let _ = retire_drive_basic_metric_series(&key);
+                        }
                         for key in retire_drive_topology_keys {
                             let _ = retire_drive_topology_metric_series(&key);
                         }
@@ -2106,14 +2155,14 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                 _ = interval.tick() => {
                     run_metrics_collector_tick(health, MetricsCollectorTaskId::NotificationStats, "notification_stats", async {
                         let snapshot = notification_metrics_snapshot();
-                        let mut metrics = collect_notification_metrics(&NotificationStats {
+                        let server = current_local_node_identity();
+                        let mut metrics = collect_notification_runtime_metrics(&NotificationStats {
                             current_send_in_progress: snapshot.current_send_in_progress,
                             events_errors_total: snapshot.events_errors_total,
                             events_sent_total: snapshot.events_sent_total,
                             events_skipped_total: snapshot.events_skipped_total,
-                        });
+                        }, &server);
 
-                        let server = current_local_node_identity();
                         let target_stats = notification_target_metrics().await
                             .into_iter()
                             .map(|snapshot| NotificationTargetRuntimeStats {
@@ -2173,6 +2222,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         let mut has_seen_scanner_snapshot = false;
         let mut prev_scanner_cycle_bucket_drive_result_keys: HashSet<ScannerCycleBucketDriveResultKey> = HashSet::new();
         let mut prev_scanner_bucket_drive_result_keys: HashSet<ScannerBucketDriveResultKey> = HashSet::new();
+        let mut prev_scanner_active_bucket_drive_keys: HashSet<ScannerActiveBucketDriveKey> = HashSet::new();
         loop {
             tokio::select! {
                 _ = interval.tick() => {
@@ -2189,9 +2239,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
 
                             let mut retire_scanner_cycle_bucket_drive_result_keys = Vec::new();
                             let mut retire_scanner_bucket_drive_result_keys = Vec::new();
+                            let mut retire_scanner_active_bucket_drive_keys = Vec::new();
                             if let Some(stats) = collect_scanner_runtime_metric_stats().await {
                                 let current_cycle_keys = scanner_cycle_bucket_drive_result_live_keys(&stats);
                                 let current_keys = scanner_bucket_drive_result_live_keys(&stats);
+                                let current_active_keys = scanner_active_bucket_drive_live_keys(&stats);
                                 if has_seen_scanner_snapshot {
                                     retire_scanner_cycle_bucket_drive_result_keys = prev_scanner_cycle_bucket_drive_result_keys
                                         .difference(&current_cycle_keys)
@@ -2201,9 +2253,14 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                                         .difference(&current_keys)
                                         .cloned()
                                         .collect();
+                                    retire_scanner_active_bucket_drive_keys = prev_scanner_active_bucket_drive_keys
+                                        .difference(&current_active_keys)
+                                        .cloned()
+                                        .collect();
                                 }
                                 prev_scanner_cycle_bucket_drive_result_keys = current_cycle_keys;
                                 prev_scanner_bucket_drive_result_keys = current_keys;
+                                prev_scanner_active_bucket_drive_keys = current_active_keys;
                                 has_seen_scanner_snapshot = true;
                                 metrics.extend(collect_scanner_runtime_metrics(&stats));
                             }
@@ -2217,6 +2274,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             for key in retire_scanner_bucket_drive_result_keys {
                                 let _ = retire_scanner_bucket_drive_result_metric_series(&key);
                             }
+                            for key in retire_scanner_active_bucket_drive_keys {
+                                let _ = retire_scanner_active_bucket_drive_metric_series(&key);
+                            }
                         },
                     ).await;
                 }
@@ -2495,6 +2555,7 @@ fn collect_system_monitoring_metrics(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::metrics::collectors::scanner::ScannerActiveBucketDriveStats;
     use std::collections::{HashMap, HashSet};
     use std::time::Duration;
     use tokio::time::Instant;
@@ -2723,6 +2784,30 @@ mod tests {
         assert!(current.contains(&("server-a".to_string(), "logs".to_string(), "/data1".to_string(), "success".to_string(),)));
     }
 
+    #[test]
+    fn scanner_active_bucket_drive_keys_detect_completed_scans() {
+        let previous = scanner_active_bucket_drive_live_keys(&ScannerRuntimeStats {
+            server: "server-a".to_string(),
+            active_bucket_drive_scans: vec![ScannerActiveBucketDriveStats {
+                source: "usage".to_string(),
+                bucket: "photos".to_string(),
+                drive: "/data1".to_string(),
+                count: 1,
+                age_seconds: 3,
+            }],
+            ..Default::default()
+        });
+        let current = scanner_active_bucket_drive_live_keys(&ScannerRuntimeStats {
+            server: "server-a".to_string(),
+            ..Default::default()
+        });
+        assert!(
+            previous
+                .difference(&current)
+                .any(|key| key == &("server-a".to_string(), "usage".to_string(), "photos".to_string(), "/data1".to_string()))
+        );
+    }
+
     #[test]
     fn replication_proxy_bucket_keys_detect_removed_buckets() {
         let previous = repl_proxy_bucket_live_keys(&[BucketReplicationRuntimeStats {

crates/obs/src/metrics/schema/cluster_notification.rs

@@ -15,6 +15,10 @@
 use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
 use std::sync::LazyLock;
 
+pub const SERVER: &str = "server";
+
+const SERVER_LABELS: [&str; 1] = [SERVER];
+
 pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::NotificationCurrentSendInProgress,
@@ -24,6 +28,15 @@ pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: LazyLock<MetricDescriptor>
     )
 });
 
+pub static NOTIFICATION_CURRENT_SEND_IN_PROGRESS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("current_send_in_progress_by_server".to_string()),
+        "Number of concurrent async Send calls active to all targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsErrorsTotal,
@@ -33,6 +46,15 @@ pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = Laz
     )
 });
 
+pub static NOTIFICATION_EVENTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_errors_total_by_server".to_string()),
+        "Events that failed to be sent to the targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsSentTotal,
@@ -42,6 +64,15 @@ pub static NOTIFICATION_EVENTS_SENT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyL
     )
 });
 
+pub static NOTIFICATION_EVENTS_SENT_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_sent_total_by_server".to_string()),
+        "Total number of events sent to the targets by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});
+
 pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::NotificationEventsSkippedTotal,
@@ -50,3 +81,12 @@ pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: LazyLock<MetricDescriptor> = La
         subsystems::NOTIFICATION,
     )
 });
+
+pub static NOTIFICATION_EVENTS_SKIPPED_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("events_skipped_total_by_server".to_string()),
+        "Notification dispatch attempts skipped before delivery by server",
+        &SERVER_LABELS,
+        subsystems::NOTIFICATION,
+    )
+});

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -371,6 +371,8 @@ pub enum MetricName {
     DriveWaitingIO,
     DriveAPILatencyMicros,
     DriveHealth,
+    DriveWritesTotal,
+    DriveDeletesTotal,
 
     DriveOfflineCount,
     DriveOnlineCount,
@@ -780,6 +782,8 @@ impl MetricName {
             Self::DriveWaitingIO => "waiting_io".to_string(),
             Self::DriveAPILatencyMicros => "api_latency_micros".to_string(),
             Self::DriveHealth => "health".to_string(),
+            Self::DriveWritesTotal => "writes_total".to_string(),
+            Self::DriveDeletesTotal => "deletes_total".to_string(),
 
             Self::DriveOfflineCount => "offline_count".to_string(),
             Self::DriveOnlineCount => "online_count".to_string(),

crates/obs/src/metrics/schema/ilm.rs

@@ -18,6 +18,10 @@ use std::sync::LazyLock;
 pub const SERVER_LABEL: &str = "server";
 pub const ACTION_LABEL: &str = "action";
 pub const STATE_LABEL: &str = "state";
+pub const QUEUE_STATE_LABEL: &str = "queue_state";
+pub const RESULT_LABEL: &str = "result";
+pub const REASON_LABEL: &str = "reason";
+pub const SOURCE_LABEL: &str = "source";
 
 pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
@@ -28,6 +32,33 @@ pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     )
 });
 
+pub static ILM_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("tasks".to_string()),
+        "Current ILM task counts by server, action, and queue state",
+        &[SERVER_LABEL, ACTION_LABEL, QUEUE_STATE_LABEL],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_TASK_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("task_events_total".to_string()),
+        "ILM task events by server, action, and result",
+        &[SERVER_LABEL, ACTION_LABEL, RESULT_LABEL],
+        subsystems::ILM,
+    )
+});
+
+pub static ILM_QUEUE_BACKPRESSURE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("queue_backpressure_total".to_string()),
+        "ILM queue backpressure events by server, action, and reason",
+        &[SERVER_LABEL, ACTION_LABEL, REASON_LABEL],
+        subsystems::ILM,
+    )
+});
+
 pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::IlmExpiryPendingTasks,
@@ -108,3 +139,12 @@ pub static ILM_VERSIONS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
         subsystems::ILM,
     )
 });
+
+pub static ILM_VERSIONS_SCANNED_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::Custom("versions_scanned_by_server".to_string()),
+        "ILM lifecycle-checked object versions by server and source",
+        &[SERVER_LABEL, SOURCE_LABEL],
+        subsystems::ILM,
+    )
+});

crates/obs/src/metrics/schema/scanner.rs

@@ -59,6 +59,24 @@ pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock<MetricDescriptor> = La
     )
 });
 
+pub static SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("active_bucket_drive_scans".to_string()),
+        "Current active scanner bucket-drive scans by server, source, bucket, and drive",
+        &[SERVER_LABEL, SOURCE_LABEL, BUCKET_LABEL, DRIVE_LABEL],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::Custom("active_bucket_drive_scan_age_seconds".to_string()),
+        "Age of the oldest active scanner bucket-drive scan by server, source, bucket, and drive",
+        &[SERVER_LABEL, SOURCE_LABEL, BUCKET_LABEL, DRIVE_LABEL],
+        subsystems::SCANNER,
+    )
+});
+
 pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::ScannerBucketScansFinished,

crates/obs/src/metrics/schema/system_drive.rs

@@ -259,6 +259,24 @@ pub static DRIVE_HEALTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     )
 });
 
+pub static DRIVE_WRITES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveWritesTotal,
+        "Total successful write operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
+pub static DRIVE_DELETES_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::DriveDeletesTotal,
+        "Total successful delete operations on a drive",
+        &ALL_DRIVE_LABELS[..],
+        subsystems::SYSTEM_DRIVE,
+    )
+});
+
 pub static DRIVE_OFFLINE_COUNT_MD: LazyLock<MetricDescriptor> =
     LazyLock::new(|| new_gauge_md(MetricName::DriveOfflineCount, "Count of offline drives", &[], subsystems::SYSTEM_DRIVE));
 

crates/obs/src/metrics/stats_collector.rs

@@ -18,15 +18,15 @@
 //! RustFS internal sources (storage layer, bucket monitor, system info)
 //! and convert them to the Stats structs used by collectors.
 
-use crate::metrics::collectors::scanner::{ScannerBucketDriveResultStats, ScannerSourceWorkStats};
+use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats};
 use crate::metrics::collectors::{
     ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats,
     BucketReplicationRuntimeStats, BucketReplicationStats, BucketReplicationTargetBacklogStats, BucketReplicationTargetFlowStats,
     BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
     ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats,
-    DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmRuntimeStats, IlmStats,
+    DriveRuntimeDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmActionTaskStats, IlmBackpressureStats,
-    MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerRuntimeStats,
+    IlmQueueTaskStats, IlmRuntimeStats, IlmStats, IlmTaskEventStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType,
-    ScannerStats,
+    ReplicationStats, ResourceStats, ScannerRuntimeStats, ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
@@ -38,7 +38,10 @@ use crate::metrics::{
 use crate::node_identity::current_local_node_identity;
 use jiff::Timestamp;
 use rustfs_common::heal_channel::HealScanMode;
-use rustfs_common::metrics::{ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, global_metrics};
+use rustfs_common::metrics::{
+    ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot,
+    global_metrics,
+};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
 use rustfs_io_metrics::{
     ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
@@ -334,7 +337,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
         return 0;
     }
 
-    u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
+    u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
 }
 
 fn scanner_scan_mode_code(scan_mode: &str) -> u64 {
@@ -835,6 +838,8 @@ pub(crate) async fn collect_disk_and_system_drive_runtime_stats()
                         drive_api_latency_micros(metrics.last_minute.values().map(|action| (action.count, action.acc_time)))
                     }),
                     health: if is_online { 1 } else { 0 },
+                    writes_total: disk.metrics.as_ref().map(|metrics| metrics.total_writes),
+                    deletes_total: disk.metrics.as_ref().map(|metrics| metrics.total_deletes),
                     reads_per_sec: None,
                     reads_kb_per_sec: None,
                     reads_await: None,
@@ -1275,6 +1280,110 @@ fn ilm_action_task_stats(ilm: &ObsIlmRuntimeSnapshot) -> Vec<IlmActionTaskStats>
     ]
 }
 
+fn ilm_queue_task_stats(metrics: &ScannerMetricsReport) -> Vec<IlmQueueTaskStats> {
+    let expiry = &metrics.lifecycle_expiry;
+    let transition = &metrics.lifecycle_transition;
+    vec![
+        IlmQueueTaskStats {
+            action: "expiry".to_string(),
+            state: "pending".to_string(),
+            value: expiry.current_queued,
+        },
+        IlmQueueTaskStats {
+            action: "expiry".to_string(),
+            state: "active".to_string(),
+            value: expiry.current_active,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "pending".to_string(),
+            value: transition.current_queued,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "active".to_string(),
+            value: transition.current_active,
+        },
+        IlmQueueTaskStats {
+            action: "transition".to_string(),
+            state: "compensation_running".to_string(),
+            value: transition.compensation_running,
+        },
+    ]
+}
+
+fn ilm_task_event_stats(metrics: &ScannerMetricsReport) -> Vec<IlmTaskEventStats> {
+    let expiry = &metrics.lifecycle_expiry;
+    let transition = &metrics.lifecycle_transition;
+    vec![
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "queued".to_string(),
+            value: expiry.scanner_queued,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "missed".to_string(),
+            value: expiry.scanner_missed,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "blocked".to_string(),
+            value: expiry.scanner_blocked,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "not_enqueued".to_string(),
+            value: expiry.scanner_not_enqueued,
+        },
+        IlmTaskEventStats {
+            action: "expiry".to_string(),
+            result: "failed".to_string(),
+            value: expiry.delete_failed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "queued".to_string(),
+            value: transition.scanner_queued,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "missed".to_string(),
+            value: transition.scanner_missed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "completed".to_string(),
+            value: transition.completed,
+        },
+        IlmTaskEventStats {
+            action: "transition".to_string(),
+            result: "failed".to_string(),
+            value: transition.failed,
+        },
+    ]
+}
+
+fn ilm_backpressure_stats(metrics: &ScannerMetricsReport) -> Vec<IlmBackpressureStats> {
+    vec![
+        IlmBackpressureStats {
+            action: "expiry".to_string(),
+            reason: "queue_missed".to_string(),
+            value: metrics.lifecycle_expiry.queue_missed,
+        },
+        IlmBackpressureStats {
+            action: "transition".to_string(),
+            reason: "queue_full".to_string(),
+            value: metrics.lifecycle_transition.queue_full,
+        },
+        IlmBackpressureStats {
+            action: "transition".to_string(),
+            reason: "send_timeout".to_string(),
+            value: metrics.lifecycle_transition.queue_send_timeout,
+        },
+    ]
+}
+
 /// Collect ILM metrics from the current lifecycle runtime state.
 pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
     collect_ilm_runtime_metric_stats().await.map(|stats| stats.stats)
@@ -1288,6 +1397,10 @@ pub(crate) async fn collect_ilm_runtime_metric_stats() -> Option<IlmRuntimeStats
     Some(IlmRuntimeStats {
         server: current_local_node_identity(),
         action_tasks: ilm_action_task_stats(&ilm),
+        queue_tasks: ilm_queue_task_stats(&metrics),
+        task_events: ilm_task_event_stats(&metrics),
+        backpressure: ilm_backpressure_stats(&metrics),
+        versions_scanned,
         stats: IlmStats {
             expiry_pending_tasks: ilm.expiry_pending_tasks,
             transition_active_tasks: ilm.transition_active_tasks,
@@ -1377,6 +1490,27 @@ fn scanner_bucket_drive_result_stats(results: &[ScannerBucketDriveResultSnapshot
     stats
 }
 
+fn scanner_active_bucket_drive_stats(results: &[ScannerActiveBucketDriveSnapshot]) -> Vec<ScannerActiveBucketDriveStats> {
+    let mut stats = results
+        .iter()
+        .filter(|result| !result.source.is_empty() && !result.bucket.is_empty() && !result.drive.is_empty() && result.count > 0)
+        .map(|result| ScannerActiveBucketDriveStats {
+            source: result.source.clone(),
+            bucket: result.bucket.clone(),
+            drive: result.drive.clone(),
+            count: result.count,
+            age_seconds: result.age_seconds,
+        })
+        .collect::<Vec<_>>();
+    stats.sort_by(|left, right| {
+        left.source
+            .cmp(&right.source)
+            .then_with(|| left.bucket.cmp(&right.bucket))
+            .then_with(|| left.drive.cmp(&right.drive))
+    });
+    stats
+}
+
 pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     collect_scanner_runtime_metric_stats().await.map(|stats| stats.stats)
 }
@@ -1418,6 +1552,7 @@ pub(crate) async fn collect_scanner_runtime_metric_stats() -> Option<ScannerRunt
             &runtime_details.current_cycle_bucket_drive_results,
         ),
         last_cycle_bucket_drive_results: scanner_bucket_drive_result_stats(&runtime_details.last_cycle_bucket_drive_results),
+        active_bucket_drive_scans: scanner_active_bucket_drive_stats(&runtime_details.active_bucket_drive_scans),
         stats: ScannerStats {
             bucket_scans_finished,
             bucket_scans_started,
@@ -1984,6 +2119,72 @@ mod tests {
         assert_eq!(scanner_lifecycle_checked_versions(&report), 37);
     }
 
+    #[test]
+    fn ilm_detail_stats_keep_expiry_and_transition_results_separate() {
+        let report = ScannerMetricsReport {
+            lifecycle_expiry: rustfs_common::metrics::ScannerLifecycleExpirySnapshot {
+                current_queued: 2,
+                current_active: 1,
+                scanner_queued: 10,
+                scanner_missed: 3,
+                delete_failed: 4,
+                ..Default::default()
+            },
+            lifecycle_transition: rustfs_common::metrics::ScannerLifecycleTransitionSnapshot {
+                current_queued: 5,
+                current_active: 6,
+                queue_full: 7,
+                queue_send_timeout: 8,
+                scanner_queued: 11,
+                completed: 12,
+                failed: 13,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let queues = ilm_queue_task_stats(&report);
+        assert!(
+            queues
+                .iter()
+                .any(|task| task.action == "expiry" && task.state == "pending" && task.value == 2)
+        );
+        assert!(
+            queues
+                .iter()
+                .any(|task| task.action == "transition" && task.state == "active" && task.value == 6)
+        );
+
+        let events = ilm_task_event_stats(&report);
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "expiry" && event.result == "failed" && event.value == 4)
+        );
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "transition" && event.result == "completed" && event.value == 12)
+        );
+        assert!(
+            events
+                .iter()
+                .any(|event| event.action == "transition" && event.result == "failed" && event.value == 13)
+        );
+
+        let backpressure = ilm_backpressure_stats(&report);
+        assert!(
+            backpressure
+                .iter()
+                .any(|event| event.action == "transition" && event.reason == "queue_full" && event.value == 7)
+        );
+        assert!(
+            backpressure
+                .iter()
+                .any(|event| event.action == "transition" && event.reason == "send_timeout" && event.value == 8)
+        );
+    }
+
     #[test]
     fn scanner_source_work_stats_sorts_and_skips_empty_source() {
         let stats = scanner_source_work_stats(&[

crates/policy/src/policy/function/date.rs

@@ -31,7 +31,7 @@ impl DateFunc {
                 return false;
             };
 
-            if !op(&inner.values.0, &rv) {
+            if !op(&rv, &inner.values.0) {
                 return false;
             }
         }
@@ -95,6 +95,7 @@ mod tests {
         key_name::KeyName::{self, *},
         key_name::S3KeyName::*,
     };
+    use std::collections::HashMap;
     use test_case::test_case;
     use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 
@@ -122,4 +123,16 @@ mod tests {
         assert_eq!(v, expect);
         Ok(())
     }
+
+    #[test]
+    fn evaluate_compares_request_date_to_policy_date() {
+        let function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
+        let later = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2099-01-01T00:00:00Z".to_string()])]);
+        let earlier = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2029-01-01T00:00:00Z".to_string()])]);
+
+        assert!(function.evaluate(OffsetDateTime::gt, &later));
+        assert!(!function.evaluate(OffsetDateTime::gt, &earlier));
+        assert!(function.evaluate(OffsetDateTime::lt, &earlier));
+        assert!(!function.evaluate(OffsetDateTime::lt, &later));
+    }
 }

crates/protocols/Cargo.toml

@@ -106,7 +106,7 @@ swift = [
     "dep:base64",
     "dep:async-compression",
 ]
-webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
+webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
 sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
 
 [dependencies]

crates/protocols/src/common/client/s3.rs

@@ -15,6 +15,9 @@
 use async_trait::async_trait;
 use s3s::dto::*;
 
+#[cfg(feature = "webdav")]
+use crate::common::session::SessionContext;
+
 #[async_trait]
 pub trait StorageBackend: Send + Sync {
     /// Error type for this storage backend
@@ -65,8 +68,24 @@ pub trait StorageBackend: Send + Sync {
         access_key: &str,
         secret_key: &str,
     ) -> Result<ListObjectsV2Output, Self::Error>;
-    /// List all buckets (requires authentication)
+    /// List all buckets (requires authentication).
     async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error>;
+    /// List buckets visible to the authenticated session.
+    ///
+    /// Backends that implement this must apply per-bucket authorization. The default denies the
+    /// request so existing backends cannot expose unfiltered bucket names.
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        _session_context: &SessionContext,
+        _request_headers: &http::HeaderMap,
+        _secure_transport: bool,
+    ) -> s3s::S3Result<ListBucketsOutput> {
+        Err(s3s::S3Error::with_message(
+            s3s::S3ErrorCode::AccessDenied,
+            "Session-aware bucket listing is not supported",
+        ))
+    }
     /// Create a new bucket
     async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error>;
     /// Delete a bucket (must be empty)

crates/protocols/src/common/dummy_storage.rs

@@ -30,6 +30,8 @@
 //! SessionContext type in common::session.
 
 use crate::common::client::s3::StorageBackend;
+#[cfg(feature = "webdav")]
+use crate::common::session::SessionContext;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures_util::stream::{self, StreamExt};
@@ -140,6 +142,8 @@ struct Inner {
     head_bucket: VecDeque<Result<HeadBucketOutput, DummyError>>,
     list_objects_v2: VecDeque<Result<ListObjectsV2Output, DummyError>>,
     list_buckets: VecDeque<Result<ListBucketsOutput, DummyError>>,
+    session_list_buckets: VecDeque<s3s::S3Result<ListBucketsOutput>>,
+    last_session_list_context: Option<(http::HeaderMap, bool)>,
     create_bucket: VecDeque<Result<CreateBucketOutput, DummyError>>,
     delete_bucket: VecDeque<Result<DeleteBucketOutput, DummyError>>,
     copy_object: VecDeque<Result<CopyObjectOutput, DummyError>>,
@@ -193,6 +197,8 @@ impl Inner {
             head_bucket: VecDeque::new(),
             list_objects_v2: VecDeque::new(),
             list_buckets: VecDeque::new(),
+            session_list_buckets: VecDeque::new(),
+            last_session_list_context: None,
             create_bucket: VecDeque::new(),
             delete_bucket: VecDeque::new(),
             copy_object: VecDeque::new(),
@@ -301,6 +307,31 @@ impl DummyBackend {
             .push_back(Ok(CreateBucketOutput::default()));
     }
 
+    /// Queue a legacy list_buckets response.
+    pub fn queue_list_buckets_ok(&self, output: ListBucketsOutput) {
+        self.inner.lock().expect("lock").list_buckets.push_back(Ok(output));
+    }
+
+    /// Queue a legacy list_buckets error.
+    pub fn queue_list_buckets_err(&self, error: DummyError) {
+        self.inner.lock().expect("lock").list_buckets.push_back(Err(error));
+    }
+
+    /// Queue a session-aware list_buckets response.
+    pub fn queue_session_list_buckets_ok(&self, output: ListBucketsOutput) {
+        self.inner.lock().expect("lock").session_list_buckets.push_back(Ok(output));
+    }
+
+    /// Queue a session-aware list_buckets error.
+    pub fn queue_session_list_buckets_err(&self, error: s3s::S3Error) {
+        self.inner.lock().expect("lock").session_list_buckets.push_back(Err(error));
+    }
+
+    /// Return the context from the last session-aware list_buckets request.
+    pub fn last_session_list_context(&self) -> Option<(http::HeaderMap, bool)> {
+        self.inner.lock().expect("lock").last_session_list_context.clone()
+    }
+
     /// Queue a put_object error. Used by the commit_write retry tests
     /// to script SlowDown / AccessDenied sequences against the
     /// rustfs_utils::retry::is_s3code_in_message_retryable predicate.
@@ -690,13 +721,29 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn list_buckets(&self, _ak: &str, _sk: &str) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
         match self.inner.lock().expect("lock").list_buckets.pop_front() {
             Some(r) => r,
             None => Ok(ListBucketsOutput::default()),
         }
     }
 
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        session_context: &SessionContext,
+        request_headers: &http::HeaderMap,
+        secure_transport: bool,
+    ) -> s3s::S3Result<ListBucketsOutput> {
+        let _ = session_context;
+        let mut inner = self.inner.lock().expect("lock");
+        inner.last_session_list_context = Some((request_headers.clone(), secure_transport));
+        inner
+            .session_list_buckets
+            .pop_front()
+            .unwrap_or_else(|| Ok(ListBucketsOutput::default()))
+    }
+
     async fn create_bucket(&self, _bucket: &str, _ak: &str, _sk: &str) -> Result<CreateBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").create_bucket.pop_front() {
             Some(r) => r,

crates/protocols/src/webdav/driver.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::common::client::s3::StorageBackend as S3StorageBackend;
-use crate::common::gateway::{S3Action, authorize_operation};
+use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation};
 use crate::common::session::SessionContext;
 use bytes::Bytes;
 use dav_server::davpath::DavPath;
@@ -24,6 +24,7 @@ use futures_util::{FutureExt, StreamExt, stream};
 use percent_encoding::percent_decode_str;
 use rustfs_utils::MaskedAccessKey;
 use rustfs_utils::path;
+use s3s::S3ErrorCode;
 use s3s::dto::*;
 use std::fmt::Debug;
 use std::io::SeekFrom;
@@ -457,6 +458,10 @@ where
     storage: S,
     /// Session context for authorization
     session_context: Arc<SessionContext>,
+    /// Policy-safe WebDAV request headers used by IAM conditions.
+    request_headers: Option<http::HeaderMap>,
+    /// Whether the WebDAV connection uses TLS.
+    secure_transport: bool,
 }
 
 enum ResolvedPath {
@@ -490,6 +495,8 @@ where
         Self {
             storage: self.storage.clone(),
             session_context: self.session_context.clone(),
+            request_headers: self.request_headers.clone(),
+            secure_transport: self.secure_transport,
         }
     }
 }
@@ -503,9 +510,18 @@ where
         Self {
             storage,
             session_context,
+            request_headers: None,
+            secure_transport: false,
         }
     }
 
+    /// Attach the request context used by IAM policy conditions.
+    pub fn with_request_context(mut self, request_headers: http::HeaderMap, secure_transport: bool) -> Self {
+        self.request_headers = Some(request_headers);
+        self.secure_transport = secure_transport;
+        self
+    }
+
     fn credentials(&self) -> (&str, &str) {
         (
             &self.session_context.principal.user_identity.credentials.access_key,
@@ -799,50 +815,41 @@ where
     /// List all buckets (for root path)
     async fn list_buckets(&self) -> FsResult<Vec<WebDavDirEntry>> {
         match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await {
-            Ok(_) => {}
+            Ok(()) => {
-            Err(_e) => {
+                let (access_key, secret_key) = self.credentials();
+                return match self.storage.list_buckets(access_key, secret_key).await {
+                    Ok(output) => Ok(Self::bucket_entries(output)),
+                    Err(error) => {
+                        error!(
+                            event = EVENT_WEBDAV_BUCKET_LIST_FAILED,
+                            component = LOG_COMPONENT_PROTOCOLS,
+                            subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                            error = %error,
+                            access_key = %MaskedAccessKey(access_key),
+                            "webdav bucket list failed"
+                        );
+                        Err(FsError::GeneralFailure)
+                    }
+                };
+            }
+            Err(AuthorizationError::AccessDenied) => {}
+            Err(AuthorizationError::IamUnavailable) => return Err(FsError::GeneralFailure),
+        }
+
+        let Some(request_headers) = self.request_headers.as_ref() else {
+            return Err(FsError::Forbidden);
+        };
+        let result = self
+            .storage
+            .list_buckets_for_session(&self.session_context, request_headers, self.secure_transport)
+            .await;
+
+        match result {
+            Ok(output) => Ok(Self::bucket_entries(output)),
+            Err(e) => {
+                if matches!(e.code(), S3ErrorCode::AccessDenied) {
                     return Err(FsError::Forbidden);
                 }
-        }
-
-        match self
-            .storage
-            .list_buckets(
-                &self.session_context.principal.user_identity.credentials.access_key,
-                &self.session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
-            Ok(output) => {
-                let mut entries = Vec::new();
-                if let Some(buckets) = output.buckets {
-                    for bucket in buckets {
-                        if let Some(ref bucket_name) = bucket.name {
-                            let modified = bucket
-                                .creation_date
-                                .map(|dt| {
-                                    let offset_dt: time::OffsetDateTime = dt.into();
-                                    SystemTime::from(offset_dt)
-                                })
-                                .unwrap_or_else(SystemTime::now);
-
-                            entries.push(WebDavDirEntry {
-                                name: bucket_name.clone(),
-                                metadata: WebDavMetaData {
-                                    size: 0,
-                                    modified,
-                                    created: modified,
-                                    is_dir: true,
-                                    etag: None,
-                                    content_type: None,
-                                },
-                            });
-                        }
-                    }
-                }
-                Ok(entries)
-            }
-            Err(e) => {
                 error!(
                     event = EVENT_WEBDAV_BUCKET_LIST_FAILED,
                     component = LOG_COMPONENT_PROTOCOLS,
@@ -856,6 +863,35 @@ where
         }
     }
 
+    fn bucket_entries(output: ListBucketsOutput) -> Vec<WebDavDirEntry> {
+        output
+            .buckets
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|bucket| {
+                let name = bucket.name?;
+                let modified = bucket
+                    .creation_date
+                    .map(|date| {
+                        let date: time::OffsetDateTime = date.into();
+                        SystemTime::from(date)
+                    })
+                    .unwrap_or_else(SystemTime::now);
+                Some(WebDavDirEntry {
+                    name,
+                    metadata: WebDavMetaData {
+                        size: 0,
+                        modified,
+                        created: modified,
+                        is_dir: true,
+                        etag: None,
+                        content_type: None,
+                    },
+                })
+            })
+            .collect()
+    }
+
     /// List objects in a bucket
     async fn list_objects(&self, bucket: &str, prefix: Option<&str>) -> FsResult<Vec<WebDavDirEntry>> {
         // Authorize the operation
@@ -1715,8 +1751,9 @@ where
 mod tests {
     use super::WebDavDriver;
     use crate::common::client::s3::StorageBackend as S3StorageBackend;
-    use crate::common::gateway::{S3Action, with_test_auth_override};
+    use crate::common::dummy_storage::DummyBackend;
-    use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
+    use crate::common::gateway::{S3Action, with_test_auth_override, with_test_iam_unavailable};
+    use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, test_session};
     use async_trait::async_trait;
     use bytes::Bytes;
     use dav_server::davpath::DavPath;
@@ -1906,6 +1943,134 @@ mod tests {
         WebDavDriver::new(DummyStorage, Arc::new(session_context))
     }
 
+    #[tokio::test]
+    async fn session_bucket_listing_does_not_require_global_list_permission() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_ok(ListBucketsOutput {
+            buckets: Some(vec![Bucket {
+                name: Some("allowed-bucket".to_string()),
+                ..Default::default()
+            }]),
+            ..Default::default()
+        });
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav))).with_request_context(
+            http::HeaderMap::from_iter([(http::header::USER_AGENT, http::HeaderValue::from_static("webdav-test"))]),
+            false,
+        );
+
+        let entries = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect("session-aware backend should own bucket filtering");
+
+        assert_eq!(entries.len(), 1);
+        assert_eq!(entries[0].name, "allowed-bucket");
+        let (headers, secure_transport) = storage
+            .last_session_list_context()
+            .expect("request context should be forwarded");
+        assert_eq!(headers.get("user-agent").expect("user agent"), "webdav-test");
+        assert!(!secure_transport);
+    }
+
+    #[tokio::test]
+    async fn list_buckets_maps_typed_access_denied_to_forbidden() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_err(s3s::S3Error::with_message(s3s::S3ErrorCode::AccessDenied, "policy denied"));
+        let driver = WebDavDriver::new(storage, Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("bucket listing should be denied");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
+    #[tokio::test]
+    async fn list_buckets_does_not_classify_error_text_as_access_denied() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_err(s3s::S3Error::with_message(
+            s3s::S3ErrorCode::InternalError,
+            "AccessDenied appears only in the message",
+        ));
+        let driver = WebDavDriver::new(storage, Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("bucket listing should fail");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+    }
+
+    #[tokio::test]
+    async fn global_list_permission_keeps_the_legacy_backend_path() {
+        let storage = DummyBackend::new();
+        storage.queue_list_buckets_ok(ListBucketsOutput {
+            buckets: Some(vec![Bucket {
+                name: Some("legacy-bucket".to_string()),
+                ..Default::default()
+            }]),
+            ..Default::default()
+        });
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)));
+
+        let entries = with_test_auth_override(|_, _, _| true, driver.list_buckets())
+            .await
+            .expect("globally authorized legacy backend should keep working");
+
+        assert_eq!(entries[0].name, "legacy-bucket");
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn legacy_bucket_list_error_is_a_general_failure() {
+        let storage = DummyBackend::new();
+        storage.queue_list_buckets_err(crate::common::dummy_storage::DummyError::Injected("backend failed".to_string()));
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)));
+
+        let error = with_test_auth_override(|_, _, _| true, driver.list_buckets())
+            .await
+            .expect_err("legacy backend error should fail the listing");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn iam_unavailable_does_not_enter_the_session_fallback() {
+        let storage = DummyBackend::new();
+        storage.queue_session_list_buckets_ok(ListBucketsOutput::default());
+        let driver = WebDavDriver::new(storage.clone(), Arc::new(test_session(Protocol::WebDav)))
+            .with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_iam_unavailable(driver.list_buckets())
+            .await
+            .expect_err("IAM outage must fail closed");
+
+        assert!(matches!(error, FsError::GeneralFailure));
+        assert!(storage.last_session_list_context().is_none());
+    }
+
+    #[tokio::test]
+    async fn missing_request_context_fails_closed() {
+        let error = with_test_auth_override(|_, _, _| false, driver().list_buckets())
+            .await
+            .expect_err("bucket listing should require the original request context");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
+    #[tokio::test]
+    async fn backend_without_session_listing_fails_closed() {
+        let driver = driver().with_request_context(http::HeaderMap::new(), false);
+
+        let error = with_test_auth_override(|_, _, _| false, driver.list_buckets())
+            .await
+            .expect_err("default session listing must deny the request");
+
+        assert!(matches!(error, FsError::Forbidden));
+    }
+
     #[derive(Default)]
     struct RecordingStorageState {
         objects: HashMap<(String, String), Vec<u8>>,

crates/protocols/src/webdav/server.rs

@@ -19,6 +19,8 @@ use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, is_tem
 use bytes::Bytes;
 use dav_server::DavHandler;
 use dav_server::fakels::FakeLs;
+use http::header::{AUTHORIZATION, REFERER, USER_AGENT};
+use http::{HeaderMap, HeaderValue};
 use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
 use hyper::body::Body as HttpBody;
 use hyper::server::conn::http1;
@@ -59,6 +61,20 @@ const EVENT_WEBDAV_CONNECTION_CAP_STATE: &str = "webdav_connection_cap_state";
 /// materialise a whole object in memory for every GET.
 type WebDavBody = Pin<Box<dyn HttpBody<Data = Bytes, Error = io::Error> + Send>>;
 
+fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
+    let mut policy_headers = HeaderMap::new();
+    for name in [USER_AGENT, REFERER] {
+        if let Some(value) = headers.get(&name) {
+            policy_headers.insert(name, value.clone());
+        }
+    }
+
+    let mut authorization = HeaderValue::from_static("Basic");
+    authorization.set_sensitive(true);
+    policy_headers.insert(AUTHORIZATION, authorization);
+    policy_headers
+}
+
 /// WebDAV server implementation
 pub struct WebDavServer<S>
 where
@@ -216,7 +232,7 @@ where
                                         match timeout(request_timeout, acceptor.accept(stream)).await {
                                             Ok(Ok(tls_stream)) => {
                                                 let io = TokioIo::new(tls_stream);
-                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, max_body_size, request_timeout).await {
+                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout).await {
                                                     debug!(
                                                         event = EVENT_WEBDAV_CONNECTION_STATE,
                                                         component = LOG_COMPONENT_PROTOCOLS,
@@ -254,7 +270,7 @@ where
                                         }
                                     } else {
                                         let io = TokioIo::new(stream);
-                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, max_body_size, request_timeout).await {
+                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout).await {
                                             debug!(
                                                 event = EVENT_WEBDAV_CONNECTION_STATE,
                                                 component = LOG_COMPONENT_PROTOCOLS,
@@ -313,6 +329,7 @@ where
         io: TokioIo<I>,
         storage: S,
         source_ip: IpAddr,
+        secure_transport: bool,
         max_body_size: u64,
         request_timeout: Duration,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
@@ -321,7 +338,7 @@ where
     {
         let service = service_fn(move |req: Request<hyper::body::Incoming>| {
             let storage = storage.clone();
-            async move { Self::handle_request(req, storage, source_ip, max_body_size, request_timeout).await }
+            async move { Self::handle_request(req, storage, source_ip, secure_transport, max_body_size, request_timeout).await }
         });
 
         // A peer that opens a connection and dribbles (or never finishes)
@@ -341,6 +358,7 @@ where
         req: Request<hyper::body::Incoming>,
         storage: S,
         source_ip: IpAddr,
+        secure_transport: bool,
         max_body_size: u64,
         request_timeout: Duration,
     ) -> Result<Response<WebDavBody>, Infallible> {
@@ -398,7 +416,8 @@ where
         };
 
         // Create WebDAV driver with session context
-        let driver = WebDavDriver::new(storage, Arc::new(session_context));
+        let driver = WebDavDriver::new(storage, Arc::new(session_context))
+            .with_request_context(policy_request_headers(req.headers()), secure_transport);
 
         // Build DAV handler with boxed filesystem
         let dav_handler = DavHandler::builder()
@@ -883,6 +902,30 @@ mod tests {
             .expect("build get request")
     }
 
+    #[test]
+    fn policy_headers_drop_credentials_and_s3_auth_spoofing() {
+        let mut headers = HeaderMap::new();
+        headers.insert(AUTHORIZATION, HeaderValue::from_static("Basic dXNlcjpwYXNzd29yZA=="));
+        headers.insert(USER_AGENT, HeaderValue::from_static("webdav-client"));
+        headers.insert(REFERER, HeaderValue::from_static("https://example.test/"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"));
+        headers.insert("x-amz-signature-age", HeaderValue::from_static("0"));
+
+        let policy_headers = policy_request_headers(&headers);
+
+        assert_eq!(policy_headers.get(AUTHORIZATION).expect("authorization marker"), "Basic");
+        assert!(
+            policy_headers
+                .get(AUTHORIZATION)
+                .expect("authorization marker")
+                .is_sensitive()
+        );
+        assert_eq!(policy_headers.get(USER_AGENT).expect("user agent"), "webdav-client");
+        assert_eq!(policy_headers.get(REFERER).expect("referer"), "https://example.test/");
+        assert!(!policy_headers.contains_key("x-amz-content-sha256"));
+        assert!(!policy_headers.contains_key("x-amz-signature-age"));
+    }
+
     /// R03-CAN-051 / R03-CAN-067 / R05-CAN-094: a chunked upload declares no
     /// Content-Length, so the limit has to hold on the bytes actually read.
     #[tokio::test]
@@ -959,6 +1002,7 @@ mod tests {
             TokioIo::new(server),
             StubStorage,
             TEST_IP,
+            false,
             1024,
             Duration::from_secs(30),
         ));

crates/s3select-query/src/lib.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![recursion_limit = "256"]
+
 pub mod data_source;
 pub mod dispatcher;
 pub mod execution;

crates/scanner/src/remote_scanner/stream.rs

@@ -22,7 +22,7 @@ use crate::scanner_io::{
 use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
 use crate::{
     DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo,
-    DataUsageScanPlanDigest, Disk, ScannerDiskExt as _, ScannerError, StorageError, resolve_scanner_object_store_handle,
+    DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle,
 };
 use hmac::{Hmac, KeyInit, Mac};
 use rustfs_common::heal_channel::HealScanMode;

crates/scanner/src/scanner.rs

@@ -1081,18 +1081,6 @@ async fn run_data_scanner_cycle(
         }
     };
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
-    let storeapi_clone = storeapi.clone();
-    let ctx_clone = ctx.clone();
-    let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
-        store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
-            ctx_clone,
-            storeapi_clone,
-            receiver,
-            Some(leader_epoch),
-            Some(usage_persist_baseline),
-        )
-        .await
-    }));
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
     let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
@@ -1107,9 +1095,38 @@ async fn run_data_scanner_cycle(
             scan_mode,
         )
         .await;
+    let publication_defer_reason = match &scan_result {
+        Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
+        Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+    };
     let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
-    let usage_persist_outcome = match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await
+    let usage_persist_outcome = match publication_defer_reason {
-    {
+        Some(reason) => {
+            drop(receiver);
+            DataUsagePersistOutcome::Deferred(reason)
+        }
+        None => {
+            // ScannerIO emits its complete or observational update only after
+            // all set workers finish. Persist after the final activity fence;
+            // this also avoids blocking the scanner on a denied publication.
+            let storeapi_clone = storeapi.clone();
+            let ctx_clone = ctx.clone();
+            let route_probe_store = storeapi.clone();
+            let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
+                store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+                    ctx_clone,
+                    storeapi_clone,
+                    receiver,
+                    Some(leader_epoch),
+                    Some(usage_persist_baseline),
+                    move || {
+                        let storeapi = route_probe_store.clone();
+                        async move { storeapi.scanner_data_usage_publication_blocked().await }
+                    },
+                )
+                .await
+            }));
+            match wait_for_data_usage_persist_task(ctx, &mut usage_persist_task, usage_persist_timeout).await {
                 DataUsagePersistTaskResult::Completed(outcome) => outcome,
                 DataUsagePersistTaskResult::JoinFailed(err) => {
                     error!(
@@ -1149,6 +1166,8 @@ async fn run_data_scanner_cycle(
                     );
                     DataUsagePersistOutcome::Failed
                 }
+            }
+        }
     };
     let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
 
@@ -1191,7 +1210,8 @@ async fn run_data_scanner_cycle(
         mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Failed;
     }
-    if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
+    match scanner_cycle_pre_commit_outcome(scan_cycle_result.required_cycle_floor(), &usage_persist_outcome) {
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => {
             warn!(
                 target: "rustfs::scanner",
                 event = EVENT_SCANNER_CYCLE_STATE,
@@ -1219,6 +1239,23 @@ async fn run_data_scanner_cycle(
                 ScannerCycleOutcome::Failed
             };
         }
+        Some(ScannerCyclePreCommitOutcome::Deferred(reason)) => {
+            info!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_CYCLE_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                cycle = cycle_info.current,
+                reason = reason.as_str(),
+                state = "deferred",
+                "Scanner cycle deferred before data usage publication"
+            );
+            emit_scan_cycle_deferred(cycle_start.elapsed());
+            mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
+            return ScannerCycleOutcome::Deferred(reason);
+        }
+        None => {}
+    }
     if usage_persist_outcome == DataUsagePersistOutcome::Failed {
         error!(
             target: "rustfs::scanner",
@@ -1804,14 +1841,13 @@ async fn run_data_scanner_with_maintenance_state(
             wait_plan.delay,
             activity_poll_interval,
             &mut scanner_activity_seen,
-            ScannerCycleObservedGenerations {
+            ScannerCycleObservedGenerations::for_wait(
-                // A non-converged cycle holds further activity notifications
+                &runtime_config,
-                // until its bounded retry timer to avoid an unbroken scan loop.
+                convergence_retry_interval,
-                dirty_usage: convergence_retry_interval.is_none().then_some(dirty_usage_generation_seen),
+                dirty_usage_generation_seen,
-                runtime_config: runtime_config_generation_seen,
+                runtime_config_generation_seen,
-                maintenance: maintenance_generation_before_wait,
+                maintenance_generation_before_wait,
-                defer_cluster_activity: convergence_retry_interval.is_some(),
+            ),
-            },
             || guard.is_lock_lost(),
             || probe_scanner_activity(storeapi.as_ref(), distributed),
         )
@@ -2001,6 +2037,56 @@ impl Drop for ScannerScanModeGuard {
     }
 }
 
+async fn final_data_usage_publication_defer_reason(
+    storeapi: &ECStore,
+    status: ScannerCycleStatus,
+) -> Option<ScannerCycleDeferReason> {
+    match status {
+        ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => {
+            if storeapi.scanner_data_usage_publication_blocked().await {
+                return Some(ScannerCycleDeferReason::DataMovement);
+            }
+            if status == ScannerCycleStatus::Complete {
+                let distributed = storeapi.setup_is_dist_erasure().await;
+                match probe_scanner_activity(storeapi, distributed).await {
+                    Ok(snapshot) if scanner_activity_allows_usage_publication(&snapshot) => None,
+                    Ok(_) => Some(ScannerCycleDeferReason::DataMovement),
+                    Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+                }
+            } else {
+                // A superseded cycle is explicitly observational and cannot
+                // replace the authoritative snapshot. It may still be
+                // persisted as a convergence baseline for the next cycle.
+                None
+            }
+        }
+        ScannerCycleStatus::Deferred(reason) => Some(reason),
+        // Incomplete cycles do not publish a usage snapshot. Keep the
+        // decision permissive so existing partial-cycle handling remains
+        // unchanged if a future scanner path emits a bookkeeping update.
+        ScannerCycleStatus::Incomplete => None,
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum ScannerCyclePreCommitOutcome {
+    RecoverCacheCycle(u64),
+    Deferred(ScannerCycleDeferReason),
+}
+
+fn scanner_cycle_pre_commit_outcome(
+    required_cycle_floor: Option<u64>,
+    usage_persist_outcome: &DataUsagePersistOutcome,
+) -> Option<ScannerCyclePreCommitOutcome> {
+    // Keep the publication barrier fail-closed: `.bloomcycle.bin` uses the
+    // same routed writer and its floor must remain pending while data movement
+    // hides the source pool.
+    match usage_persist_outcome {
+        DataUsagePersistOutcome::Deferred(reason) => Some(ScannerCyclePreCommitOutcome::Deferred(*reason)),
+        _ => required_cycle_floor.map(ScannerCyclePreCommitOutcome::RecoverCacheCycle),
+    }
+}
+
 fn scanner_cycle_completion_outcome(
     scan_status: ScannerCycleStatus,
     usage_persist_outcome: DataUsagePersistOutcome,
@@ -2008,6 +2094,7 @@ fn scanner_cycle_completion_outcome(
     has_failed_dirty_usage: bool,
 ) -> ScannerCycleOutcome {
     match (scan_status, usage_persist_outcome) {
+        (_, DataUsagePersistOutcome::Deferred(reason)) => ScannerCycleOutcome::Deferred(reason),
         (_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
         (ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
             if !has_dirty_usage && !has_failed_dirty_usage =>

crates/scanner/src/scanner/activity.rs

@@ -229,6 +229,27 @@ pub(super) struct ScannerCycleObservedGenerations {
     pub(super) defer_cluster_activity: bool,
 }
 
+impl ScannerCycleObservedGenerations {
+    pub(super) fn for_wait(
+        runtime_config: &ScannerRuntimeConfig,
+        convergence_retry_interval: Option<Duration>,
+        dirty_usage_generation_seen: u64,
+        runtime_config_generation: u64,
+        maintenance_generation: u64,
+    ) -> Self {
+        Self {
+            // An explicit cycle override is a duty-cycle policy; dirty usage
+            // wakes stay on the default adaptive path so the interval holds.
+            dirty_usage: (convergence_retry_interval.is_none()
+                && runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default)
+                .then_some(dirty_usage_generation_seen),
+            runtime_config: runtime_config_generation,
+            maintenance: maintenance_generation,
+            defer_cluster_activity: convergence_retry_interval.is_some(),
+        }
+    }
+}
+
 pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
 
 #[derive(Clone, Debug, PartialEq, Eq)]

crates/scanner/src/scanner/tests.rs

@@ -153,6 +153,7 @@ struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
+    object_not_found_put_number: Mutex<HashMap<String, usize>>,
     error_after_commit_put_number: Mutex<HashMap<String, usize>>,
     interleaving_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
     cancel_after_interleaving_puts: Mutex<HashMap<String, CancellationToken>>,
@@ -224,6 +225,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
         if self.fail_put_number.lock().await.get(&key) == Some(&put_count) {
             return Err(EcstoreError::other("injected put failure"));
         }
+        if self.object_not_found_put_number.lock().await.get(&key) == Some(&put_count) {
+            return Err(EcstoreError::ObjectNotFound(bucket.to_string(), object.to_string()));
+        }
 
         let interleaving_data = {
             let mut interleaving_puts = self.interleaving_puts.lock().await;
@@ -1431,6 +1435,170 @@ async fn test_store_data_usage_in_backend_preserves_newer_snapshot() {
     assert_eq!(outcome, DataUsagePersistOutcome::Current);
 }
 
+#[tokio::test]
+async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier() {
+    for (route_blocked, expected) in [
+        (true, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)),
+        (false, DataUsagePersistOutcome::Failed),
+    ] {
+        let store = Arc::new(MemoryConfigStore::default());
+        let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+        let baseline = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1);
+        let baseline_data = serde_json::to_vec(&baseline).expect("baseline usage snapshot should encode");
+        store.objects.lock().await.insert(key.clone(), baseline_data.clone());
+        store.revisions.lock().await.insert(key.clone(), 1);
+        store.object_not_found_put_number.lock().await.insert(key.clone(), 1);
+
+        let (sender, receiver) = mpsc::channel(1);
+        sender
+            .send(complete_usage_with_bucket_count(
+                Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
+                2,
+            ))
+            .await
+            .expect("new usage snapshot should enqueue");
+        drop(sender);
+        let probe_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let route_probe_calls = probe_calls.clone();
+
+        let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+            CancellationToken::new(),
+            store.clone(),
+            receiver,
+            None,
+            Some(DataUsagePersistBaseline {
+                data: Some(Bytes::from(baseline_data.clone())),
+                revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
+            }),
+            move || {
+                let probe_calls = route_probe_calls.clone();
+                async move {
+                    let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                    route_blocked && call > 1
+                }
+            },
+        )
+        .await;
+
+        assert_eq!(outcome, expected);
+        assert_eq!(
+            probe_calls.load(std::sync::atomic::Ordering::SeqCst),
+            3,
+            "ObjectNotFound must be followed by a fresh route-barrier probe"
+        );
+        assert_eq!(
+            store.objects.lock().await.get(&key),
+            Some(&baseline_data),
+            "a route failure must not replace the authoritative baseline"
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
+    for observational in [false, true] {
+        let store = Arc::new(MemoryConfigStore::default());
+        let target_path = if observational {
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
+        } else {
+            DATA_USAGE_OBJ_NAME_PATH.as_str()
+        };
+        let target_key = memory_config_key(RUSTFS_META_BUCKET, target_path);
+        let mut incoming = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
+        incoming.usage_snapshot_converged = Some(!observational);
+        let (sender, receiver) = mpsc::channel(1);
+        sender.send(incoming).await.expect("usage snapshot should enqueue");
+        drop(sender);
+
+        let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+            CancellationToken::new(),
+            store.clone(),
+            receiver,
+            None,
+            Some(DataUsagePersistBaseline {
+                data: None,
+                revision: DataUsageCacheRevision::Missing,
+            }),
+            || async { true },
+        )
+        .await;
+
+        assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+        assert!(!store.objects.lock().await.contains_key(&target_key));
+        assert_eq!(
+            store.put_counts.lock().await.get(&target_key),
+            None,
+            "the final pool-state fence must run before the first PUT"
+        );
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn test_usage_route_barrier_precedes_durable_reconciliation() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let snapshot = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
+    let snapshot_data = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
+    let (sender, receiver) = mpsc::channel(1);
+    sender.send(snapshot).await.expect("usage snapshot should enqueue");
+    drop(sender);
+
+    let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        CancellationToken::new(),
+        store.clone(),
+        receiver,
+        None,
+        Some(DataUsagePersistBaseline {
+            data: Some(Bytes::from(snapshot_data)),
+            revision: DataUsageCacheRevision::Etag("memory-1".to_string()),
+        }),
+        || async { true },
+    )
+    .await;
+
+    assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    assert_eq!(store.put_counts.lock().await.get(&key), None);
+}
+
+#[tokio::test]
+#[serial]
+async fn test_deferred_usage_save_keeps_last_real_save_metric() {
+    let metrics = global_metrics();
+    metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
+    let before = metrics.report().await.usage_freshness;
+
+    let store = Arc::new(MemoryConfigStore::default());
+    let (sender, receiver) = mpsc::channel(1);
+    sender
+        .send(complete_usage_with_bucket_count(
+            Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
+            1,
+        ))
+        .await
+        .expect("usage snapshot should enqueue");
+    drop(sender);
+
+    let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        CancellationToken::new(),
+        store,
+        receiver,
+        None,
+        Some(DataUsagePersistBaseline {
+            data: None,
+            revision: DataUsageCacheRevision::Missing,
+        }),
+        || async { true },
+    )
+    .await;
+
+    assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    let after = metrics.report().await.usage_freshness;
+    assert_eq!(after.last_usage_save_result, before.last_usage_save_result);
+    assert_eq!(after.last_usage_save_result_code, before.last_usage_save_result_code);
+    assert_eq!(after.last_usage_save_unix_secs, before.last_usage_save_unix_secs);
+}
+
 #[tokio::test]
 async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2325,6 +2493,15 @@ async fn test_store_data_usage_in_backend_reports_missing_snapshot() {
 
 #[test]
 fn test_scanner_cycle_completion_prioritizes_persist_failure() {
+    assert_eq!(
+        scanner_cycle_completion_outcome(
+            ScannerCycleStatus::Complete,
+            DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement),
+            true,
+            false,
+        ),
+        ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
+    );
     assert_eq!(
         scanner_cycle_completion_outcome(
             ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
@@ -2421,6 +2598,33 @@ fn test_scanner_cycle_completion_prioritizes_persist_failure() {
     );
 }
 
+#[test]
+fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
+    for reason in [
+        ScannerCycleDeferReason::DataMovement,
+        ScannerCycleDeferReason::ActivityBaselineUnavailable,
+    ] {
+        let deferred = DataUsagePersistOutcome::Deferred(reason);
+        assert_eq!(
+            scanner_cycle_pre_commit_outcome(Some(19), &deferred),
+            Some(ScannerCyclePreCommitOutcome::Deferred(reason)),
+            "a blocked publication must not persist the routed scanner cycle floor"
+        );
+        assert_eq!(
+            scanner_cycle_pre_commit_outcome(None, &deferred),
+            Some(ScannerCyclePreCommitOutcome::Deferred(reason))
+        );
+    }
+    assert_eq!(
+        scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Saved),
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
+    );
+    assert_eq!(
+        scanner_cycle_pre_commit_outcome(Some(19), &DataUsagePersistOutcome::Failed),
+        Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(19))
+    );
+}
+
 #[test]
 #[serial]
 fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
@@ -2448,6 +2652,23 @@ fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() {
     assert!(!crate::scanner_io::dirty_usage_buckets_pending());
 }
 
+#[test]
+#[serial]
+fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+    crate::scanner_io::record_dirty_usage_bucket("photos");
+    let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
+    let deferred = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot));
+
+    let (outcome, _, acknowledgements) =
+        finalize_scanner_cycle_result(deferred, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+
+    assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
+    assert!(acknowledgements.is_empty());
+    assert!(crate::scanner_io::dirty_usage_buckets_pending());
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+}
+
 #[tokio::test]
 async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
     let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
@@ -3107,6 +3328,31 @@ fn clean_idle_backoff_requires_activity_probes() {
     assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config));
 }
 
+#[test]
+fn dirty_usage_wakes_are_disabled_for_explicit_cycle_policy() {
+    let default_config = ScannerRuntimeConfig::default();
+
+    let default_observed = ScannerCycleObservedGenerations::for_wait(&default_config, None, 7, 11, 13);
+    assert_eq!(default_observed.dirty_usage, Some(7));
+    assert_eq!(default_observed.runtime_config, 11);
+    assert_eq!(default_observed.maintenance, 13);
+    assert!(!default_observed.defer_cluster_activity);
+
+    let retry_observed = ScannerCycleObservedGenerations::for_wait(&default_config, Some(Duration::from_secs(11)), 7, 11, 13);
+    assert_eq!(retry_observed.dirty_usage, None);
+    assert!(retry_observed.defer_cluster_activity);
+
+    for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] {
+        let explicit_cycle = ScannerRuntimeConfig {
+            cycle_interval_source: source,
+            ..default_config.clone()
+        };
+        let explicit_observed = ScannerCycleObservedGenerations::for_wait(&explicit_cycle, None, 7, 11, 13);
+        assert_eq!(explicit_observed.dirty_usage, None);
+        assert!(!explicit_observed.defer_cluster_activity);
+    }
+}
+
 #[test]
 #[serial]
 fn clean_idle_cap_preserves_default_bitrot_coverage_window() {

crates/scanner/src/scanner/usage_store.rs

@@ -22,6 +22,10 @@ pub(super) enum DataUsagePersistOutcome {
     AlreadyDurable,
     PriorCycleDurable,
     Saved,
+    /// The metadata route is temporarily unavailable (for example while a
+    /// terminal decommission state keeps the source pool suspended).  The
+    /// caller must retry without acknowledging dirty usage.
+    Deferred(ScannerCycleDeferReason),
     Failed,
 }
 
@@ -92,10 +96,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch(
 pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
     ctx: CancellationToken,
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
-    mut receiver: mpsc::Receiver<DataUsageInfo>,
+    receiver: mpsc::Receiver<DataUsageInfo>,
     leader_epoch: Option<u64>,
     initial_baseline: Option<DataUsagePersistBaseline>,
 ) -> DataUsagePersistOutcome {
+    store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        ctx,
+        storeapi,
+        receiver,
+        leader_epoch,
+        initial_baseline,
+        || async { false },
+    )
+    .await
+}
+
+pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
+    ctx: CancellationToken,
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    mut receiver: mpsc::Receiver<DataUsageInfo>,
+    leader_epoch: Option<u64>,
+    initial_baseline: Option<DataUsagePersistBaseline>,
+    route_probe: F,
+) -> DataUsagePersistOutcome
+where
+    F: Fn() -> Fut + Send + Sync,
+    Fut: Future<Output = bool> + Send,
+{
     let mut outcome = DataUsagePersistOutcome::NoUpdate;
     let mut next_baseline = initial_baseline;
 
@@ -113,6 +140,19 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
         } else {
             DATA_USAGE_OBJ_NAME_PATH.as_str()
         };
+        if route_probe().await {
+            debug!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                path = %target_path,
+                state = "publication_blocked_before_reconcile",
+                "Scanner data usage publication deferred by the pool-state fence"
+            );
+            outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+            break;
+        }
 
         if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
             let authoritative_data = match next_baseline.as_ref() {
@@ -275,6 +315,18 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
             if ctx.is_cancelled() {
                 break 'updates;
             }
+            if route_probe().await {
+                debug!(
+                    target: "rustfs::scanner",
+                    event = EVENT_SCANNER_PERSIST_STATE,
+                    component = LOG_COMPONENT_SCANNER,
+                    subsystem = LOG_SUBSYSTEM_RUNTIME,
+                    path = %target_path,
+                    state = "publication_blocked_before_save",
+                    "Scanner data usage publication deferred by the final pool-state fence"
+                );
+                break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+            }
 
             let done_save = Metrics::time(Metric::SaveUsage);
             let save_result = save_config_shared_with_preconditions(
@@ -313,6 +365,33 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
                         "Scanner data usage CAS conflict will be reconciled"
                     );
                 }
+                Err(e @ EcstoreError::ObjectNotFound(_, _)) => {
+                    let route_blocked = route_probe().await;
+                    if route_blocked {
+                        warn!(
+                            target: "rustfs::scanner",
+                            event = EVENT_SCANNER_PERSIST_STATE,
+                            component = LOG_COMPONENT_SCANNER,
+                            subsystem = LOG_SUBSYSTEM_RUNTIME,
+                            path = %target_path,
+                            state = "publication_deferred",
+                            error = %e,
+                            "Scanner data usage route is blocked by data movement; retrying later"
+                        );
+                        break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+                    }
+                    error!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        path = %target_path,
+                        state = "save_failed",
+                        error = %e,
+                        "Scanner data usage save failed"
+                    );
+                    break DataUsagePersistOutcome::Failed;
+                }
                 Err(e) => {
                     error!(
                         target: "rustfs::scanner",
@@ -370,6 +449,13 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
                 outcome = DataUsagePersistOutcome::Failed;
                 continue;
             }
+            DataUsagePersistOutcome::Deferred(reason) => {
+                // A deferred publication is an intentional retryable state, not a
+                // failed save. Keep the last real save result so admin freshness
+                // reporting does not turn a pool-recovery fence into a false error.
+                outcome = DataUsagePersistOutcome::Deferred(reason);
+                break 'updates;
+            }
             DataUsagePersistOutcome::Saved => {
                 if observational {
                     invalidate_admin_data_usage_snapshot_cache().await;

crates/scanner/src/scanner_folder/item_actions.rs

@@ -274,18 +274,13 @@ impl ScannerItem {
     /// Transform meta directory by splitting prefix and extracting object name
     /// This converts a directory path like "bucket/dir1/dir2/file" to prefix="bucket/dir1/dir2" and object_name="file"
     pub fn transform_meta_dir(&mut self) {
-        let prefix = self.prefix.clone(); // Clone to avoid borrow checker issues
+        let prefix = std::mem::take(&mut self.prefix);
-        let split: Vec<&str> = prefix.split(SLASH_SEPARATOR).collect();
+        if let Some((parent, object_name)) = prefix.rsplit_once(SLASH_SEPARATOR) {
-
+            self.prefix = path_join_buf(&[parent]);
-        if split.len() > 1 {
+            self.object_name = object_name.to_string();
-            let prefix_parts: Vec<&str> = split[..split.len() - 1].to_vec();
-            self.prefix = path_join_buf(&prefix_parts);
         } else {
-            self.prefix = String::new();
+            self.object_name = prefix;
         }
-
-        // Object name is the last element
-        self.object_name = split.last().unwrap_or(&"").to_string();
     }
 
     pub(super) fn metadata_object_path(&self) -> String {
@@ -301,13 +296,14 @@ impl ScannerItem {
         versioning_config: VersioningConfiguration,
         size_summary: &mut SizeSummary,
     ) {
+        let object_path = self.object_path();
         if object_infos.is_empty() {
             debug!(
                 target: "rustfs::scanner::folder",
                 event = EVENT_SCANNER_LIFECYCLE_ACTION,
                 component = LOG_COMPONENT_SCANNER,
                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                object_path = %self.object_path(),
+                object_path = %object_path,
                 state = "no_object_versions",
                 "Scanner lifecycle action skipped"
             );
@@ -318,7 +314,7 @@ impl ScannerItem {
             event = EVENT_SCANNER_LIFECYCLE_ACTION,
             component = LOG_COMPONENT_SCANNER,
             subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-            object_path = %self.object_path(),
+            object_path = %object_path,
             state = "started",
             "Scanner lifecycle evaluation started"
         );
@@ -360,7 +356,7 @@ impl ScannerItem {
                 event = EVENT_SCANNER_LIFECYCLE_ACTION,
                 component = LOG_COMPONENT_SCANNER,
                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                object_path = %self.object_path(),
+                object_path = %object_path,
                 state = "no_lifecycle_config",
                 "Scanner lifecycle action finished without lifecycle rules"
             );
@@ -385,7 +381,7 @@ impl ScannerItem {
                     event = EVENT_SCANNER_LIFECYCLE_ACTION,
                     component = LOG_COMPONENT_SCANNER,
                     subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                    object_path = %self.object_path(),
+                    object_path = %object_path,
                     state = "evaluate_failed",
                     error = %e,
                     "Scanner lifecycle action evaluation failed"
@@ -502,7 +498,7 @@ impl ScannerItem {
                         emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
                         if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
                             done_ilm(1)();
-                            if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
+                            if !versioning_config.prefix_enabled(&object_path) && event.action == IlmAction::DeleteAction {
                                 remaining_versions -= 1;
                                 size = 0;
                             }
@@ -570,7 +566,7 @@ impl ScannerItem {
                 trace_emit(|| {
                     TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
                         .with_bucket(self.bucket.as_str())
-                        .with_object(self.object_path())
+                        .with_object(object_path.as_str())
                         .with_duration(trace_started_at.elapsed())
                         .with_attr("state", state)
                         .with_attr("action", action.as_str())
@@ -889,3 +885,48 @@ pub(super) async fn contains_erasure_part_file(path: &str) -> Result<bool, Scann
 
     Ok(false)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn scanner_item_with_prefix(prefix: &str) -> ScannerItem {
+        ScannerItem {
+            path: String::new(),
+            bucket: "bucket".to_string(),
+            prefix: prefix.to_string(),
+            object_name: String::new(),
+            file_type: std::fs::metadata(std::env::temp_dir())
+                .expect("temp dir metadata should be readable")
+                .file_type(),
+            lifecycle: None,
+            object_lock: None,
+            replication: None,
+            heal_enabled: false,
+            heal_bitrot: false,
+            debug: false,
+        }
+    }
+
+    #[test]
+    fn transform_meta_dir_splits_parent_and_object_without_extra_components() {
+        let mut item = scanner_item_with_prefix("bucket/prefix/object");
+
+        item.transform_meta_dir();
+
+        assert_eq!(item.prefix, "bucket/prefix");
+        assert_eq!(item.object_name, "object");
+        assert_eq!(item.object_path(), "bucket/prefix/object");
+    }
+
+    #[test]
+    fn transform_meta_dir_moves_single_component_into_object_name() {
+        let mut item = scanner_item_with_prefix("object");
+
+        item.transform_meta_dir();
+
+        assert_eq!(item.prefix, "");
+        assert_eq!(item.object_name, "object");
+        assert_eq!(item.object_path(), "object");
+    }
+}

crates/scanner/src/scanner_io/guards.rs

@@ -147,11 +147,19 @@ impl Drop for DiskBucketScanActiveGuard {
 
 pub(super) struct BucketDriveFailureGuard {
     failed: bool,
+    source: rustfs_common::metrics::ScannerWorkSource,
+    bucket: String,
+    drive: String,
 }
 
 impl BucketDriveFailureGuard {
-    pub(super) fn new() -> Self {
+    pub(super) fn new(source: rustfs_common::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
-        Self { failed: true }
+        Self {
+            failed: true,
+            source,
+            bucket: bucket.to_string(),
+            drive: drive.to_string(),
+        }
     }
 
     pub(super) fn mark_not_failed(&mut self) {
@@ -161,6 +169,7 @@ impl BucketDriveFailureGuard {
 
 impl Drop for BucketDriveFailureGuard {
     fn drop(&mut self) {
+        global_metrics().record_scan_bucket_drive_end(self.source, &self.bucket, &self.drive);
         if self.failed {
             global_metrics().record_scan_bucket_drive_failure();
         }
@@ -272,3 +281,28 @@ pub(super) fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error)
 pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError) -> Error {
     Error::other(format!("{stage} task join failed: {err}"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_common::metrics::{ScannerWorkSource, global_metrics};
+
+    #[test]
+    fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_active_lifecycle_test__";
+        let drive = "/__guard_active_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+        {
+            let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            guard.mark_not_failed();
+        }
+        assert!(
+            !global_metrics()
+                .scanner_runtime_details_report()
+                .active_bucket_drive_scans
+                .iter()
+                .any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
+        );
+    }
+}

crates/scanner/src/scanner_io/io_cycle.rs

@@ -49,6 +49,25 @@ impl ScannerIOCycle for ECStore {
     ) -> Result<ScannerCycleResult> {
         let child_token = ctx.child_token();
 
+        // Check the local pool metadata before listing buckets. A failed or
+        // canceled decommission remains suspended after its worker exits, so
+        // starting a scan in that state could build a snapshot that cannot be
+        // routed to the authoritative metadata object.
+        if self.scanner_data_usage_publication_blocked().await {
+            debug!(
+                target: "rustfs::scanner::io",
+                event = EVENT_SCANNER_SET_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_IO,
+                state = "cycle_data_usage_route_blocked",
+                "Scanner cycle deferred while data usage metadata remains hidden by data movement"
+            );
+            return Ok(ScannerCycleResult::new(
+                ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
+                None,
+            ));
+        }
+
         let distributed = self.setup_is_dist_erasure().await;
         let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
             ScannerActivityPreflight::Ready(snapshot) => snapshot,

crates/scanner/src/scanner_io/io_disk.rs

@@ -42,7 +42,8 @@ impl ScannerIODisk for Disk {
             return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
         }
 
-        let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
+        let metadata_object_path = item.object_path();
+        let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
             Ok(data) => data,
             Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
                 return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -51,23 +52,23 @@ impl ScannerIODisk for Disk {
                 return Err(scanner_metadata_transient_error(
                     format!("failed to read metadata: {e}"),
                     &item.bucket,
-                    &item.object_path(),
+                    &metadata_object_path,
                 ));
             }
         };
 
         item.transform_meta_dir();
+        let object_path = item.object_path();
 
-        let meta = FileMeta::load(&data).map_err(|e| {
+        let meta = FileMeta::load(&data)
-            scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
+            .map_err(|e| scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &object_path))?;
-        })?;
+        let fivs = match meta.get_file_info_versions(item.bucket.as_str(), object_path.as_str(), false) {
-        let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
             Ok(versions) => versions,
             Err(e) => {
                 return Err(scanner_metadata_corrupt_error(
                     format!("failed to resolve file info versions: {e}"),
                     &item.bucket,
-                    &item.object_path(),
+                    &object_path,
                 ));
             }
         };
@@ -91,17 +92,17 @@ impl ScannerIODisk for Disk {
                 VersioningConfiguration::default()
             }
         };
-        let versioned = versioning_config.versioned(&item.object_path());
+        let versioned = versioning_config.versioned(&object_path);
 
         let object_infos = fivs
             .versions
             .iter()
-            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
+            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
             .collect::<Vec<ObjectInfo>>();
         let free_version_infos = fivs
             .free_versions
             .iter()
-            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
+            .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
             .collect::<Vec<ObjectInfo>>();
 
         let mut size_summary = SizeSummary::default();
@@ -147,8 +148,12 @@ impl ScannerIODisk for Disk {
         let drive_start = std::time::Instant::now();
         let bucket = cache.info.name.clone();
         let disk_path = self.path().to_string_lossy().to_string();
-        global_metrics().record_scan_bucket_drive_start();
+        let source = match scan_mode {
-        let mut failure_guard = BucketDriveFailureGuard::new();
+            HealScanMode::Deep => rustfs_common::metrics::ScannerWorkSource::Bitrot,
+            HealScanMode::Normal | HealScanMode::Unknown => rustfs_common::metrics::ScannerWorkSource::Usage,
+        };
+        global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
+        let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
         let _guard = self.start_scan();
 
         let mut cache = cache;
@@ -196,32 +201,32 @@ impl ScannerIODisk for Disk {
         match result {
             Ok(mut data_usage_info) => {
                 done_drive();
-                emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_complete(source, true, &bucket, &disk_path, drive_start.elapsed());
                 data_usage_info.info.last_update = Some(SystemTime::now());
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::Complete(data_usage_info))
             }
             Err(ScannerError::PartialCache(mut partial_cache)) => {
                 done_drive();
-                emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                 partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::Partial(*partial_cache))
             }
             Err(ScannerError::NamespaceNotFoundCache(mut partial_cache)) => {
                 done_drive();
-                emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                 partial_cache.info.last_update.get_or_insert_with(SystemTime::now);
                 failure_guard.mark_not_failed();
                 Ok(ScannerDiskScanOutcome::NamespaceNotFound(*partial_cache))
             }
             Err(e) => {
                 if ctx.is_cancelled() {
-                    emit_scan_bucket_drive_partial(&bucket, &disk_path, drive_start.elapsed());
+                    emit_scan_bucket_drive_partial(source, &bucket, &disk_path, drive_start.elapsed());
                     failure_guard.mark_not_failed();
                 } else {
                     done_drive();
-                    emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
+                    emit_scan_bucket_drive_complete(source, false, &bucket, &disk_path, drive_start.elapsed());
                 }
                 Err(StorageError::other(format!("Failed to scan data folder: {e}")))
             }

crates/scanner/src/scanner_io/tests.rs

@@ -17,7 +17,9 @@ use super::io_disk::tier_stats_template;
 use super::*;
 use crate::scanner_budget::ScannerCycleBudgetConfig;
 use crate::scanner_folder::ScannerItem;
-use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
+use crate::storage_api::owner::{
+    EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
+};
 use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
 use crate::{
     DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
@@ -182,6 +184,39 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
     assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
 }
 
+#[tokio::test]
+#[serial]
+async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
+    let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+    for decommission in [
+        EcstorePoolDecommissionInfo {
+            failed: true,
+            ..Default::default()
+        },
+        EcstorePoolDecommissionInfo {
+            canceled: true,
+            ..Default::default()
+        },
+    ] {
+        store.pool_meta.write().await.pools[0].decommission = Some(decommission);
+        assert!(store.scanner_data_usage_publication_blocked().await);
+
+        let ctx = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
+        let (updates, mut receiver) = mpsc::channel(1);
+        let result = tokio::time::timeout(
+            Duration::from_secs(30),
+            ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
+        )
+        .await
+        .expect("terminal-decommission-deferred scanner cycle should finish")
+        .expect("terminal-decommission-deferred scanner cycle should succeed");
+
+        assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
+        assert!(receiver.recv().await.is_none(), "blocked cycle must not publish usage");
+    }
+}
+
 #[tokio::test]
 async fn data_usage_publish_fails_when_receiver_is_closed() {
     let (updates, receiver) = mpsc::channel(1);
@@ -236,6 +271,10 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
     assert_eq!(bucket_usage.size, 11);
     assert_eq!(usage.objects_total_count, 2);
     assert_eq!(usage.objects_total_size, 11);
+    assert!(
+        receiver.recv().await.is_none(),
+        "a scanner cycle must publish at most one terminal usage snapshot"
+    );
 }
 
 #[tokio::test]

crates/scanner/src/storage_api.rs

@@ -47,6 +47,8 @@ pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys
 pub(crate) use rustfs_ecstore::api::cache::{
     ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
 };
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::api::capacity::PoolDecommissionInfo as EcstorePoolDecommissionInfo;
 pub(crate) use rustfs_ecstore::api::capacity::{
     is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
     path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
@@ -127,9 +129,9 @@ pub(crate) mod owner {
     #[cfg(test)]
     pub(crate) use super::{
         EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
-        EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
+        EcstoreInstanceContext, EcstorePoolDecommissionInfo, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo,
-        EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
+        EcstoreRebalanceMeta, EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys,
-        ecstore_new_disk,
+        ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
     };
 }
 

docs/operations/scanner-runtime-controls.md

@@ -70,6 +70,11 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
 `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
 needs a precise override.
 
+An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
+cadence: dirty-usage notifications do not bypass that configured interval.
+The default adaptive policy continues to use dirty-usage notifications to wake
+the scanner between timer-driven cycles.
+
 ## Single-disk clean-idle scheduling
 
 An erasure single-disk deployment using the built-in cycle and bitrot defaults

docs/testing/metrics-1649-prometheus-smoke.md

@@ -0,0 +1,44 @@
+# Backlog #1649 Prometheus smoke
+
+`scripts/prometheus_metrics_1649_smoke.py` is a read-only environment check for
+the metric dimensions delivered by backlog #1649 and issues #1650-#1653. It
+uses Prometheus' instant-query API and does not start, stop, reconfigure, or
+load RustFS nodes.
+
+Run the parser and selector self-test without a live environment:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py --self-test
+```
+
+For a live cluster, pass a Prometheus base URL (or its `/api/v1/query`
+endpoint), one or more expected server label values, and the built-in profile:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py \
+  --query-url http://prometheus.example:9090 \
+  --profile backlog-1649 \
+  --server rustfs-node1 \
+  --server rustfs-node2
+```
+
+The profile checks the disk, scanner, ILM, audit, and notification series and
+their required labels. It also requires the legacy aggregate audit and
+notification series, so an additive label change cannot silently break
+existing dashboards.
+
+Dynamic series retirement is checked with an exact label set after the
+scheduler retirement window has elapsed:
+
+```bash
+python3 scripts/prometheus_metrics_1649_smoke.py \
+  --query-url http://prometheus.example:9090 \
+  --retired 'rustfs_scanner_bucket_drive_result_total|server=node1,bucket=removed,drive=d1,result=success' \
+  --retired 'rustfs_audit_total_messages_by_server|server=node1,target_id=removed'
+```
+
+`--require METRIC|key=value,...` requires a matching series;
+`--require-labels METRIC|key1,key2` requires every returned series to carry
+the named labels. Use `--bearer` for a bearer token or `--basic` for a
+`username:password` credential when Prometheus is protected. Do not put
+credentials in committed commands, logs, or issue comments.

rustfs/src/admin/handlers/inspect_archive.rs

@@ -643,7 +643,7 @@ mod tests {
     fn decode_hex_fixture(value: &str) -> Vec<u8> {
         value
             .split_ascii_whitespace()
-            .flat_map(|line| line.as_bytes().chunks_exact(2))
+            .flat_map(|line| line.as_bytes().as_chunks::<2>().0.iter())
             .map(|pair| {
                 let pair = std::str::from_utf8(pair).expect("fixture contains ASCII hex");
                 u8::from_str_radix(pair, 16).expect("fixture contains valid hex")

rustfs/src/admin/router.rs

@@ -2694,7 +2694,7 @@ async fn ssec_passthrough_probe_object(
     let head = target_client
         .head_object(target_bucket, probe_key, head_version)
         .await
-        .map_err(S3ClientError::from)?;
+        .map_err(|err| S3ClientError::from(*err))?;
 
     Ok(ReplicationSsecProbeOutcome {
         evidence_present: head.sse_customer_algorithm().is_some_and(|algorithm| !algorithm.is_empty()),

rustfs/src/app/object_usecase.rs

@@ -3683,6 +3683,13 @@ where
 
     let authorization_headers = pax_headers.clone();
 
+    if let Some(value) = pax_headers.remove("x-amz-tagging") {
+        let value = value
+            .to_str()
+            .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball object tagging value"))?;
+        metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), value.to_owned());
+    }
+
     let object_lock_mode = pax_headers
         .remove(AMZ_OBJECT_LOCK_MODE_LOWER)
         .map(|value| {
@@ -3962,6 +3969,14 @@ fn delete_creates_delete_marker(opts: &ObjectOptions) -> bool {
     opts.version_id.is_none() && opts.versioned && !opts.version_suspended
 }
 
+/// `DeleteObjects` is idempotent. A raw filesystem `NotFound` can cross the
+/// distributed delete path instead of its usual typed missing-object error.
+fn is_delete_objects_not_found(error: &EcstoreError) -> bool {
+    is_err_object_not_found(error)
+        || is_err_version_not_found(error)
+        || matches!(error, StorageError::Io(source) if source.kind() == std::io::ErrorKind::NotFound)
+}
+
 /// Bounded concurrency for the per-object pre-delete stat fanout in
 /// `execute_delete_objects` (backlog#929 / HP-8). Keeps the metadata reads for
 /// a 1000-key batch from serializing while capping the disk fanout pressure.
@@ -4023,6 +4038,27 @@ fn delete_response_version_id(version_id: Option<Uuid>, synthetic_version_id: bo
     }
 }
 
+fn reduce_delete_objects_result<'a>(
+    object: &ObjectToDelete,
+    deleted: &'a StorageDeletedObject,
+    error: Option<&EcstoreError>,
+    synthetic_version_id: bool,
+) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
+    match error {
+        None => Ok(deleted),
+        Some(error) if is_delete_objects_not_found(error) => Ok(deleted),
+        Some(error) => {
+            let api_error = ApiError::from(error.clone());
+            Err(s3s::dto::Error {
+                code: Some(api_error.code.as_str().to_string()),
+                key: Some(object.object_name.clone()),
+                message: Some(api_error.message),
+                version_id: delete_response_version_id(object.version_id, synthetic_version_id),
+            })
+        }
+    }
+}
+
 fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObjectExtractOptions> {
     let prefix = snowball_meta_value(headers, SNOWBALL_PREFIX_HEADER_KEYS, SNOWBALL_PREFIX_SUFFIX_LOWER)
         .map(|value| normalize_snowball_prefix(&value))
@@ -8469,12 +8505,14 @@ impl DefaultObjectUsecase {
         for (i, err) in errs.iter().enumerate() {
             let didx = object_to_delete_idx[i];
 
-            if err.is_none()
+            match reduce_delete_objects_result(
-                || err
+                &object_to_delete[i],
-                    .clone()
+                &dobjs[i],
-                    .is_some_and(|v| is_err_object_not_found(&v) || is_err_version_not_found(&v))
+                err.as_ref(),
-            {
+                delete_results[didx].synthetic_version_id,
-                delete_results[didx].delete_object = Some(dobjs[i].clone());
+            ) {
+                Ok(deleted_object) => {
+                    delete_results[didx].delete_object = Some(deleted_object.clone());
                     let (versioned, version_suspended) = object_versioning[i];
                     let creates_delete_marker = object_to_delete[i].version_id.is_none() && versioned && !version_suspended;
                     if creates_delete_marker {
@@ -8488,20 +8526,10 @@ impl DefaultObjectUsecase {
                         )
                         .await;
                     }
-                continue;
                 }
-
+                Err(error) => {
-            if let Some(err) = err.clone() {
+                    delete_results[didx].error = Some(error);
-                let api_error = ApiError::from(err);
+                }
-                delete_results[didx].error = Some(s3s::dto::Error {
-                    code: Some(api_error.code.as_str().to_string()),
-                    key: Some(object_to_delete[i].object_name.clone()),
-                    message: Some(api_error.message),
-                    version_id: delete_response_version_id(
-                        object_to_delete[i].version_id,
-                        delete_results[didx].synthetic_version_id,
-                    ),
-                });
             }
         }
 
@@ -10740,6 +10768,7 @@ mod tests {
         let mut record = pax_record("minio.metadata.Content-Type", b"text/plain");
         record.extend(pax_record("minio.metadata.X-Amz-Meta-Owner", b"alice"));
         record.extend(pax_record("minio.metadata.project", b"alpha-demo"));
+        record.extend(pax_record("minio.metadata.x-amz-tagging", b"classification=public"));
         record.extend(pax_record("minio.versionId", Uuid::nil().to_string().as_bytes()));
         record.extend(pax_record("minio.metadata.x-amz-replication-status", b"REPLICA"));
         record.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"GOVERNANCE"));
@@ -10778,6 +10807,8 @@ mod tests {
         assert_eq!(metadata.get("content-type").map(String::as_str), Some("text/plain"));
         assert_eq!(metadata.get("owner").map(String::as_str), Some("alice"));
         assert_eq!(metadata.get("project").map(String::as_str), Some("alpha-demo"));
+        assert_eq!(metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("classification=public"));
+        assert!(!metadata.contains_key("x-amz-tagging"));
         assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
         assert_eq!(
             metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER).map(String::as_str),
@@ -17682,6 +17713,35 @@ mod tests {
         assert_eq!(internal_version_id, None);
     }
 
+    #[test]
+    fn delete_objects_treats_raw_io_not_found_as_idempotent() {
+        assert!(is_delete_objects_not_found(&StorageError::FileNotFound));
+        assert!(is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
+            std::io::ErrorKind::NotFound,
+        ))));
+        assert!(!is_delete_objects_not_found(&StorageError::Io(std::io::Error::from(
+            std::io::ErrorKind::PermissionDenied,
+        ))));
+        assert!(!is_delete_objects_not_found(&StorageError::DiskNotFound));
+    }
+
+    #[test]
+    fn delete_objects_result_reducer_reports_raw_not_found_as_deleted() {
+        let object = ObjectToDelete {
+            object_name: "missing-key".to_string(),
+            ..Default::default()
+        };
+        let deleted = StorageDeletedObject {
+            object_name: object.object_name.clone(),
+            ..Default::default()
+        };
+        let error = StorageError::Io(std::io::Error::from(std::io::ErrorKind::NotFound));
+
+        let deleted = reduce_delete_objects_result(&object, &deleted, Some(&error), false)
+            .expect("raw not-found must produce a deleted result");
+        assert_eq!(deleted.object_name, "missing-key");
+    }
+
     #[test]
     fn recursive_force_delete_requires_administrative_or_replica_context() {
         let mut headers = HeaderMap::new();

rustfs/src/protocols/client.rs

@@ -13,10 +13,16 @@
 // limitations under the License.
 
 use crate::runtime_sources::current_action_credentials;
+#[cfg(feature = "webdav")]
+use crate::shared_types::RemoteAddr;
 use crate::storage_api::protocols::client::{FS, ReqInfo, RequestContext};
 use http::{HeaderMap, Method};
 use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
 use rustfs_credentials;
+#[cfg(feature = "webdav")]
+use rustfs_protocols::common::SessionContext;
+#[cfg(feature = "webdav")]
+use rustfs_trusted_proxies::ClientInfo;
 use rustfs_utils::MaskedAccessKey;
 use s3s::dto::*;
 use s3s::{S3, S3Request, S3Result};
@@ -90,6 +96,50 @@ fn trace_protocol_request(operation: &str, bucket: Option<&str>, object: Option<
     );
 }
 
+#[cfg(feature = "webdav")]
+fn session_list_buckets_request(
+    input: ListBucketsInput,
+    session_context: &SessionContext,
+    request_headers: &HeaderMap,
+    secure_transport: bool,
+) -> S3Request<ListBucketsInput> {
+    let credentials = &session_context.principal.user_identity.credentials;
+    let mut extensions = http::Extensions::default();
+    let remote_addr = std::net::SocketAddr::new(session_context.source_ip, 0);
+    extensions.insert(Some(RemoteAddr(remote_addr)));
+    let mut client_info = ClientInfo::direct(remote_addr);
+    client_info.forwarded_proto = Some(if secure_transport { "https" } else { "http" }.to_string());
+    extensions.insert(client_info);
+
+    let is_owner = current_action_credentials().is_some_and(|global_cred| credentials.access_key == global_cred.access_key);
+    extensions.insert(ReqInfo {
+        cred: Some(credentials.clone()),
+        is_owner,
+        bucket: None,
+        object: None,
+        version_id: None,
+        replication_request_authorized: false,
+        region: None,
+        request_context: Some(RequestContext::fallback()),
+        suppress_denial_log: false,
+    });
+
+    S3Request {
+        input,
+        method: Method::GET,
+        uri: http::Uri::from_static("/"),
+        headers: request_headers.clone(),
+        extensions,
+        credentials: Some(s3s::auth::Credentials {
+            access_key: credentials.access_key.clone(),
+            secret_key: credentials.secret_key.clone().into(),
+        }),
+        region: None,
+        service: None,
+        trailing_headers: None,
+    }
+}
+
 fn build_bucket_uri(bucket: &str, query: &[(&str, Option<&str>)]) -> S3Result<http::Uri> {
     let mut uri = format!("/{}", encode_path_segment(bucket));
     let mut first = true;
@@ -469,6 +519,29 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
+    #[cfg(feature = "webdav")]
+    async fn list_buckets_for_session(
+        &self,
+        session_context: &SessionContext,
+        request_headers: &HeaderMap,
+        secure_transport: bool,
+    ) -> Result<ListBucketsOutput, Self::Error> {
+        trace!(
+            event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
+            component = LOG_COMPONENT_PROTOCOLS,
+            subsystem = LOG_SUBSYSTEM_STORAGE_CLIENT,
+            operation = "list_buckets",
+            access_key = %MaskedAccessKey(&session_context.principal.user_identity.credentials.access_key),
+            "Protocol storage client request"
+        );
+
+        let input = ListBucketsInput::builder().build().map_err(|e| {
+            s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build ListBucketsInput: {}", e))
+        })?;
+        let request = session_list_buckets_request(input, session_context, request_headers, secure_transport);
+        self.fs.list_buckets(request).await.map(|response| response.output)
+    }
+
     async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error> {
         trace_protocol_request("create_bucket", Some(bucket), None);
 
@@ -872,6 +945,60 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[cfg(feature = "webdav")]
+    #[test]
+    fn request_extensions_preserve_authenticated_identity_and_source_ip() {
+        use std::collections::HashMap;
+        use std::net::{IpAddr, Ipv4Addr};
+
+        let claims = HashMap::from([("parent".to_string(), serde_json::json!("alice"))]);
+        let credentials = rustfs_credentials::Credentials {
+            access_key: "service-account".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: "session-token".to_string(),
+            parent_user: "alice".to_string(),
+            groups: Some(vec!["developers".to_string()]),
+            claims: Some(claims.clone()),
+            ..Default::default()
+        };
+        let source_ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10));
+
+        let identity = rustfs_policy::auth::UserIdentity {
+            credentials: credentials.clone(),
+            ..Default::default()
+        };
+        let principal = rustfs_protocols::common::ProtocolPrincipal::new(std::sync::Arc::new(identity));
+        let session_context = SessionContext::new(principal, rustfs_protocols::Protocol::WebDav, source_ip);
+        let mut headers = HeaderMap::new();
+        headers.insert("user-agent", http::HeaderValue::from_static("webdav-client"));
+        let request = session_list_buckets_request(ListBucketsInput::default(), &session_context, &headers, true);
+        let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
+        let copied = request_info.cred.as_ref().expect("credentials should be present");
+        let remote_addr = request
+            .extensions
+            .get::<Option<RemoteAddr>>()
+            .and_then(Option::as_ref)
+            .expect("remote address should be present");
+        let client_info = request.extensions.get::<ClientInfo>().expect("client info should be present");
+
+        assert_eq!(copied.access_key, credentials.access_key);
+        assert_eq!(copied.secret_key, credentials.secret_key);
+        assert_eq!(copied.session_token, credentials.session_token);
+        assert_eq!(copied.parent_user, credentials.parent_user);
+        assert_eq!(copied.groups, credentials.groups);
+        assert_eq!(copied.claims, Some(claims));
+        assert_eq!(remote_addr.0.ip(), source_ip);
+        assert_eq!(client_info.real_ip, source_ip);
+        assert_eq!(client_info.forwarded_proto.as_deref(), Some("https"));
+        assert_eq!(request.headers.get("user-agent").expect("user agent"), "webdav-client");
+
+        let insecure_request = session_list_buckets_request(ListBucketsInput::default(), &session_context, &headers, false);
+        let insecure_client_info = insecure_request
+            .extensions
+            .get::<ClientInfo>()
+            .expect("client info should be present");
+        assert_eq!(insecure_client_info.forwarded_proto.as_deref(), Some("http"));
+    }
 
     #[test]
     fn build_object_uri_encodes_key_segments_without_flattening_slashes() {

rustfs/src/server/http.rs

@@ -1537,7 +1537,7 @@ fn process_connection(
                 None
             }
         };
-        // ── Canonical Middleware Stack Order (outermost → innermost) ──
+        // ── Canonical External Middleware Stack Order (outermost → innermost) ──
         // This order MUST be preserved across refactorings.
         // Only AddExtensionLayer (layers 1-2) are per-connection; most remaining layers are stateless.
         //
@@ -1565,6 +1565,8 @@ fn process_connection(
         // 22. PublicHealthEndpointLayer              — handles public health before s3s host parsing
         // 23. VirtualHostStyleHintLayer              — actionable error for unroutable virtual-hosted-style (conditional)
         // 24. DoubleSlashListBucketsCompatLayer      — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
+        // The internode lane below intentionally keeps only the shared
+        // transport/auth/observability subset needed by `/rustfs/rpc/...`.
         // ─────────────────────────────────────────────────────────────
         let build_external_stack = |service| {
             ServiceBuilder::new()
@@ -1747,16 +1749,9 @@ fn process_connection(
                 .layer(PropagateRequestIdLayer::x_request_id())
                 .layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
                 .option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
-                .layer(S3ErrorMessageCompatLayer)
+                // The internode lane only serves `/rustfs/rpc/...` gRPC requests.
-                .layer(IcebergRestErrorCompatLayer)
+                // Keep safety/observability layers above, but leave S3/REST
-                .layer(ObjectAttributesEtagFixLayer)
+                // compatibility rewrites on the external lane.
-                .layer(ConditionalCorsLayer::new())
-                .option_layer(if is_console { Some(RedirectLayer) } else { None })
-                .layer(BodylessStatusFixLayer)
-                .layer(HeadRequestBodyFixLayer)
-                .layer(PublicHealthEndpointLayer::new(Arc::clone(&server_ctx)))
-                .option_layer((!server_domains_configured && !is_console).then_some(VirtualHostStyleHintLayer))
-                .layer(DoubleSlashListBucketsCompatLayer)
                 .service(service)
         };
         let external_stack_service = build_external_stack(external_service);

rustfs/src/storage/rpc/http_service.rs

@@ -1500,7 +1500,7 @@ where
         return write_body_chunks_to_writer(body, writer).await;
     };
 
-    let expected_size = (!query.append && query.size >= 0)
+    let expected_size = (!query.append && query.size > 0)
         .then(|| {
             u64::try_from(query.size)
                 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "put_file auth size cannot be represented"))
@@ -2325,6 +2325,39 @@ mod tests {
         assert_eq!(writer, b"append-data");
     }
 
+    #[tokio::test]
+    async fn put_file_auth_zero_size_create_uses_trailing_auth_record() {
+        let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-body-test-secret".to_string());
+        let nonce = uuid::Uuid::parse_str("43434343-4444-4555-8666-777777777777").expect("nonce");
+        let url = concat!(
+            "/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
+            "&append=false&size=0&put_file_auth=digest-trailer-v1&put_file_nonce=43434343-4444-4555-8666-777777777777"
+        );
+        let digest = hex_simd::encode_to_string(sha2::Sha256::digest(b"unknown-size-data"), hex_simd::AsciiCase::Lower);
+        let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &digest).expect("trailer should build");
+        let query = PutFileQuery {
+            disk: "disk-a".to_string(),
+            volume: "bucket".to_string(),
+            path: "object/part.1".to_string(),
+            append: false,
+            size: 0,
+            put_file_auth: Some("digest-trailer-v1".to_string()),
+            put_file_nonce: Some(nonce),
+            put_file_server_epoch: Some(*super::PUT_FILE_CAPABILITY_SERVER_EPOCH),
+        };
+        let mut payload = b"unknown-size-data".to_vec();
+        payload.extend_from_slice(&trailer);
+        let body = iter(vec![Ok::<Bytes, io::Error>(Bytes::from(payload))]);
+        let mut writer = Vec::new();
+
+        let copied = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
+            .await
+            .expect("zero-size create body should verify");
+
+        assert_eq!(copied, 17);
+        assert_eq!(writer, b"unknown-size-data");
+    }
+
     #[tokio::test]
     async fn put_file_auth_append_body_rejects_missing_trailer() {
         let nonce = uuid::Uuid::parse_str("44444444-5555-4666-8777-888888888888").expect("nonce");

rustfs/tests/inspect_cli.rs

@@ -25,7 +25,9 @@ fn decode_hex(source: &str) -> Vec<u8> {
         .collect::<String>();
     digits
         .as_bytes()
-        .chunks_exact(2)
+        .as_chunks::<2>()
+        .0
+        .iter()
         .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).expect("hex pair"), 16).expect("fixture hex byte"))
         .collect()
 }

scripts/prometheus_metrics_1649_smoke.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Read-only Prometheus smoke checks for backlog #1649 metric dimensions.
+
+The harness queries Prometheus' instant-query API. It never writes to RustFS,
+Prometheus, or the scrape targets. A check is ``metric|label=value,...``;
+``--require-labels`` accepts ``metric|label1,label2``. ``--retired`` checks
+that an exact label set is absent after the scheduler's retirement window.
+"""
+
+import argparse
+import base64
+import json
+import sys
+from dataclasses import dataclass
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode, urlparse
+from urllib.request import Request, urlopen
+
+
+@dataclass(frozen=True)
+class Series:
+    labels: dict[str, str]
+
+
+def query_url(value: str) -> str:
+    parsed = urlparse(value)
+    if parsed.path.rstrip("/").endswith("/api/v1/query"):
+        return value
+    return value.rstrip("/") + "/api/v1/query"
+
+
+def parse_spec(spec: str, separator: str = "|") -> tuple[str, dict[str, str]]:
+    metric, _, labels = spec.partition(separator)
+    if not metric or any(c in metric for c in "{} \t"):
+        raise ValueError(f"invalid metric check: {spec!r}")
+    expected: dict[str, str] = {}
+    if labels:
+        for pair in labels.split(","):
+            key, sep, value = pair.partition("=")
+            if not sep or not key or not value:
+                raise ValueError(f"invalid label selector in {spec!r}")
+            if key in expected:
+                raise ValueError(f"duplicate label {key!r} in {spec!r}")
+            expected[key] = value
+    return metric, expected
+
+
+def parse_label_names(spec: str) -> tuple[str, list[str]]:
+    metric, separator, labels = spec.partition("|")
+    names = [item for item in labels.split(",") if item] if separator else []
+    if not metric or any(c in metric for c in "{} \t") or not names or any(
+        "=" in item or not item.replace("_", "a").isalnum() for item in names
+    ):
+        raise ValueError(f"invalid label-name check: {spec!r}")
+    return metric, names
+
+
+def fetch_series(endpoint: str, metric: str, headers: dict[str, str], timeout: float) -> list[Series]:
+    query = f"{metric}{{}}" if "{" not in metric else metric
+    request = Request(f"{endpoint}?{urlencode({'query': query})}", headers=headers)
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            payload = json.load(response)
+    except (HTTPError, URLError, TimeoutError) as error:
+        raise RuntimeError(f"Prometheus query failed for {query!r}: {error}") from error
+    if payload.get("status") != "success":
+        raise RuntimeError(f"Prometheus returned non-success for {query!r}: {payload}")
+    data = payload.get("data", {})
+    if data.get("resultType") != "vector":
+        raise RuntimeError(f"Prometheus query did not return an instant vector: {query!r}")
+    return [Series(dict(item.get("metric", {}))) for item in data.get("result", [])]
+
+
+def has_labels(series: Iterable[Series], expected: dict[str, str]) -> bool:
+    return any(all(item.labels.get(key) == value for key, value in expected.items()) for item in series)
+
+
+def run(args: argparse.Namespace) -> int:
+    headers = {"Accept": "application/json"}
+    if args.bearer:
+        headers["Authorization"] = f"Bearer {args.bearer}"
+    if args.basic:
+        headers["Authorization"] = "Basic " + base64.b64encode(args.basic.encode()).decode()
+    endpoint = query_url(args.query_url)
+    required = list(args.require)
+    labels = list(args.require_labels)
+    retired = list(args.retired)
+    if args.profile == "backlog-1649":
+        required += [
+            "rustfs_system_drive_total_bytes",
+            "rustfs_system_drive_writes_total",
+            "rustfs_system_drive_deletes_total",
+            "rustfs_scanner_source_work_total",
+            "rustfs_scanner_active_bucket_drive_scans",
+            "rustfs_scanner_bucket_drive_result_total",
+            "rustfs_ilm_action_tasks",
+            "rustfs_ilm_tasks",
+            "rustfs_ilm_task_events_total",
+            "rustfs_ilm_queue_backpressure_total",
+            "rustfs_ilm_versions_scanned_by_server",
+            "rustfs_notification_current_send_in_progress_by_server",
+            "rustfs_notification_events_errors_total_by_server",
+            "rustfs_notification_events_sent_total_by_server",
+            "rustfs_notification_events_skipped_total_by_server",
+            "rustfs_audit_failed_messages_by_server",
+            "rustfs_audit_target_queue_length_by_server",
+            "rustfs_audit_total_messages_by_server",
+            "rustfs_notification_events_errors_total",
+            "rustfs_notification_events_sent_total",
+            "rustfs_notification_events_skipped_total",
+            "rustfs_audit_failed_messages",
+            "rustfs_audit_target_queue_length",
+            "rustfs_audit_total_messages",
+        ]
+        labels += [
+            "rustfs_system_drive_total_bytes|server,drive",
+            "rustfs_system_drive_writes_total|server,drive",
+            "rustfs_system_drive_deletes_total|server,drive",
+            "rustfs_scanner_source_work_total|server,source,state",
+            "rustfs_scanner_active_bucket_drive_scans|server,source,bucket,drive",
+            "rustfs_scanner_bucket_drive_result_total|server,bucket,drive,result",
+            "rustfs_ilm_action_tasks|server,action,state",
+            "rustfs_ilm_tasks|server,action,queue_state",
+            "rustfs_ilm_task_events_total|server,action,result",
+            "rustfs_ilm_queue_backpressure_total|server,action,reason",
+            "rustfs_ilm_versions_scanned_by_server|server,source",
+            "rustfs_notification_current_send_in_progress_by_server|server",
+            "rustfs_notification_events_errors_total_by_server|server",
+            "rustfs_notification_events_sent_total_by_server|server",
+            "rustfs_notification_events_skipped_total_by_server|server",
+            "rustfs_audit_failed_messages_by_server|server,target_id",
+            "rustfs_audit_target_queue_length_by_server|server,target_id",
+            "rustfs_audit_total_messages_by_server|server,target_id",
+            "rustfs_audit_failed_messages|target_id",
+            "rustfs_audit_target_queue_length|target_id",
+            "rustfs_audit_total_messages|target_id",
+        ]
+    if not required and not labels and not retired:
+        raise ValueError("provide --profile backlog-1649 or at least one check")
+    failures: list[str] = []
+    cache: dict[str, list[Series]] = {}
+
+    def get(metric: str) -> list[Series]:
+        if metric not in cache:
+            cache[metric] = fetch_series(endpoint, metric, headers, args.timeout)
+        return cache[metric]
+
+    def missing_servers(series: list[Series]) -> list[str]:
+        if not args.server or not any("server" in item.labels for item in series):
+            return []
+        observed = {item.labels["server"] for item in series if "server" in item.labels}
+        return sorted(set(args.server) - observed)
+
+    for spec in required:
+        metric, expected = parse_spec(spec)
+        series = get(metric)
+        if not series:
+            failures.append(f"{metric}: no series returned")
+        elif expected and not has_labels(series, expected):
+            failures.append(f"{metric}: no series has labels {expected}; observed {len(series)} series")
+        elif missing_servers(series):
+            failures.append(f"{metric}: missing requested server series {missing_servers(series)}")
+    for spec in labels:
+        metric, required_labels = parse_label_names(spec)
+        series = get(metric)
+        if not series:
+            failures.append(f"{metric}: aggregate series absent")
+        elif any(not all(label in item.labels for label in required_labels) for item in series):
+            failures.append(f"{metric}: at least one series is missing labels {required_labels}")
+        elif missing_servers(series):
+            failures.append(f"{metric}: missing requested server series {missing_servers(series)}")
+    for spec in retired:
+        metric, expected = parse_spec(spec)
+        if has_labels(get(metric), expected):
+            failures.append(f"{metric}: retired series still present with labels {expected}")
+    if failures:
+        for failure in failures:
+            print(f"FAIL: {failure}", file=sys.stderr)
+        return 1
+    print(f"PASS: {len(required)} required, {len(labels)} label, {len(retired)} retirement checks")
+    return 0
+
+
+def self_test() -> None:
+    assert parse_spec("metric|server=node1,drive=d1") == ("metric", {"server": "node1", "drive": "d1"})
+    assert parse_label_names("metric|server,drive") == ("metric", ["server", "drive"])
+    assert has_labels([Series({"server": "node1", "drive": "d1"})], {"server": "node1"})
+    assert not has_labels([Series({"server": "node1"})], {"server": "node2"})
+    try:
+        parse_spec("metric|server")
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("malformed value selector accepted")
+    print("PASS: self-test")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--query-url", help="Prometheus base URL or /api/v1/query endpoint")
+    parser.add_argument("--profile", choices=["backlog-1649"])
+    parser.add_argument("--server", action="append", default=[], help="server label value required in every server-scoped check")
+    parser.add_argument("--require", action="append", default=[], metavar="METRIC|k=v,...")
+    parser.add_argument("--require-labels", action="append", default=[], metavar="METRIC|k1,k2")
+    parser.add_argument("--retired", action="append", default=[], metavar="METRIC|k=v,...")
+    parser.add_argument("--bearer")
+    parser.add_argument("--basic", help="username:password; prefer --bearer in shared shells")
+    parser.add_argument("--timeout", type=float, default=10.0)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+    if args.self_test:
+        self_test()
+        return 0
+    if not args.query_url:
+        parser.error("--query-url is required unless --self-test is used")
+    try:
+        return run(args)
+    except (RuntimeError, ValueError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())