fix(swift): route metadata limits through storage domain

Signed-off-by: overtrue <anzhengchao@gmail.com>

.config/make/tests.mak

@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_build_rustfs_options.sh
 	./scripts/test_docker_runtime_timezone.sh
 	./scripts/test_entrypoint_credentials.sh
+	./scripts/test_entrypoint_directories.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_package_service_scripts.sh
@@ -70,6 +71,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py

.docker/README.md

@@ -110,6 +110,36 @@ docker compose up -d
 
 ---
 
+## Docker Compose environment values
+
+Use map syntax when quoting environment values in YAML:
+
+```yaml
+services:
+  rustfs:
+    image: rustfs/rustfs:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ./data:/data
+      - ./logs:/var/log/rustfs
+    environment:
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_CONSOLE_ADDRESS: ":9001"
+      RUSTFS_CONSOLE_ENABLE: "true"
+      RUSTFS_OBS_LOGGER_LEVEL: error
+      RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
+      RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
+      RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
+```
+
+Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
+
+With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
+
+---
+
 ## 🔧 Common Operations
 
 ### Cleaning Up

.docker/observability/otel-collector-config.yaml

@@ -28,6 +28,15 @@ processors:
     check_interval: 1s
     limit_mib: 1024
     spike_limit_mib: 256
+  transform/metrics:
+    error_mode: ignore
+    metric_statements:
+      # RustFS process metrics already expose process_pid as a metric label.
+      # Remove only the duplicate datapoint label and keep the shared resource
+      # process.pid available to sibling metrics and lock histograms.
+      - context: datapoint
+        statements:
+          - delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
   transform/logs:
     error_mode: ignore
     log_statements:
@@ -112,7 +121,7 @@ service:
       exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
-      processors: [ batch ]
+      processors: [ transform/metrics, memory_limiter, batch ]
       exporters: [ prometheus ]
     logs:
       receivers: [ otlp ]

.docker/rejoin-readiness.Dockerfile

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1.6
+# Copyright 2026 RustFS Team
+# Licensed under the Apache License, Version 2.0.
+
+# A compatible native runtime with Python 3.11+, getent and stat is required by
+# the isolated fixture's staging/provenance checks. This is a test image.
+ARG RUNTIME_BASE=python:3.13-slim-trixie
+FROM ${RUNTIME_BASE}
+
+ARG SOURCE_REVISION
+ARG BINARY_SHA256
+ARG BUILD_PROFILE=dev
+COPY rustfs /usr/bin/rustfs
+RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
+import hashlib
+import os
+import re
+import shutil
+import sys
+
+source, digest, profile = sys.argv[1:]
+if not re.fullmatch(r"[0-9a-f]{40}", source):
+    raise SystemExit("invalid source revision")
+if not re.fullmatch(r"[0-9a-f]{64}", digest):
+    raise SystemExit("invalid binary digest")
+if profile not in ("dev", "release"):
+    raise SystemExit("invalid build profile")
+with open("/usr/bin/rustfs", "rb") as binary:
+    if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
+        raise SystemExit("binary digest mismatch")
+for tool in ("getent", "stat"):
+    if shutil.which(tool) is None:
+        raise SystemExit(f"missing runtime tool: {tool}")
+os.chmod("/usr/bin/rustfs", 0o755)
+PY
+LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
+      io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
+      io.rustfs.repro.build_profile="${BUILD_PROFILE}"
+USER 10001:10001
+ENTRYPOINT ["/usr/bin/rustfs"]

.github/workflows/cla.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Run CLA Bot
         if: github.event_name != 'merge_group'
-        uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
+        uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
         with:
           github-token: ${{ github.token }}
           registry-token: ${{ steps.registry-token.outputs.token }}

Cargo.lock

@@ -9612,6 +9612,7 @@ dependencies = [
  "rustfs-log-analyzer",
  "rustfs-madmin",
  "rustfs-mimalloc",
+ "rustfs-mimalloc-sys",
  "rustfs-notify",
  "rustfs-object-capacity",
  "rustfs-object-data-cache",
@@ -9862,6 +9863,7 @@ dependencies = [
  "path-absolutize",
  "pin-project-lite",
  "proptest",
+ "quick-xml",
  "rand 0.10.3",
  "ratelimit",
  "rcgen",
@@ -9894,7 +9896,9 @@ dependencies = [
  "rustfs-s3-ops",
  "rustfs-s3-types",
  "rustfs-scanner-metrics",
+ "rustfs-signer",
  "rustfs-storage-api",
+ "rustfs-tls-runtime",
  "rustfs-uring",
  "rustfs-utils",
  "rustix",
@@ -10896,6 +10900,7 @@ dependencies = [
  "time",
  "tracing",
  "tracing-subscriber",
+ "url",
 ]
 
 [[package]]

crates/data-usage/src/data_usage.rs

@@ -675,6 +675,10 @@ impl DataUsageInfo {
 /// cycle. The timestamp fallback preserves ordering for legacy snapshots that
 /// predate those fields.
 pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
+    data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
+}
+
+fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
     match (
         candidate.scanner_epoch.zip(candidate.scanner_cycle),
         baseline.scanner_epoch.zip(baseline.scanner_cycle),
@@ -697,10 +701,18 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs
 /// already advance the authoritative timestamp when deleting a bucket, so a
 /// rollback delete/recreate fences the previous bucket incarnation too.
 pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
+    observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
+}
+
+/// Return whether a nonconverged observation may supersede `authoritative`.
+///
+/// This applies the structural, exact-baseline, and generation checks of
+/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
+pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
     observed.usage_snapshot_converged == Some(false)
         && (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
-        && observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
-        && data_usage_snapshot_is_newer(observed, authoritative)
+        && observed.usage_snapshot_authoritative_baseline == Some(authoritative)
+        && data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
 }
 
 /// Metadata describing the status of a disk-level data usage snapshot.
@@ -3140,12 +3152,17 @@ mod tests {
         };
 
         assert!(observed_data_usage_is_newer(&observed, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
 
         authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
         assert!(
             !observed_data_usage_is_newer(&observed, &authoritative),
             "an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
         );
+        assert!(
+            !observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+            "a captured identity must preserve the exact namespace fence"
+        );
     }
 
     #[test]
@@ -3172,6 +3189,18 @@ mod tests {
         assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
+        for (observed, expected) in [
+            (candidate(2, 11, Some(false), true), true),
+            (candidate(2, 9, Some(false), true), false),
+            (candidate(2, 11, Some(true), true), false),
+            (candidate(2, 11, Some(false), false), false),
+        ] {
+            assert_eq!(
+                observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+                expected,
+                "observation identity selection must preserve convergence, completeness, and generation gates"
+            );
+        }
 
         let mut partial = candidate(2, 11, Some(false), false);
         partial.usage_snapshot_partial = true;
@@ -3186,6 +3215,30 @@ mod tests {
             segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
+    }
+
+    #[test]
+    fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
+        let baseline = DataUsageSnapshotIdentity {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
+            // An epoch without a cycle is not a coordinated position.
+            scanner_epoch: Some(2),
+            scanner_cycle: None,
+        };
+        let mut observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
+            scanner_epoch: Some(1),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(baseline),
+            ..Default::default()
+        };
+        assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = baseline.last_update;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = None;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
     }
 
     #[test]

crates/e2e_test/src/conditional_write_policy_test.rs

@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
     let key = "guarded.txt";
 
     admin_create_user(env, &user, user_secret).await?;
-    let policy = serde_json::json!({
+    let mut policy = serde_json::json!({
         "Version": "2012-10-17",
         "Statement": [{
             "Effect": "Allow",
@@ -268,9 +268,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
             "Resource": [format!("arn:aws:s3:::{bucket}/*")],
             "Condition": { "Null": { "s3:if-match": "false" } }
         }]
-    })
-    .to_string();
-    admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
+    });
+    admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
 
     admin.create_bucket().bucket(&bucket).send().await?;
     let original = admin
@@ -293,6 +292,46 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .expect_err("PutObject without If-Match must be denied");
     assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
 
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .if_match(&etag)
+        .body(ByteStream::from_static(b"missing-read-permission"))
+        .send()
+        .await
+        .expect_err("If-Match without GetObject must be denied");
+    assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
+
+    // S3 requires GetObject as well as PutObject for an ETag-based write.
+    policy["Statement"]
+        .as_array_mut()
+        .expect("policy statements")
+        .push(serde_json::json!({
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
+        }));
+    admin_add_canned_policy_via(
+        AdminTransport::Signed,
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        &policy_name,
+        &policy.to_string(),
+    )
+    .await?;
+
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"blind-overwrite"))
+        .send()
+        .await
+        .expect_err("GetObject permission must not allow an unconditional PutObject");
+    assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
+
     client
         .put_object()
         .bucket(&bucket)
@@ -301,6 +340,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .body(ByteStream::from_static(b"v2"))
         .send()
         .await?;
+    let written = client.get_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
 
     let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
     let _ = admin.delete_bucket().bucket(&bucket).send().await;

crates/e2e_test/src/protocols/README.md

@@ -77,9 +77,7 @@ core protocol surface plus cross-protocol consistency:
   existing path returns an error, bad-password authentication is rejected
 - Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
   S3 write then SFTP read with SHA256 match
-- Cross-API directory visibility: SFTP-created sub-directory visible via S3
-  ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
-  directory entry
+- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
 
 #### sftp_compliance (`test_sftp_compliance_suite`)
 

crates/e2e_test/src/protocols/sftp_core.rs

@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
 
         // --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
-        // SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
-        // decodes that marker back to a trailing-slash key, so the asserted
-        // pattern is "subdir_sftp/".
+        // SFTP mkdir writes a zero-byte object at the trailing-slash key.
+        // Storage encodes that key to the __XLDIR__ marker, and the S3
+        // listing path decodes the marker back to a trailing-slash key.
         info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
         let sftp_subdir_name = "subdir_sftp";
         let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
@@ -422,20 +422,21 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         );
         info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
 
-        // --- Cross-API directory visibility: S3 marker, SFTP readdir ---
-        // aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
-        // __XLDIR__ suffix. SFTP readdir must decode the marker back to a
-        // bare directory entry whose file_type reports as a directory.
-        info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
+        // --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
+        // aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
+        // key. The __XLDIR__ suffix is reserved on the client-key boundary, so
+        // a direct put of that suffix is rejected. Storage encodes the slash
+        // key, and SFTP readdir must show a directory entry.
+        info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
         let s3_subdir_name = "subdir_s3";
-        let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
+        let s3_subdir_key = format!("{s3_subdir_name}/");
         s3.put_object()
             .bucket(bucket)
-            .key(&s3_subdir_marker_key)
+            .key(&s3_subdir_key)
             .body(ByteStream::from_static(b""))
             .send()
             .await
-            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
+            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
         let bucket_entries: Vec<(String, bool)> = sftp
             .read_dir(&bucket_path)
             .await?
@@ -444,9 +445,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
         assert!(
             visible_via_sftp,
-            "S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
+            "S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
         );
-        info!("PASS: S3 marker visible to SFTP readdir as a directory");
+        info!("PASS: S3 directory object visible to SFTP readdir as a directory");
 
         // --- Pre-cleanup of cross-protocol fixtures ---
         // Removes the new files and sub-directories so the existing rmdir

crates/e2e_test/src/quota_test.rs

@@ -412,6 +412,38 @@ mod integration_tests {
         );
     }
 
+    async fn upload_after_quota_reconciliation(
+        env: &QuotaTestEnv,
+        key: &str,
+        size_bytes: usize,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let upload = async {
+            loop {
+                let result = env
+                    .client
+                    .put_object()
+                    .bucket(&env.bucket_name)
+                    .key(key)
+                    .body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
+                    .send()
+                    .await;
+                match result {
+                    Ok(_) => break,
+                    Err(error) => assert_quota_rejection(
+                        error.raw_response().map(|response| response.status().as_u16()),
+                        error.as_service_error(),
+                        &error,
+                    ),
+                }
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        // The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
+        timeout(Duration::from_secs(45), upload)
+            .await
+            .map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
+    }
+
     async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
         let error = env
             .client
@@ -573,8 +605,7 @@ mod integration_tests {
 
         assert!(!env.object_exists("file1.txt").await?);
 
-        // Now we should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;
@@ -981,8 +1012,7 @@ mod integration_tests {
         );
         assert_eq!(delete_result.deleted().len(), 2);
 
-        // Now should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -42,6 +42,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
         success: response.success,
         error_info: response.error,
         lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
+        ..Default::default()
     }
 }
 
@@ -50,6 +51,7 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
         success: false,
         error_info: Some(error.into()),
         lock_info: None,
+        ..Default::default()
     }
 }
 
@@ -59,6 +61,7 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
             success: true,
             error_info: None,
             lock_info: None,
+            ..Default::default()
         }
     } else {
         lock_result_from_error(format!("lock not found for release: {lock_id}"))
@@ -164,6 +167,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -175,6 +179,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error,
                     lock_info: lock_info_json,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -184,6 +189,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -197,6 +203,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -208,6 +215,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -217,6 +225,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -230,6 +239,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -241,6 +251,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -250,6 +261,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -263,6 +275,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -272,11 +285,13 @@ impl NodeService for MinimalLockNodeService {
                 success,
                 error_info: None,
                 lock_info: None,
+                ..Default::default()
             })),
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!("can not refresh, err: {err}")),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }

crates/ecstore/Cargo.toml

@@ -31,7 +31,7 @@ workspace = true
 
 [features]
 default = []
-fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]
@@ -203,6 +203,9 @@ libc.workspace = true
 # (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
 rustix = { workspace = true, features = ["process", "fs", "thread"] }
 rustfs-madmin.workspace = true
+rustfs-signer.workspace = true
+rustfs-tls-runtime.workspace = true
+quick-xml.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }

crates/ecstore/src/api/mod.rs

@@ -191,7 +191,10 @@ pub mod bucket {
             OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
             table_catalog_path_hash, unreadable_config_refusal,
         };
-        pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{
+            BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
+            SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX,
+        };
         pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
     }
 
@@ -210,11 +213,11 @@ pub mod bucket {
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
-            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            ObjectLockConfigState, ScannerBucketIncarnationReadGuard, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
-            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
-            delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
-            get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
+            acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
+            delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
+            get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
@@ -450,6 +453,7 @@ pub mod data_usage {
 pub mod disk {
     pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
     pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
+    pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
     pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,15 +256,15 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
-/// Shared tag-entry validation for object requests and persisted bucket tags.
+/// Validate tag entries supplied to the S3 API.
 pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
-    fn is_valid_tag_text(value: &str) -> bool {
-        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
-        VALID_TAG_TEXT
-            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
-            .as_ref()
-            .is_ok_and(|pattern| pattern.is_match(value))
-    }
+    validate_tag_entries(tag_set, validate_s3_tag_entry)
+}
+
+fn validate_tag_entries(
+    tag_set: &[s3s::dto::Tag],
+    mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
+) -> std::result::Result<(), &'static str> {
     let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
     for tag in tag_set {
         let key = tag
@@ -272,13 +272,8 @@ pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Resul
             .as_deref()
             .filter(|key| !key.is_empty())
             .ok_or("Tag key cannot be empty")?;
-        if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
-            return Err("The TagKey you have provided is invalid");
-        }
         let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
-        if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
-            return Err("The TagValue you have provided is invalid");
-        }
+        validate_entry(key, value)?;
         if !keys.insert(key) {
             return Err("Cannot provide multiple Tags with the same key");
         }
@@ -286,8 +281,86 @@ pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Resul
     Ok(())
 }
 
+fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+        return Err("The TagKey you have provided is invalid");
+    }
+    if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+        return Err("The TagValue you have provided is invalid");
+    }
+    Ok(())
+}
+
 pub const MAX_BUCKET_TAGS: usize = 50;
 
+/// Maximum number of metadata headers on a Swift resource.
+pub const MAX_SWIFT_METADATA_COUNT: usize = 90;
+/// Maximum byte length of a Swift metadata value.
+pub const MAX_SWIFT_METADATA_VALUE_SIZE: usize = 256;
+
+/// Prefix for Swift account metadata persisted in tagging.xml.
+pub const SWIFT_ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
+/// Prefix for Swift container metadata persisted in tagging.xml.
+pub const SWIFT_CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
+
+fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    const SWIFT_ACL_MAX_BYTES: usize = 8192;
+    const SWIFT_META_NAME_MAX_BYTES: usize = 128;
+    const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 3;
+    if tags.len() > MAX_PERSISTED_TAGS {
+        return Err("Too many persisted bucket tags");
+    }
+
+    // Swift stores ACLs, versioning and metadata in the same XML as S3 tags.
+    // Only those known keys may carry values outside the S3 tag alphabet.
+    let mut account_metadata = 0;
+    let mut container_metadata = 0;
+    let mut s3_tags = 0;
+    validate_tag_entries(tags, |key, value| {
+        let swift_metadata = key
+            .strip_prefix(SWIFT_ACCOUNT_META_TAG_PREFIX)
+            .map(|name| (name, &mut account_metadata))
+            .or_else(|| {
+                key.strip_prefix(SWIFT_CONTAINER_META_TAG_PREFIX)
+                    .map(|name| (name, &mut container_metadata))
+            });
+        if let Some((name, count)) = swift_metadata
+            && name.len() <= SWIFT_META_NAME_MAX_BYTES
+            && http::HeaderName::from_bytes(name.as_bytes()).is_ok()
+            && value.len() <= MAX_SWIFT_METADATA_VALUE_SIZE
+            && http::HeaderValue::from_str(value).is_ok()
+        {
+            *count += 1;
+            if *count > MAX_SWIFT_METADATA_COUNT {
+                return Err("Too many Swift metadata tags");
+            }
+            return Ok(());
+        }
+
+        let valid_swift_header = match key {
+            "swift-acl-read" | "swift-acl-write" => value.len() <= SWIFT_ACL_MAX_BYTES,
+            "swift-versions-location" => value.len() <= 256 && !value.is_empty() && !value.contains('/'),
+            _ => false,
+        };
+        if valid_swift_header && http::HeaderValue::from_str(value).is_ok() {
+            return Ok(());
+        }
+
+        s3_tags += 1;
+        if s3_tags > MAX_BUCKET_TAGS {
+            return Err("Cannot have more than 50 tags per bucket");
+        }
+        validate_s3_tag_entry(key, value)
+    })
+}
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -1026,14 +1099,7 @@ impl BucketMetadata {
                     let tagging = deserialize::<Tagging>(&data).map_err(|_| {
                         Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
                     })?;
-                    if tagging.tag_set.len() > MAX_BUCKET_TAGS {
-                        return Err(Error::InvalidArgument(
-                            self.name.clone(),
-                            String::new(),
-                            "Cannot have more than 50 tags per bucket".to_owned(),
-                        ));
-                    }
-                    validate_tag_set_entries(&tagging.tag_set)
+                    validate_persisted_bucket_tags(&tagging.tag_set)
                         .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
                 }
                 self.tagging_config_xml = data;
@@ -1827,6 +1893,40 @@ mod test {
         assert_eq!(metadata.bucket_incarnation_id, incarnation);
     }
 
+    #[test]
+    fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
+        use s3s::dto::Tag;
+
+        let tag = |key: String, value: &str| Tag {
+            key: Some(key),
+            value: Some(value.to_owned()),
+        };
+        let acl = tag("swift-acl-read".to_owned(), ".r:*,AUTH_other");
+        assert_eq!(
+            validate_tag_set_entries(std::slice::from_ref(&acl)),
+            Err("The TagValue you have provided is invalid")
+        );
+
+        let mut tags = vec![acl.clone()];
+        tags.extend((0..MAX_BUCKET_TAGS).map(|i| tag(format!("team-{i}"), "blue")));
+        tags.extend((0..MAX_SWIFT_METADATA_COUNT).map(|i| tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}item!{i}"), "a,b")));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Ok(()));
+
+        tags.push(tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}overflow"), "a,b"));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("Too many Swift metadata tags"));
+        tags.pop();
+
+        tags.push(acl);
+        assert_eq!(
+            validate_persisted_bucket_tags(&tags),
+            Err("Cannot provide multiple Tags with the same key")
+        );
+        tags.pop();
+
+        tags[0].value = Some(".r:*,".repeat(1700));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("The TagValue you have provided is invalid"));
+    }
+
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.

crates/ecstore/src/bucket/metadata_sys.rs

@@ -22,6 +22,7 @@ use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
 use crate::bucket::utils::is_meta_bucketname;
+use crate::diagnostics::object_lock::ObjectLockAttempt;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
 use crate::object_api::ObjectOptions;
@@ -599,7 +600,9 @@ pub(crate) async fn get_bucket_incarnation_id_for_options_in(
     }
     let sys = bucket_metadata_sys_of(ctx)?;
     let sys = sys.read().await.clone();
-    let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
+    let guard =
+        acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts, "bucket_incarnation_options_read")
+            .await?;
     sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
 }
 
@@ -866,6 +869,30 @@ impl BucketMetadataMutationGuard {
     }
 }
 
+/// Read-only scanner authority for one bucket incarnation.
+///
+/// Lock order is bucket lifecycle (read), then metadata transaction (read).
+/// This guard cannot authorize a configuration mutation or scanner ACK write.
+pub struct ScannerBucketIncarnationReadGuard {
+    bucket: String,
+    incarnation_id: Uuid,
+    lifecycle_guard: rustfs_lock::NamespaceLockGuard,
+    transaction_guard: rustfs_lock::NamespaceLockGuard,
+}
+
+impl ScannerBucketIncarnationReadGuard {
+    /// Returns the verified identity while both read fences remain valid.
+    ///
+    /// # Errors
+    /// Returns an error if either lock has lost its lease.
+    pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
+        if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() {
+            return Err(Error::other("scanner bucket incarnation read lock was lost"));
+        }
+        Ok((&self.bucket, self.incarnation_id))
+    }
+}
+
 async fn acquire_config_write_guard(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str) -> Result<BucketMetadataMutationGuard> {
     acquire_config_write_guard_for_incarnation(sys, bucket, None).await
 }
@@ -892,6 +919,68 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
     acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
 }
 
+/// Validate scanner read authority without serializing compatible metadata readers.
+///
+/// The returned guard retains both read locks. It never migrates metadata or
+/// creates an incarnation; mutation callers must use the exclusive scanner fence.
+///
+/// # Errors
+/// Returns an error for invalid bucket/owner identities, unavailable locks or
+/// storage, absent incarnation metadata, or a deleted/recreated bucket.
+pub async fn acquire_scanner_bucket_incarnation_read_fence(
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    acquire_scanner_bucket_incarnation_read_fence_with_sys(
+        get_bucket_metadata_sys()?,
+        bucket,
+        expected_incarnation_id,
+        expected_owner_id,
+    )
+    .await
+}
+
+async fn acquire_scanner_bucket_incarnation_read_fence_with_sys(
+    sys: Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    super::utils::check_valid_bucket_name(bucket)?;
+    let metadata_sys = sys.read().await.clone();
+    if expected_owner_id.is_nil() || metadata_sys.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
+        return Err(Error::other("scanner bucket incarnation owner does not match"));
+    }
+    let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
+    let transaction_guard = await_bucket_namespace_operation(
+        Some(&lifecycle_guard),
+        bucket,
+        "scanner incarnation read transaction lock acquisition",
+        acquire_metadata_transaction_guard(
+            metadata_sys.object_store(),
+            bucket,
+            "scanner_incarnation_probe",
+            rustfs_lock::LockType::Shared,
+        ),
+    )
+    .await?;
+    let incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        Some(expected_incarnation_id),
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+    Ok(ScannerBucketIncarnationReadGuard {
+        bucket: bucket.to_owned(),
+        incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
 async fn acquire_config_write_guard_with_migration(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
@@ -917,15 +1006,47 @@ async fn acquire_config_write_guard_with_migration(
         Some(&lifecycle_guard),
         bucket,
         "bucket config transaction lock acquisition",
-        acquire_transaction_lock_with_sys(&sys, bucket),
+        acquire_transaction_lock_with_sys_for_purpose(
+            &sys,
+            bucket,
+            if migrate {
+                "bucket_config_update"
+            } else {
+                "scanner_incarnation_validation"
+            },
+        ),
     )
     .await?;
+    let current_incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        expected_incarnation_id,
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+
+    Ok(BucketMetadataMutationGuard {
+        bucket: bucket.to_string(),
+        incarnation_id: current_incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
+async fn validate_bucket_metadata_guard_incarnation(
+    metadata_sys: &BucketMetadataSys,
+    bucket: &str,
+    expected_incarnation_id: Option<Uuid>,
+    lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
+    transaction_guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<Uuid> {
     await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config existence validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config existence transaction validation",
             async {
@@ -943,11 +1064,11 @@ async fn acquire_config_write_guard_with_migration(
     )
     .await?;
     let current_incarnation_id = await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config incarnation validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config incarnation transaction validation",
             load_bucket_incarnation(metadata_sys.object_store(), bucket),
@@ -960,12 +1081,7 @@ async fn acquire_config_write_guard_with_migration(
         return Err(Error::BucketNotFound(bucket.to_string()));
     }
 
-    Ok(BucketMetadataMutationGuard {
-        bucket: bucket.to_string(),
-        incarnation_id: current_incarnation_id,
-        lifecycle_guard,
-        transaction_guard,
-    })
+    Ok(current_incarnation_id)
 }
 
 /// Rewrite one config file while the caller already holds this bucket's
@@ -1235,13 +1351,18 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
 pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, "bucket_metadata_read").await
+}
+
+pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     let sys = bucket_metadata_sys_of(ctx)?;
     let api = sys.read().await.object_store();
-    let lock = api
-        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-        .await?;
-    Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
+    acquire_metadata_transaction_guard(api, bucket, op, rustfs_lock::LockType::Shared).await
 }
 
 /// Readers already holding an Object Lock snapshot must share its guard:
@@ -1250,6 +1371,7 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
     opts: &ObjectOptions,
+    op: &'static str,
 ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
     if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
         let store = object_store_in(ctx).await?;
@@ -1259,12 +1381,22 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
                 Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
             });
     }
-    Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
+    Ok(Arc::new(
+        acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, op).await?,
+    ))
 }
 
 async fn acquire_transaction_lock_with_sys(
     sys: &Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_transaction_lock_with_sys_for_purpose(sys, bucket, "bucket_metadata_transaction_write").await
+}
+
+async fn acquire_transaction_lock_with_sys_for_purpose(
+    sys: &Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     // Resolve the store under a short-lived read guard: this runs before the
     // write guard in `acquire_config_write_guards`, and must not still hold a
@@ -1273,12 +1405,22 @@ async fn acquire_transaction_lock_with_sys(
     let lock = api
         .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
         .await?;
-    let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mut attempt = ObjectLockAttempt::start(
+        op,
+        RUSTFS_META_BUCKET,
+        &bucket_metadata_transaction_lock_key(bucket),
+        None,
+        &lock,
+        "write",
+        timeout,
+    );
+    let acquire = lock.get_write_lock(timeout);
     #[cfg(any(test, feature = "test-util"))]
     {
         tokio::pin!(acquire);
         let mut notified = false;
-        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+        let result = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
             std::task::Poll::Pending => {
                 if !notified {
                     notify_config_write_lock_attempt(bucket);
@@ -1288,11 +1430,38 @@ async fn acquire_transaction_lock_with_sys(
             }
             std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
         })
-        .await?;
-        Ok(guard)
+        .await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
     }
     #[cfg(not(any(test, feature = "test-util")))]
-    Ok(acquire.await?)
+    {
+        let result = acquire.await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
+    }
+}
+
+pub(crate) async fn acquire_metadata_transaction_guard(
+    api: Arc<ECStore>,
+    bucket: &str,
+    op: &'static str,
+    lock_type: rustfs_lock::LockType,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    let object = bucket_metadata_transaction_lock_key(bucket);
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mode = match lock_type {
+        rustfs_lock::LockType::Shared => "read",
+        rustfs_lock::LockType::Exclusive => "write",
+    };
+    let mut attempt = ObjectLockAttempt::start(op, RUSTFS_META_BUCKET, &object, None, &lock, mode, timeout);
+    let result = match lock_type {
+        rustfs_lock::LockType::Shared => lock.get_read_lock(timeout).await,
+        rustfs_lock::LockType::Exclusive => lock.get_write_lock(timeout).await,
+    };
+    attempt.observe_acquisition(&result);
+    Ok(result?)
 }
 
 /// The lock resource name is deliberately still the `bucket-targets` one it
@@ -2794,13 +2963,13 @@ impl BucketMetadataSys {
     }
 
     async fn find_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Option<Uuid>> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_incarnation_disk_read",
+            rustfs_lock::LockType::Shared,
+        )
+        .await?;
         self.find_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
             .await
     }
@@ -2876,13 +3045,13 @@ impl BucketMetadataSys {
     /// revalidated at write quorum, so a stale snapshot can neither revert an
     /// acknowledged configuration update nor outlive a delete/recreate.
     async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_creation_commit_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
         let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
@@ -2932,13 +3101,13 @@ impl BucketMetadataSys {
     }
 
     async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_metadata_legacy_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
 
         let authority = self
             .load_authoritative_metadata_from_disk_under_transaction_lock(bucket)
@@ -3977,6 +4146,12 @@ mod tests {
                 .await
                 .is_err()
         );
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "read-only scanner validation must not migrate an absent incarnation"
+        );
         assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
         assert!(
             sys.read()
@@ -4033,6 +4208,128 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn scanner_incarnation_read_fence_shares_reads_and_blocks_mutations() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
+        let bucket = "scanner-read-fence";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("bucket incarnation");
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+            .await
+            .expect("read-only scanner fence");
+        assert_eq!(guard.checked_bucket_incarnation().expect("valid read fences"), (bucket, incarnation));
+
+        let transaction_lock = store
+            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
+            .await
+            .expect("metadata transaction namespace");
+        assert!(
+            transaction_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude metadata writers"
+        );
+        let lifecycle_lock = store
+            .new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
+            .await
+            .expect("bucket lifecycle namespace");
+        assert!(
+            lifecycle_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude delete and recreation"
+        );
+        let another = timeout(
+            Duration::from_secs(1),
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id),
+        )
+        .await
+        .expect("scanner readers must not serialize")
+        .expect("second read-only scanner fence");
+        let compatible = timeout(
+            Duration::from_secs(1),
+            acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket),
+        )
+        .await
+        .expect("metadata readers must remain compatible")
+        .expect("metadata transaction reader");
+        drop(compatible);
+        drop(another);
+
+        let mut mutation = Box::pin(acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(incarnation), false));
+        assert!(matches!(futures::poll!(&mut mutation), std::task::Poll::Pending));
+        let mut deletion = Box::pin(store.acquire_bucket_lifecycle_write_lock(bucket));
+        assert!(matches!(futures::poll!(&mut deletion), std::task::Poll::Pending));
+        drop(deletion);
+        drop(guard);
+        let mutation = timeout(Duration::from_secs(1), mutation)
+            .await
+            .expect("metadata mutation must progress after scanner read release")
+            .expect("exclusive mutation fence");
+        drop(mutation);
+        let lifecycle = timeout(Duration::from_secs(1), store.acquire_bucket_lifecycle_write_lock(bucket))
+            .await
+            .expect("bucket delete fence must progress after readers release")
+            .expect("exclusive lifecycle fence");
+        drop(lifecycle);
+
+        for (owner, expected) in [
+            (Uuid::new_v4(), incarnation),
+            (Uuid::nil(), incarnation),
+            (store.id, Uuid::nil()),
+        ] {
+            assert!(
+                acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, expected, owner)
+                    .await
+                    .is_err(),
+                "invalid owner or nil incarnation must not receive authority"
+            );
+        }
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "a stale incarnation must not receive authority"
+        );
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete bucket");
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "deleted bucket must not receive authority"
+        );
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let replacement = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("replacement incarnation");
+        assert_ne!(replacement, incarnation);
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "recreation must reject the previous scanner identity"
+        );
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys, bucket, replacement, store.id)
+            .await
+            .expect("replacement read-only scanner fence");
+        assert_eq!(
+            guard.checked_bucket_incarnation().expect("replacement remains fenced"),
+            (bucket, replacement)
+        );
+    }
+
     #[tokio::test]
     async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;

crates/ecstore/src/bucket/quota/reservation.rs

@@ -886,7 +886,13 @@ pub(crate) async fn begin(
     }
 
     let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
-    let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
+    let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(
+        ctx,
+        bucket,
+        opts,
+        "quota_context_metadata_read",
+    )
+    .await?;
     rustfs_io_metrics::record_put_object_stage_duration_from(
         rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
         metadata_lock_started,

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -366,10 +366,84 @@ fn resolve_replay_cache_capacity() -> usize {
     decision.capacity
 }
 
+const RPC_NONCE_EXPIRATIONS_PER_PAGE: usize = 1_024;
+
+struct RpcNonceExpiration {
+    nonce: Uuid,
+    expiry_offset_ns: u32,
+    valid_until_offset_secs: i32,
+}
+
+struct RpcNonceExpirationPage {
+    expires_at: Instant,
+    valid_until: i64,
+    entries: VecDeque<RpcNonceExpiration>,
+}
+
+#[derive(Default)]
+struct RpcNonceExpirations {
+    pages: VecDeque<RpcNonceExpirationPage>,
+}
+
+impl RpcNonceExpirations {
+    fn push_back(&mut self, expires_at: Instant, valid_until: i64, nonce: Uuid) {
+        // Use exact offsets and start another page if either offset is too large.
+        // Fully drained pages can be freed while newer nonces are still valid.
+        if let Some(page) = self.pages.back_mut()
+            && page.entries.len() < RPC_NONCE_EXPIRATIONS_PER_PAGE
+            && let Some(offset) = expires_at.checked_duration_since(page.expires_at)
+            && let Ok(expiry_offset_ns) = u32::try_from(offset.as_nanos())
+            && let Some(offset) = valid_until.checked_sub(page.valid_until)
+            && let Ok(valid_until_offset_secs) = i32::try_from(offset)
+        {
+            page.entries.push_back(RpcNonceExpiration {
+                nonce,
+                expiry_offset_ns,
+                valid_until_offset_secs,
+            });
+        } else {
+            self.pages.push_back(RpcNonceExpirationPage {
+                expires_at,
+                valid_until,
+                entries: VecDeque::from([RpcNonceExpiration {
+                    nonce,
+                    expiry_offset_ns: 0,
+                    valid_until_offset_secs: 0,
+                }]),
+            });
+        }
+    }
+
+    fn pop_expired(&mut self, now: Instant, wall_time: i64) -> Option<Uuid> {
+        let page = self.pages.front_mut()?;
+        let entry = page.entries.front()?;
+        let elapsed = now.checked_duration_since(page.expires_at)?;
+        let valid_until = page.valid_until + i64::from(entry.valid_until_offset_secs);
+        if elapsed <= Duration::from_nanos(u64::from(entry.expiry_offset_ns)) || wall_time <= valid_until {
+            return None;
+        }
+
+        let nonce = entry.nonce;
+        page.entries.pop_front();
+        if page.entries.is_empty() {
+            self.pages.pop_front();
+            if self.pages.is_empty() {
+                self.pages.shrink_to_fit();
+            }
+        }
+        Some(nonce)
+    }
+
+    #[cfg(test)]
+    fn capacity(&self) -> usize {
+        self.pages.iter().map(|page| page.entries.capacity()).sum()
+    }
+}
+
 #[derive(Default)]
 struct RpcNonceCache {
     nonces: HashSet<Uuid>,
-    expirations: VecDeque<(Instant, i64, Uuid)>,
+    expirations: RpcNonceExpirations,
     max_wall_time: i64,
 }
 
@@ -425,16 +499,13 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
 impl RpcNonceCache {
     fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
         let mut removed = 0;
-        while matches!(
-            self.expirations.front(),
-            Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
-        ) {
-            let Some((_, _, nonce)) = self.expirations.pop_front() else {
-                break;
-            };
+        while let Some(nonce) = self.expirations.pop_expired(now, wall_time) {
             self.nonces.remove(&nonce);
             removed += 1;
         }
+        if removed > 0 && self.nonces.is_empty() {
+            self.nonces.shrink_to_fit();
+        }
         removed
     }
 
@@ -468,7 +539,7 @@ impl RpcNonceCache {
         }
         self.nonces.insert(record.nonce);
         self.expirations
-            .push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
+            .push_back(record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce);
         (
             Ok(()),
             Some(RpcNonceCacheMetrics {
@@ -3178,6 +3249,51 @@ mod tests {
         assert!(cache.nonces.contains(&nonce_b));
     }
 
+    #[test]
+    fn nonce_cache_releases_expiration_storage_after_a_burst() {
+        let start = Instant::now();
+        let expiry = start + REPLAY_CACHE_RETENTION;
+        let mut cache = RpcNonceCache::default();
+
+        for nonce in 1..=4_096 {
+            let record = test_nonce_record(Uuid::from_u128(nonce), 100, start, 100, expiry, 4_096);
+            cache.check_and_record(record).0.expect("first use of nonce");
+        }
+        assert!(cache.expirations.capacity() >= 4_096);
+        assert_eq!(cache.remove_expired(expiry, 701), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 400), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 701), 4_096);
+        assert!(cache.nonces.is_empty());
+        assert_eq!(cache.nonces.capacity(), 0);
+        assert_eq!(cache.expirations.capacity(), 0, "expired entries must release their expiration storage");
+    }
+
+    #[test]
+    fn nonce_expirations_preserve_exact_deadlines_across_pages() {
+        let expiry = Instant::now();
+        let first = Uuid::from_u128(1);
+        let second = Uuid::from_u128(2);
+        let third = Uuid::from_u128(3);
+        let fourth = Uuid::from_u128(4);
+        let mut queue = super::RpcNonceExpirations::default();
+        let next = expiry + Duration::from_nanos(1);
+        let later = expiry + Duration::from_secs(5);
+
+        queue.push_back(expiry, 501, first);
+        queue.push_back(next, 500, second);
+        queue.push_back(later, 505, third);
+        queue.push_back(later, i64::MAX, fourth);
+        assert_eq!(queue.pages.len(), 3);
+        assert_eq!(queue.pop_expired(next, 501), None);
+        assert_eq!(queue.pop_expired(next, 502), Some(first));
+        assert_eq!(queue.pop_expired(next, 502), None);
+        assert_eq!(queue.pop_expired(later, 502), Some(second));
+        assert_eq!(queue.pages.len(), 2);
+        assert_eq!(queue.pop_expired(later, 506), None);
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), 506), Some(third));
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), i64::MAX), None);
+    }
+
     #[test]
     fn nonce_cache_metrics_mark_successful_records_only() {
         let now = Instant::now();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -27,13 +27,13 @@ use crate::disk::multipart_source::{
     MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, MultipartSourceResponse,
 };
 use crate::disk::{
-    BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
-    DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
-    DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
-    DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
-    PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse, ReadMultipleReq, ReadMultipleResp,
-    ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
-    batch_read_version_one_by_one,
+    BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate,
+    DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
+    DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo,
+    DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload,
+    GuardedFileInfoVersions, PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo,
+    WalkDirOptions, batch_read_version_one_by_one,
     disk_store::{
         DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
         get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -996,21 +996,32 @@ fn internode_idempotent_read_retries() -> usize {
 static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
     std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
 
+fn claim_control_channel_prewarm(addr: &str) -> bool {
+    let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
+        return false;
+    };
+    prewarmed.insert(addr.to_owned())
+}
+
+fn release_control_channel_prewarm(addr: &str) {
+    if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
+        prewarmed.remove(addr);
+    }
+}
+
 /// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
-/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
+/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
 fn spawn_control_channel_prewarm(addr: String) {
-    {
-        let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
-            return;
-        };
-        if !prewarmed.insert(addr.clone()) {
-            return;
-        }
+    if !claim_control_channel_prewarm(&addr) {
+        return;
     }
     tokio::spawn(async move {
         match node_service_time_out_client_no_auth(&addr).await {
             Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
-            Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
+            Err(err) => {
+                release_control_channel_prewarm(&addr);
+                debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
+            }
         }
     });
 }
@@ -1021,6 +1032,7 @@ impl RemoteDisk {
         volume: &str,
         payload: Vec<u8>,
         probe_only: bool,
+        bounded_prepare: bool,
     ) -> Result<DeleteVersionsResponse> {
         if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
             return Err(DiskError::FileCorrupt);
@@ -1045,6 +1057,8 @@ impl RemoteDisk {
                     .map_err(|status| {
                         if status.code() == Code::Unimplemented {
                             DiskError::GuardedDeleteUnsupported
+                        } else if bounded_prepare && status.code() == Code::InvalidArgument {
+                            DiskError::UnsupportedDisk
                         } else {
                             status.into()
                         }
@@ -1078,6 +1092,14 @@ impl RemoteDisk {
         volume: &str,
         operation: DeleteRecoveryRpcOperation,
     ) -> Result<DeleteRecoveryRpcReply> {
+        let protocol_version = match &operation {
+            DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. }
+                if opts.requires_bounded_delete_recovery =>
+            {
+                BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION
+            }
+            _ => DELETE_RECOVERY_PROTOCOL_VERSION,
+        };
         let payload = rmp_serde::to_vec_named(&operation)?;
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
@@ -1108,7 +1130,7 @@ impl RemoteDisk {
                 let mut request = Request::new(RecoverDeleteTransactionRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_owned(),
-                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                    protocol_version,
                     operation: payload.into(),
                 });
                 let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref())
@@ -1126,7 +1148,7 @@ impl RemoteDisk {
                         }
                     })?
                     .into_inner();
-                if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION {
+                if response.protocol_version != protocol_version {
                     return Err(DiskError::UnsupportedDisk);
                 }
                 if !response.success {
@@ -2768,7 +2790,7 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
-        let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
+        let response = self.guarded_delete_versions_rpc("", Vec::new(), true, false).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
             return Err(DiskError::FileCorrupt);
         }
@@ -2781,7 +2803,12 @@ impl DiskAPI for RemoteDisk {
         opts: DeleteOptions,
     ) -> Vec<Option<DiskError>> {
         let count = versions.len();
-        let payload = GuardedDeleteVersionsPayload { versions, opts };
+        let bounded_prepare = opts.requires_bounded_delete_recovery;
+        let payload = GuardedDeleteVersionsPayload {
+            versions,
+            opts,
+            bounded_recovery_protocol: bounded_prepare.then_some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION),
+        };
         if let Err(error) = payload.validate() {
             return vec![Some(error); count];
         }
@@ -2789,7 +2816,10 @@ impl DiskAPI for RemoteDisk {
             Ok(encoded) => encoded,
             Err(error) => return vec![Some(error.into()); count],
         };
-        match self.guarded_delete_versions_rpc(volume, encoded, false).await {
+        match self
+            .guarded_delete_versions_rpc(volume, encoded, false, bounded_prepare)
+            .await
+        {
             Ok(response) => {
                 // New guarded peers must return typed errors, never lossy strings.
                 if response.item_errors.len() != count {
@@ -3680,7 +3710,13 @@ impl DiskAPI for RemoteDisk {
                         );
                     }
                     let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
-                    let response = match client.batch_read_version(request).await {
+                    let response_result = client.batch_read_version(request).await;
+                    if let Some(rpc_started) = rpc_started {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
+                            rpc_started.elapsed(),
+                        );
+                    }
+                    let response = match response_result {
                         Ok(response) => {
                             record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                             response.into_inner()
@@ -4695,7 +4731,10 @@ impl DiskAPI for RemoteDisk {
                     read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
-                let response = client.read_multiple(request).await?.into_inner();
+                let rpc_started = Instant::now();
+                let response = client.read_multiple(request).await;
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
+                let response = response?.into_inner();
 
                 if !response.success {
                     return Err(response.error.unwrap_or_default().into());
@@ -4746,6 +4785,7 @@ impl DiskAPI for RemoteDisk {
                 attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.write_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4754,6 +4794,7 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
 
                 if !response.success {
@@ -4851,6 +4892,7 @@ impl DiskAPI for RemoteDisk {
                 });
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.read_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4859,6 +4901,8 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
+
                 if !response.success {
                     crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
                     return Err(response.error.unwrap_or_default().into());
@@ -11975,4 +12019,15 @@ mod tests {
 
         accept_task.abort();
     }
+
+    #[test]
+    fn prewarm_claim_is_deduplicated_and_released_after_failure() {
+        let addr = "http://prewarm-retry-test.invalid:9000";
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        assert!(!claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+    }
 }

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -20,12 +20,12 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use futures::{FutureExt, future::BoxFuture};
 use rustfs_lock::{
-    LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
+    LockCleanupOutcome, LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
 };
 use rustfs_protos::proto_gen::node_service::{
     BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult,
-    PingRequest,
+    LockReleaseStatus, PingRequest,
 };
 use rustfs_protos::{
     ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
@@ -281,6 +281,31 @@ fn acquired_lock_ids(lock_ids: &[LockId], results: &[GenerallyLockResult]) -> Ve
         .collect()
 }
 
+fn cleanup_outcome(success: bool, error_info: Option<&str>, release_status: i32) -> Result<LockCleanupOutcome> {
+    let status =
+        LockReleaseStatus::try_from(release_status).map_err(|_| LockError::internal("Unknown remote lock release status"))?;
+    match status {
+        LockReleaseStatus::AlreadyAbsent if !success => Ok(LockCleanupOutcome::AlreadyAbsent),
+        LockReleaseStatus::Released if success && error_info.is_none() => Ok(LockCleanupOutcome::Released),
+        LockReleaseStatus::Unspecified => {
+            if let Some(error) = error_info {
+                Err(LockError::internal(error))
+            } else if success {
+                Ok(LockCleanupOutcome::Released)
+            } else {
+                Ok(LockCleanupOutcome::Unconfirmed)
+            }
+        }
+        _ => Err(LockError::internal("Inconsistent remote lock release status")),
+    }
+}
+
+#[derive(Clone, Copy)]
+enum AcquisitionCleanup {
+    Confirmed,
+    Uncertain,
+}
+
 /// Remote lock client implementation
 #[derive(Debug, Clone)]
 pub struct RemoteClient {
@@ -290,13 +315,18 @@ pub struct RemoteClient {
 impl RemoteClient {
     const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
 
-    fn record_release_ack(&self, lock_id: &LockId) {
+    fn record_release_ack(&self, lock_id: &LockId, outcome: LockCleanupOutcome) {
         tracing::trace!(
             target: "rustfs_ecstore::object_lock_diag",
             event = EVENT_NAMESPACE_LOCK_RELEASE,
             component = LOG_COMPONENT_ECSTORE,
             subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
             state = "acknowledged",
+            result = match outcome {
+                LockCleanupOutcome::Released => "released",
+                LockCleanupOutcome::AlreadyAbsent => "already_absent",
+                LockCleanupOutcome::Unconfirmed => "unconfirmed",
+            },
             peer_kind = "remote",
             lock_id = %lock_id,
             peer = %self.addr,
@@ -803,7 +833,9 @@ impl RemoteClient {
         Some(Box::new(move |response: Response<GenerallyLockResponse>| {
             Box::pin(async move {
                 if response.get_ref().success {
-                    client.release_late_acquisitions(vec![lock_id]).await;
+                    client
+                        .release_late_acquisitions(vec![lock_id], AcquisitionCleanup::Confirmed)
+                        .await;
                 }
             })
         }))
@@ -815,7 +847,9 @@ impl RemoteClient {
             Box::pin(async move {
                 let acquired = acquired_lock_ids(&lock_ids, &response.get_ref().results);
                 if !acquired.is_empty() {
-                    client.release_late_acquisitions(acquired).await;
+                    client
+                        .release_late_acquisitions(acquired, AcquisitionCleanup::Confirmed)
+                        .await;
                 }
             })
         }))
@@ -825,19 +859,98 @@ impl RemoteClient {
         let client = self.clone();
         Some(Box::new(move |_| {
             Box::pin(async move {
-                client.release_late_acquisitions(lock_ids).await;
+                // Absence cannot settle an acquisition whose grant is still
+                // uncertain. Keep the conservative retry schedule.
+                client
+                    .release_late_acquisitions(lock_ids, AcquisitionCleanup::Uncertain)
+                    .await;
             })
         }))
     }
 
-    /// A lock granted after its caller stopped waiting is an orphan until its
-    /// lease expires; hand it back right away, best effort.
-    async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
+    async fn request_release(&self, lock_id: &LockId) -> Result<GenerallyLockResponse> {
+        let unlock_request = Self::create_unlock_request(lock_id);
+        let request_string = serde_json::to_string(&unlock_request)
+            .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
+        let mut client = self.get_client().await?;
+        let resource_summary = unlock_request.resource.to_string();
+        let mut req = Request::new(GenerallyLockRequest { args: request_string });
+        attach_lock_mutation_body_digest(&mut req)?;
+        Ok(self
+            .execute_rpc(
+                "release",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.un_lock(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner())
+    }
+
+    async fn request_release_batch(&self, lock_ids: &[LockId]) -> Result<BatchGenerallyLockResponse> {
+        let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
+        let mut client = self.get_client().await?;
+        let resource_summary = Self::summarize_resources(&unlock_requests);
+        let mut req = Request::new(BatchGenerallyLockRequest {
+            args: unlock_requests
+                .iter()
+                .map(|request| {
+                    serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
+                })
+                .collect::<Result<Vec<_>>>()?,
+        });
+        attach_lock_mutation_body_digest(&mut req)?;
+
+        Ok(self
+            .execute_rpc(
+                "release_batch",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.un_lock_batch(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner())
+    }
+
+    fn cleanup_batch_results(&self, lock_ids: &[LockId], results: &[GenerallyLockResult]) -> Vec<LockCleanupOutcome> {
+        lock_ids
+            .iter()
+            .enumerate()
+            .map(|(idx, lock_id)| {
+                let outcome = results
+                    .get(idx)
+                    .and_then(|result| cleanup_outcome(result.success, result.error_info.as_deref(), result.release_status).ok())
+                    .unwrap_or(LockCleanupOutcome::Unconfirmed);
+                if matches!(outcome, LockCleanupOutcome::Released | LockCleanupOutcome::AlreadyAbsent) {
+                    self.record_release_ack(lock_id, outcome);
+                }
+                outcome
+            })
+            .collect()
+    }
+
+    async fn cleanup_grants_batch(&self, lock_ids: &[LockId]) -> Result<Vec<LockCleanupOutcome>> {
+        if lock_ids.is_empty() {
+            return Ok(Vec::new());
+        }
+        let response = self.request_release_batch(lock_ids).await?;
+        Ok(self.cleanup_batch_results(lock_ids, &response.results))
+    }
+
+    /// A confirmed late grant can settle on authoritative absence. An
+    /// uncertain acquisition must retain retries because release may race it.
+    async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>, acquisition: AcquisitionCleanup) {
         if !self.admit_late_release() {
             rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
             let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
             if log_warning {
                 warn!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "budget_exhausted",
                     addr = %self.addr,
                     count = lock_ids.len(),
                     outcome = "budget_exhausted",
@@ -855,12 +968,27 @@ impl RemoteClient {
                 tokio::time::sleep(delay).await;
             }
 
-            match self.release_locks_batch(&lock_ids).await {
+            let released = match acquisition {
+                AcquisitionCleanup::Confirmed => self.cleanup_grants_batch(&lock_ids).await,
+                AcquisitionCleanup::Uncertain => self.release_locks_batch(&lock_ids).await.map(|results| {
+                    results
+                        .into_iter()
+                        .map(|released| {
+                            if released {
+                                LockCleanupOutcome::Released
+                            } else {
+                                LockCleanupOutcome::Unconfirmed
+                            }
+                        })
+                        .collect()
+                }),
+            };
+            match released {
                 Ok(released) => {
                     lock_ids = lock_ids
                         .into_iter()
                         .zip(released)
-                        .filter_map(|(lock_id, released)| (!released).then_some(lock_id))
+                        .filter_map(|(lock_id, outcome)| matches!(outcome, LockCleanupOutcome::Unconfirmed).then_some(lock_id))
                         .collect();
                     if lock_ids.is_empty() {
                         outcome = "released";
@@ -884,10 +1012,10 @@ impl RemoteClient {
                 event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                state = "late_grants_released",
+                state = "late_cleanup_completed",
                 addr = %self.addr,
                 count = original_count,
-                "Released remote locks granted after their caller stopped waiting"
+                "Confirmed cleanup of late remote lock acquisitions"
             );
         } else {
             let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
@@ -896,15 +1024,25 @@ impl RemoteClient {
                     event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
-                    state = "late_grants_unreleased",
+                    state = "late_cleanup_unconfirmed",
                     addr = %self.addr,
                     count = original_count,
                     outcome,
                     suppressed_timeouts,
-                    "Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
+                    "Could not confirm cleanup of every late remote lock acquisition"
                 );
             } else {
-                debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
+                debug!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "warning_suppressed",
+                    addr = %self.addr,
+                    count = original_count,
+                    outcome,
+                    suppressed_timeouts,
+                    "Suppressed repeated late remote lock cleanup warning"
+                );
             }
         }
     }
@@ -1352,68 +1490,37 @@ impl LockClient for RemoteClient {
     }
 
     async fn release(&self, lock_id: &LockId) -> Result<bool> {
-        let unlock_request = Self::create_unlock_request(lock_id);
-        let request_string = serde_json::to_string(&unlock_request)
-            .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
-        let mut client = self.get_client().await?;
-        let resource_summary = unlock_request.resource.to_string();
-        let mut req = Request::new(GenerallyLockRequest { args: request_string });
-        attach_lock_mutation_body_digest(&mut req)?;
-        let resp = self
-            .execute_rpc(
-                "release",
-                &resource_summary,
-                Self::rpc_timeout(),
-                async move { client.un_lock(req).await },
-                RpcCleanup::new(None, None, None),
-            )
-            .await?
-            .into_inner();
+        let resp = self.request_release(lock_id).await?;
         if let Some(error_info) = resp.error_info {
             return Err(LockError::internal(error_info));
         }
         if resp.success {
-            self.record_release_ack(lock_id);
+            self.record_release_ack(lock_id, LockCleanupOutcome::Released);
         }
         Ok(resp.success)
     }
 
+    async fn cleanup_grant(&self, lock_id: &LockId) -> Result<LockCleanupOutcome> {
+        let resp = self.request_release(lock_id).await?;
+        let outcome = cleanup_outcome(resp.success, resp.error_info.as_deref(), resp.release_status)?;
+        if matches!(outcome, LockCleanupOutcome::Released | LockCleanupOutcome::AlreadyAbsent) {
+            self.record_release_ack(lock_id, outcome);
+        }
+        Ok(outcome)
+    }
+
     async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
         if lock_ids.is_empty() {
             return Ok(Vec::new());
         }
-
-        let unlock_requests = lock_ids.iter().map(Self::create_unlock_request).collect::<Vec<_>>();
-        let mut client = self.get_client().await?;
-        let resource_summary = Self::summarize_resources(&unlock_requests);
-        let mut req = Request::new(BatchGenerallyLockRequest {
-            args: unlock_requests
-                .iter()
-                .map(|request| {
-                    serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
-                })
-                .collect::<Result<Vec<_>>>()?,
-        });
-        attach_lock_mutation_body_digest(&mut req)?;
-
-        let resp = self
-            .execute_rpc(
-                "release_batch",
-                &resource_summary,
-                Self::rpc_timeout(),
-                async move { client.un_lock_batch(req).await },
-                RpcCleanup::new(None, None, None),
-            )
-            .await?
-            .into_inner();
-
+        let resp = self.request_release_batch(lock_ids).await?;
         Ok(lock_ids
             .iter()
             .enumerate()
             .map(|(idx, lock_id)| {
                 let result = resp.results.get(idx);
                 if result.is_some_and(|result| result.success && result.error_info.is_none()) {
-                    self.record_release_ack(lock_id);
+                    self.record_release_ack(lock_id, LockCleanupOutcome::Released);
                 }
                 result.map(|result| result.success).unwrap_or(false)
             })
@@ -1615,6 +1722,84 @@ impl LockClient for RemoteClient {
 #[cfg(test)]
 mod tests {
     use super::*;
+
+    #[test]
+    fn cleanup_outcome_requires_authoritative_absence() {
+        assert_eq!(
+            cleanup_outcome(false, Some("diagnostic text is not a protocol"), LockReleaseStatus::AlreadyAbsent.into())
+                .expect("structured absence is authoritative"),
+            LockCleanupOutcome::AlreadyAbsent
+        );
+        assert!(cleanup_outcome(false, Some("lock not found for release"), LockReleaseStatus::Unspecified.into()).is_err());
+        assert_eq!(
+            cleanup_outcome(false, None, LockReleaseStatus::Unspecified.into()).expect("legacy result is uncertain"),
+            LockCleanupOutcome::Unconfirmed
+        );
+        for status in [LockReleaseStatus::Unspecified, LockReleaseStatus::Released] {
+            assert_eq!(
+                cleanup_outcome(true, None, status.into()).expect("positive release ACK"),
+                LockCleanupOutcome::Released
+            );
+        }
+    }
+
+    #[test]
+    fn cleanup_outcome_rejects_inconsistent_and_unknown_statuses() {
+        for (success, error, status) in [
+            (true, None, LockReleaseStatus::AlreadyAbsent.into()),
+            (false, None, LockReleaseStatus::Released.into()),
+            (true, Some("backend error"), LockReleaseStatus::Released.into()),
+            (true, None, 99),
+            (false, Some("backend error"), 99),
+        ] {
+            assert!(
+                cleanup_outcome(success, error, status).is_err(),
+                "must retain cleanup ownership for status {status}"
+            );
+        }
+    }
+
+    #[test]
+    fn cleanup_batch_results_preserves_each_lock_id_and_unknown_entries() {
+        let client = RemoteClient::new("http://cleanup-batch-results-test".to_string());
+        let lock_ids = (0..5)
+            .map(|idx| LockId::new_unique(&ObjectKey::new("bucket", format!("cleanup-{idx}"))))
+            .collect::<Vec<_>>();
+        let results = vec![
+            GenerallyLockResult {
+                success: true,
+                release_status: LockReleaseStatus::Released.into(),
+                ..Default::default()
+            },
+            GenerallyLockResult {
+                success: false,
+                error_info: Some("already released".to_string()),
+                release_status: LockReleaseStatus::AlreadyAbsent.into(),
+                ..Default::default()
+            },
+            GenerallyLockResult {
+                success: false,
+                error_info: Some("lock not found for release".to_string()),
+                ..Default::default()
+            },
+            GenerallyLockResult {
+                success: true,
+                release_status: 99,
+                ..Default::default()
+            },
+        ];
+        assert_eq!(
+            client.cleanup_batch_results(&lock_ids, &results),
+            vec![
+                LockCleanupOutcome::Released,
+                LockCleanupOutcome::AlreadyAbsent,
+                LockCleanupOutcome::Unconfirmed,
+                LockCleanupOutcome::Unconfirmed,
+                LockCleanupOutcome::Unconfirmed,
+            ],
+            "missing or unrecognized entries must not hide confirmed cleanup of other IDs"
+        );
+    }
     use crate::runtime::sources as runtime_sources;
     use rustfs_lock::{ObjectKey, types::LockPriority};
     use tokio::net::TcpListener;

crates/ecstore/src/cluster/rpc/runtime_sources.rs

@@ -481,6 +481,14 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
     global_internode_metrics().record_sent_bytes_for_operation_and_backend(
         INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -521,6 +529,14 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 #[cfg(feature = "diagnostic-read-version-caller")]
 fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
     match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -598,6 +614,17 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
     );
 }
 
+pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;
@@ -690,6 +717,14 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
     record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 /// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
 /// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
 fn internode_rpc_large_payload_warn_bytes() -> usize {

crates/ecstore/src/core/pool_meta_recovery_codec.rs

@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
         .encode_with_shard_integrity(
             Cursor::new(Bytes::copy_from_slice(canonical)),
             &mut writers,
-            raw.len(),
+            crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
             1,
             IntegrityEncodeMode::Inline(canonical.len()),
             protected,

crates/ecstore/src/core/pools.rs

@@ -8292,16 +8292,18 @@ where
     S: EcstoreObjectIO,
 {
     // Bound the CAS state retained by callers, including ordinary metadata
-    // transactions that never enter identity repair. Large inline retry states
+    // transactions that never enter owner repair. Large inline retry states
     // otherwise propagate through startup and cross-crate storage futures.
     Box::pin(async move {
-        if object != POOL_META_IDENTITY_NAME {
+        let native_owner_repair = object == POOL_META_NAME && matches!(phase, "native_heal" | "prepare_cas");
+        if object != POOL_META_IDENTITY_NAME && !native_owner_repair {
             return execute_pool_meta_object_cas_once(pool, object, data, opts, fence, phase).await;
         }
-        // Keep the exact identity owned by this transaction. A quorum read alone
-        // cannot discharge repair responsibility for a missing disk replica.
-        let identity = data;
-        let mut outcome = execute_pool_meta_object_cas_once(pool.clone(), object, identity.clone(), opts, fence, phase).await;
+        // Repair only the exact payload already published by this owner. A
+        // quorum read cannot discharge responsibility for a missing replica.
+        let committed_payload = data;
+        let mut outcome =
+            execute_pool_meta_object_cas_once(pool.clone(), object, committed_payload.clone(), opts, fence, phase).await;
         for delay_secs in [1, 2, 4] {
             if !outcome.source.as_deref().is_some_and(|err| {
                 matches!(err, Error::Io(inner) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataRepairRequired>()))
@@ -8314,19 +8316,28 @@ where
             let repair_options = async {
                 fence.ensure_held()?;
                 let (current, info) = read_config_no_lock_preserve_empty_with_metadata(pool.clone(), object).await?;
-                if current != identity {
+                if current != committed_payload {
                     return Err(Error::PreconditionFailed);
                 }
-                let token = info
-                    .etag
-                    .filter(|etag| !etag.trim().is_empty())
-                    .map(PoolMetaCasToken::Existing)
-                    .unwrap_or(PoolMetaCasToken::Unsafe);
+                let token = if native_owner_repair {
+                    match pool_meta_cas_token_from_info(info) {
+                        token @ PoolMetaCasToken::ExistingNative { .. } => token,
+                        _ => return Err(Error::PreconditionFailed),
+                    }
+                } else {
+                    info.etag
+                        .filter(|etag| !etag.trim().is_empty())
+                        .map(PoolMetaCasToken::Existing)
+                        .unwrap_or(PoolMetaCasToken::Unsafe)
+                };
                 pool_meta_cas_options(object, &token, fence)
             }
             .await;
             outcome = match repair_options {
-                Ok(opts) => execute_pool_meta_object_cas_once(pool.clone(), object, identity.clone(), opts, fence, phase).await,
+                Ok(opts) => {
+                    execute_pool_meta_object_cas_once(pool.clone(), object, committed_payload.clone(), opts, fence, phase)
+                        .await
+                }
                 Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
             };
             // A rejected repair cannot turn the preceding committed write into a
@@ -31049,6 +31060,10 @@ mod pools_tests {
                 std::future::pending::<()>().await;
             }
             let mut reader = self.inner.get_object_reader(bucket, object, range, headers, opts).await?;
+            if object == POOL_META_NAME {
+                reader.object_info.mod_time = Some(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1));
+                reader.object_info.data_dir = Some(uuid::Uuid::from_u128(7));
+            }
             match self.fault {
                 IdentityRepairFault::Changed => {
                     reader.stream = Box::new(Cursor::new(b"other-identity".to_vec()));
@@ -31078,16 +31093,30 @@ mod pools_tests {
                         .and_then(HTTPPreconditions::if_none_match_value),
                     None
                 );
+                let written = if object == POOL_META_NAME {
+                    &self.inner.stored
+                } else {
+                    &self.inner.identity
+                };
+                let current_etag = written.lock().expect("metadata lock").as_ref().map(|(_, etag)| etag.clone());
                 assert_eq!(
                     opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value),
-                    self.inner
-                        .identity
-                        .lock()
-                        .expect("identity lock")
-                        .as_ref()
-                        .map(|(_, etag)| etag.as_str()),
+                    current_etag.as_deref(),
                     "repair must use the observed post-commit CAS token",
                 );
+                if object == POOL_META_NAME {
+                    let condition = rustfs_utils::http::metadata_compat::get_consistent_str(
+                        &opts.user_defined,
+                        crate::disk::POOL_META_NATIVE_CAS_METADATA_SUFFIX,
+                    )
+                    .expect("native repair requires a physical predecessor condition");
+                    let condition: crate::disk::PoolMetaNativeCasCondition =
+                        serde_json::from_str(condition).expect("parse native CAS condition");
+                    let parent = condition.parent.expect("native CAS uses the actual committed head");
+                    assert_eq!(Some(parent.etag), current_etag);
+                    assert_eq!(parent.mod_time_unix_nanos, 1_000_000_000);
+                    assert_eq!(parent.data_dir, Some(uuid::Uuid::from_u128(7)));
+                }
                 if self.fault == IdentityRepairFault::Reject {
                     return Err(Error::PreconditionFailed);
                 }
@@ -31192,6 +31221,46 @@ mod pools_tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn pool_meta_prepare_owner_repair_uses_exact_native_head() {
+        let payload = b"owned-pending-pool-meta";
+        for (fault, writes, reads, succeeds) in [
+            (IdentityRepairFault::Once, 2, 1, true),
+            (IdentityRepairFault::Changed, 1, 1, false),
+            (IdentityRepairFault::MissingEtag, 1, 1, false),
+            (IdentityRepairFault::Unknown, 1, 0, false),
+        ] {
+            let pool = IdentityRepairStorage::new(fault);
+            let state = super::PoolMetaWriteState::default();
+            let mut arm = state.arm_transaction();
+            let result = save_pool_meta_object_cas(
+                pool.clone(),
+                POOL_META_NAME,
+                payload.to_vec(),
+                &PoolMetaCasToken::Missing,
+                &PoolMetaPersistenceFence::OfflineRecovery,
+                "prepare_cas",
+                &mut arm,
+            )
+            .await;
+
+            if succeeds {
+                result.expect("prepare must repair its own partially replicated payload");
+                arm.disarm();
+                assert_eq!(pool.inner.stored.lock().expect("metadata lock").as_ref().expect("written").0, payload);
+            } else {
+                let error = result.expect_err("uncertain or changed authority must remain blocked");
+                let failure = error.pool_metadata_failure().expect("typed transaction uncertainty");
+                assert_eq!(failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
+                assert_eq!(failure.phase, "prepare_cas");
+            }
+            drop(arm);
+            assert_eq!(state.ensure_write_safe("after prepare owner repair").is_ok(), succeeds);
+            assert_eq!(pool.writes.load(Ordering::SeqCst), writes, "{fault:?}");
+            assert_eq!(pool.reads.load(Ordering::SeqCst), reads, "{fault:?}");
+        }
+    }
+
     #[tokio::test(start_paused = true)]
     async fn pool_identity_owner_repair_cancellation_keeps_the_transaction_armed() {
         for fault in [

crates/ecstore/src/data_usage/mod.rs

@@ -34,7 +34,7 @@ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapsho
 use rustfs_data_usage::{
     BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
     DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
-    observed_data_usage_is_newer,
+    observed_data_usage_is_newer, observed_data_usage_is_newer_than_identity,
 };
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -227,7 +227,10 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
 /// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
 #[derive(Debug, Clone)]
 struct CachedDataUsageSnapshot {
+    /// Last successful load, retained through failures for lineage comparison.
     info: Option<DataUsageInfo>,
+    /// Controls serving independently from the retained ordering evidence.
+    last_load_failed: bool,
     loaded_at: tokio::time::Instant,
     /// Pre-discard per-bucket sizes from the same load, retained even when the
     /// snapshot is incomplete and its bucket data is discarded. Consumed only
@@ -240,6 +243,7 @@ impl CachedDataUsageSnapshot {
     fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
         self.info
             .as_ref()
+            .filter(|_| !self.last_load_failed)
             .map(project)
             .ok_or_else(|| Error::other("data usage snapshot load recently failed"))
     }
@@ -275,6 +279,17 @@ fn cache_data_usage_snapshot_result<T>(
             mut degraded_baseline,
             observed_unavailable,
         }) => {
+            // A primary read can fall back to an older backup. Refresh the
+            // retry age without replacing a newer snapshot this process has
+            // already read; snapshot size itself is not an ordering signal.
+            if let Some(previous) = cache.as_mut()
+                && let Some(cached_info) = previous.info.as_ref()
+                && snapshot_predates(&info, cached_info)
+            {
+                previous.loaded_at = loaded_at;
+                previous.last_load_failed = false;
+                return Some(Ok(project(cached_info)));
+            }
             // A flaky observed read must not shrink quota coverage for a TTL
             // window: carry the previous refresh's baseline entries forward,
             // letting the fresh (authoritative) values win where they exist.
@@ -286,21 +301,23 @@ fn cache_data_usage_snapshot_result<T>(
             let projected = project(&info);
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info),
+                last_load_failed: false,
                 loaded_at,
                 degraded_baseline,
             });
             Ok(projected)
         }
         Err(e) => {
-            // Keep the previous baseline through a failed refresh: quota
-            // admission must not lose its last grounded values because one
-            // backend read errored.
-            let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
-            *cache = Some(CachedDataUsageSnapshot {
+            // A failed load is negatively cached, but cannot erase the last
+            // good snapshot's ordering proof or grounded quota baseline.
+            let cached = cache.get_or_insert_with(|| CachedDataUsageSnapshot {
                 info: None,
+                last_load_failed: true,
                 loaded_at,
-                degraded_baseline,
+                degraded_baseline: HashMap::new(),
             });
+            cached.loaded_at = loaded_at;
+            cached.last_load_failed = true;
             Err(e)
         }
     })
@@ -1245,7 +1262,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
     // The observed-newness gate below compares against the snapshot as
     // persisted, before normalization demotes or discards anything.
-    let authoritative_as_persisted = loaded_snapshot.clone();
+    let authoritative_as_persisted = loaded_snapshot.snapshot_identity();
     let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
 
     // A bucket without a converged scanner cycle behind it — a freshly joined
@@ -1263,7 +1280,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     // read per snapshot refresh.
     let mut observed_unavailable = false;
     match load_observed_data_usage_snapshot(store).await {
-        Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
+        Ok(Some(observed)) if observed_data_usage_is_newer_than_identity(&observed, authoritative_as_persisted) => {
             backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
         }
         Ok(_) => {}
@@ -2367,6 +2384,9 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
     let Some(cached) = cache.as_ref() else {
         return false;
     };
+    if cached.last_load_failed {
+        return false;
+    }
     let Some(info) = cached.info.as_ref() else {
         return false;
     };
@@ -2382,11 +2402,35 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
         .is_none_or(|attempted_at| now.duration_since(attempted_at) >= refresh_age)
 }
 
-/// Whether a freshly loaded snapshot is older than the cached one, as when a
-/// primary read fails and the loader falls back to the previous `.bkp` copy.
-fn snapshot_predates(loaded: Option<SystemTime>, cached: Option<SystemTime>) -> bool {
+/// Compare authoritative lineage by the writer's epoch, cycle, then timestamp
+/// order. Within one lineage, compare the observation's coordinated position
+/// before its timestamp; a later observation cannot revive an older lineage.
+fn snapshot_predates(loaded: &DataUsageInfo, cached: &DataUsageInfo) -> bool {
+    let loaded_baseline = loaded
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| loaded.snapshot_identity());
+    let cached_baseline = cached
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| cached.snapshot_identity());
+    let loaded_position = (loaded_baseline.scanner_epoch, loaded_baseline.scanner_cycle);
+    let cached_position = (cached_baseline.scanner_epoch, cached_baseline.scanner_cycle);
+    if loaded_position != cached_position {
+        return loaded_position < cached_position;
+    }
+    let (loaded, cached) = if loaded_baseline == cached_baseline {
+        let loaded_observation_position = loaded.scanner_epoch.zip(loaded.scanner_cycle);
+        let cached_observation_position = cached.scanner_epoch.zip(cached.scanner_cycle);
+        if loaded_observation_position != cached_observation_position {
+            return loaded_observation_position < cached_observation_position;
+        }
+        (loaded.last_update, cached.last_update)
+    } else {
+        (loaded_baseline.last_update, cached_baseline.last_update)
+    };
     match (loaded, cached) {
-        (Some(loaded), Some(cached)) => loaded < cached,
+        (Some(loaded), Some(cached)) => {
+            loaded < cached && !rustfs_data_usage::usage_last_update_is_untrusted_future(cached, SystemTime::now())
+        }
         (None, Some(_)) => true,
         _ => false,
     }
@@ -2427,22 +2471,15 @@ where
         return false;
     };
     let mut cache = data_usage_snapshot_cache().write().await;
-    let cached_last_update = cache
-        .as_ref()
-        .and_then(|cached| cached.info.as_ref())
-        .and_then(|info| info.last_update);
-    if snapshot_predates(loaded.info.last_update, cached_last_update) {
-        return false;
-    }
     cache_data_usage_snapshot_result(
         &mut cache,
         Ok(loaded),
         tokio::time::Instant::now(),
         refresh_generation,
         data_usage_snapshot_generation(),
-        |_| (),
+        |info| info.buckets_usage.contains_key(bucket),
     )
-    .is_some()
+    .is_some_and(|result| matches!(result, Ok(true)))
 }
 
 // Calls of `reload_unknown_bucket_usage` per bucket. Unit tests do not run in
@@ -4224,6 +4261,7 @@ mod tests {
         let degraded_baseline = info.bucket_sizes.clone();
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(info),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now()
                 .checked_sub(age)
                 .expect("the monotonic clock must reach back by the requested age"),
@@ -4700,6 +4738,355 @@ mod tests {
         assert_snapshot_bucket(&cached, "bucket");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn expired_usage_snapshot_refresh_keeps_newer_counts_over_an_older_backup() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut current = usage_snapshot_with_new_bucket(SystemTime::UNIX_EPOCH + Duration::from_secs(2), 100);
+        current.scanner_epoch = Some(4);
+        current.scanner_cycle = Some(11);
+        current.usage_snapshot_converged = Some(true);
+        let mut backup = data_usage_info_for_test("existing", 1, 5, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        backup.scanner_epoch = Some(4);
+        backup.scanner_cycle = Some(10);
+        backup.usage_snapshot_converged = Some(true);
+        for (path, info) in [
+            (DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
+            (DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
+        ] {
+            save_config(store.clone(), path, serde_json::to_vec(info).expect("encode usage snapshot"))
+                .await
+                .expect("persist initial usage snapshot");
+        }
+        invalidate_data_usage_snapshot_cache().await;
+        for loaded in [
+            load_data_usage_from_backend_cached(store.clone()).await,
+            load_admin_data_usage_from_backend_cached(store.clone()).await,
+        ] {
+            assert_eq!(loaded.expect("prime usage cache"), current);
+        }
+
+        // Bucket cleanup can reduce usage without advancing the scanner
+        // position. Only its publication timestamp advances.
+        let mut after_delete = data_usage_info_for_test("existing", 1, 3, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        after_delete.scanner_epoch = current.scanner_epoch;
+        after_delete.scanner_cycle = current.scanner_cycle;
+        after_delete.usage_snapshot_converged = Some(true);
+        let mut stale_observation =
+            data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        stale_observation.scanner_epoch = Some(4);
+        stale_observation.scanner_cycle = Some(12);
+        stale_observation.usage_snapshot_converged = Some(false);
+        stale_observation.usage_snapshot_authoritative_baseline = Some(backup.snapshot_identity());
+        assert!(observed_data_usage_is_newer(&stale_observation, &backup));
+        assert!(!observed_data_usage_is_newer(&stale_observation, &current));
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&stale_observation).expect("encode observation of an older namespace"),
+        )
+        .await
+        .expect("persist an observation anchored to the old backup");
+        for (payload, expected) in [
+            (b"not a usage snapshot".to_vec(), &current),
+            (serde_json::to_vec(&after_delete).expect("encode post-delete usage"), &after_delete),
+        ] {
+            save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), payload)
+                .await
+                .expect("replace primary usage snapshot");
+            for cache in [data_usage_snapshot_cache(), admin_data_usage_snapshot_cache()] {
+                cache
+                    .write()
+                    .await
+                    .as_mut()
+                    .expect("usage cache should be populated")
+                    .loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            for loaded in [
+                load_data_usage_from_backend_cached(store.clone()).await,
+                load_admin_data_usage_from_backend_cached(store.clone()).await,
+            ] {
+                assert_eq!(loaded.expect("refresh an expired usage cache"), *expected);
+            }
+            assert!(
+                !data_usage_snapshot_cache()
+                    .read()
+                    .await
+                    .as_ref()
+                    .expect("quota cache should remain populated")
+                    .degraded_baseline
+                    .contains_key("retired-incarnation"),
+                "a newer observation anchored to an older namespace must not backfill ghost usage"
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn expired_admin_usage_refresh_orders_observations_by_generation_before_time() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        authoritative.scanner_epoch = Some(7);
+        authoritative.scanner_cycle = Some(10);
+        authoritative.usage_snapshot_converged = Some(true);
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&authoritative).expect("encode authoritative usage"),
+        )
+        .await
+        .expect("persist the common authoritative baseline");
+        let observation = |epoch, cycle, seconds, objects, size| {
+            let mut info =
+                data_usage_info_for_test("bucket", objects, size, SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
+            info.scanner_epoch = Some(epoch);
+            info.scanner_cycle = Some(cycle);
+            info.usage_snapshot_converged = Some(false);
+            info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
+            info
+        };
+        let first = observation(7, 11, 200, 11, 1_100);
+        let newer_cycle = observation(7, 12, 150, 12, 1_200);
+        let older_cycle = observation(7, 11, 300, 99, 9_900);
+        let newer_epoch = observation(8, 1, 50, 1, 100);
+        invalidate_data_usage_snapshot_cache().await;
+
+        for (observed, expected) in [
+            (&first, &first),
+            (&newer_cycle, &newer_cycle),
+            (&older_cycle, &newer_cycle),
+            (&newer_epoch, &newer_epoch),
+        ] {
+            assert!(
+                observed_data_usage_is_newer(observed, &authoritative),
+                "the persisted observation must pass the real loader's selection gate"
+            );
+            save_config(
+                store.clone(),
+                DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                serde_json::to_vec(observed).expect("encode scanner observation"),
+            )
+            .await
+            .expect("persist scanner observation");
+            if let Some(cached) = admin_data_usage_snapshot_cache().write().await.as_mut() {
+                cached.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            let loaded = load_admin_data_usage_from_backend_cached(store.clone())
+                .await
+                .expect("refresh admin usage through the real backend loader");
+            assert_eq!(
+                &loaded,
+                expected,
+                "same-baseline observation ordering must follow epoch/cycle before wall clock: incoming={:?}",
+                observed.snapshot_identity()
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_new_authority_over_a_later_old_observation() {
+        let loaded_at = tokio::time::Instant::now();
+        let baseline = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        let mut observed = data_usage_info_for_test("bucket", 5, 60, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        observed.usage_snapshot_converged = Some(false);
+        observed.usage_snapshot_authoritative_baseline = Some(baseline.snapshot_identity());
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(observed),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let after_delete = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(after_delete.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a newer authority must retire observations of its predecessor");
+        assert_eq!(refreshed, after_delete);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_orders_lineage_by_epoch_cycle_then_time() {
+        let loaded_at = tokio::time::Instant::now();
+        for (cached_position, loaded_position, loaded_time, accept) in [
+            ((Some(2), Some(10)), (Some(1), Some(99)), 110, false),
+            ((Some(2), Some(10)), (Some(2), Some(9)), 110, false),
+            ((Some(2), Some(10)), (Some(3), Some(1)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(11)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(10)), 110, true),
+            ((Some(3), None), (Some(2), Some(10)), 110, false),
+            ((Some(2), Some(10)), (Some(3), None), 90, true),
+            ((None, None), (None, None), 110, true),
+            ((None, None), (None, None), 90, false),
+        ] {
+            let mut previous = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+            (previous.scanner_epoch, previous.scanner_cycle) = cached_position;
+            let mut loaded = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(loaded_time));
+            (loaded.scanner_epoch, loaded.scanner_cycle) = loaded_position;
+            let expected = if accept { &loaded } else { &previous };
+            let mut cache = Some(CachedDataUsageSnapshot {
+                info: Some(previous.clone()),
+                last_load_failed: false,
+                loaded_at,
+                degraded_baseline: HashMap::new(),
+            });
+            let refreshed = cache_data_usage_snapshot_result(
+                &mut cache,
+                Ok(loaded_usage_for_test(loaded.clone())),
+                loaded_at,
+                1,
+                1,
+                DataUsageInfo::clone,
+            )
+            .expect("matching refresh generation")
+            .expect("a valid usage snapshot remains available");
+            assert_eq!(
+                &refreshed, expected,
+                "lineage ordering must match scanner publication: cached={cached_position:?}, loaded={loaded_position:?}, loaded_time={loaded_time}"
+            );
+        }
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_preserves_rejection_backoff_and_baseline() {
+        let loaded_at = tokio::time::Instant::now();
+        let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(expected.clone()),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::from([("bucket".to_string(), 42)]),
+        });
+        let refreshed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            refreshed_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("the newer cached snapshot remains available");
+        assert_eq!(refreshed, expected);
+        let retained = cache.as_ref().expect("cache should remain populated");
+        assert_eq!(retained.degraded_baseline.get("bucket"), Some(&42));
+        assert_eq!(retained.loaded_at, refreshed_at, "an older fallback must retain the TTL backoff");
+
+        let result =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), refreshed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(result, Err(Error::ErasureReadQuorum)));
+        let failed = cache.as_ref().expect("negative cache should be populated");
+        assert!(failed.last_load_failed);
+        assert_eq!(failed.info.as_ref(), Some(&expected), "failure must retain the last good ordering proof");
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_clock_recovery() {
+        let loaded_at = tokio::time::Instant::now();
+        let now = SystemTime::now();
+        let future = now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(60);
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(data_usage_info_for_test("bucket", 3, 42, future)),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let recovered = data_usage_info_for_test("bucket", 1, 7, now);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(recovered.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a clock recovery snapshot must not be frozen behind an untrusted timestamp");
+        assert_eq!(refreshed, recovered);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_keeps_last_good_lineage_across_a_failed_load() {
+        let loaded_at = tokio::time::Instant::now();
+        let current = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = None;
+        let first = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(current.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("prime the last successful snapshot");
+        assert_eq!(first, current);
+
+        let failed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let failure =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), failed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+        assert!(
+            fresh_cached_data_usage_snapshot(
+                &cache,
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+                Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS),
+                |_| panic!("a failed load must not serve its retained snapshot as a success"),
+            )
+            .expect("the failed load remains inside its negative TTL")
+            .is_err()
+        );
+        assert!(
+            !unknown_bucket_reload_due(
+                &cache,
+                "unknown",
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+            ),
+            "an unknown bucket must not bypass the failed load's TTL"
+        );
+
+        let recovered_at = failed_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let recovered = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            recovered_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a readable older backup must preserve the last good lineage");
+        assert_eq!(recovered, current);
+        assert_eq!(
+            cache.as_ref().expect("recovered cache").degraded_baseline.get("bucket"),
+            Some(&42),
+            "a failed load must not erase the baseline's ordering proof"
+        );
+        assert_eq!(
+            fresh_cached_data_usage_snapshot(&cache, recovered_at, Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS), |info| info
+                .objects_total_size,)
+            .expect("the recovered load starts a fresh TTL")
+            .expect("the last successful snapshot is available again"),
+            42
+        );
+    }
+
     #[test]
     #[serial]
     fn unavailable_observed_read_keeps_previous_baseline_coverage() {
@@ -4707,6 +5094,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
         });
@@ -4740,6 +5128,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::new(),
         });
@@ -5928,6 +6317,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -5999,6 +6389,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -6051,6 +6442,7 @@ mod tests {
         let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(stale),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -21,6 +21,7 @@ fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
     info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
     let cache = Some(CachedDataUsageSnapshot {
         info: Some(info),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -66,6 +67,7 @@ fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
 
     cache = Some(CachedDataUsageSnapshot {
         info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -127,6 +129,33 @@ fn projected_publication_moves_bucket_maps_into_cache() {
         cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
         original_key
     );
+
+    let failure = cache_data_usage_snapshot_result(
+        &mut cache,
+        Err(Error::ErasureReadQuorum),
+        tokio::time::Instant::now(),
+        1,
+        1,
+        |_: &DataUsageInfo| -> u64 { panic!("a failed load must not project its retained snapshot") },
+    )
+    .expect("matching refresh generation");
+    assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+    let retained = cache
+        .as_ref()
+        .expect("negative cache")
+        .info
+        .as_ref()
+        .expect("last good ordering proof");
+    assert_eq!(
+        retained
+            .buckets_usage
+            .get_key_value("known")
+            .expect("retained bucket")
+            .0
+            .as_ptr(),
+        original_key,
+        "a failed load must retain the snapshot without copying its bucket maps"
+    );
 }
 
 #[test]

crates/ecstore/src/diagnostics/object_lock.rs

@@ -167,6 +167,16 @@ impl ObjectLockAttempt {
             }
         }
     }
+
+    /// Correlate a raw guard's acquisition with its peer identities without
+    /// claiming ownership of the guard's lifetime. Peer release events retain
+    /// the identities needed to reconstruct when the actual owner releases it.
+    pub(crate) fn observe_acquisition(&mut self, result: &Result<NamespaceLockGuard, LockError>) {
+        self.observe(result);
+        if let Some(context) = self.0.as_mut() {
+            context.finished = true;
+        }
+    }
 }
 
 fn duration_ms(duration: Duration) -> u64 {
@@ -500,6 +510,45 @@ pub(crate) mod tests {
         )
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn raw_guard_acquisition_diagnostic_does_not_claim_guard_lifetime() {
+        temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
+            let capture = CapturedLockEvents::default();
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("raw guard diagnostic runtime");
+            tracing::subscriber::with_default(capture.subscriber(), || {
+                runtime.block_on(async {
+                    let lock = namespace();
+                    let mut attempt = ObjectLockAttempt::start(
+                        "bucket_incarnation_disk_read",
+                        "bucket",
+                        "transaction.lock",
+                        None,
+                        &lock,
+                        "read",
+                        Duration::from_secs(1),
+                    );
+                    let result = lock.get_read_lock(Duration::from_secs(1)).await;
+                    attempt.observe_acquisition(&result);
+                    drop(attempt);
+                    let guard = result.expect("retain the raw guard after diagnostic completion");
+                    assert!(!guard.is_lock_lost());
+                    let states = capture
+                        .rows()
+                        .iter()
+                        .map(|row| row["fields"]["state"].as_str().expect("diagnostic state string").to_owned())
+                        .collect::<Vec<_>>();
+                    assert_eq!(states, ["acquiring", "acquired"]);
+                    drop(guard);
+                    assert_eq!(capture.rows().len(), 2, "the acquisition observer must not invent a guard release");
+                });
+            });
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn object_lock_diagnostics_disabled_does_not_allocate_context_or_emit() {

crates/ecstore/src/disk/local.rs

@@ -3159,6 +3159,7 @@ pub(super) fn has_rename_data_test_failpoints(dst_path: &str) -> bool {
             == Some(dst_path)
         || os::has_dir_fsync_test_failures()
         || os::file_sync_probe::is_active()
+        || os::prepared_publication_test_hooks::has_any_hooks()
 }
 
 #[cfg(all(not(test), feature = "test-util", target_os = "linux"))]
@@ -3169,6 +3170,7 @@ pub(super) fn has_rename_data_test_failpoints(dst_path: &str) -> bool {
         .iter()
         .any(|target| target == dst_path)
         || os::has_dir_fsync_test_failures()
+        || os::prepared_publication_test_hooks::has_any_hooks()
 }
 
 #[cfg(any(not(any(test, feature = "test-util")), not(target_os = "linux")))]
@@ -4198,48 +4200,43 @@ fn reclaim_std_file_cache(file: &std::fs::File, reclaim_len: usize, reclaim_on_s
 #[cfg(not(target_os = "linux"))]
 fn reclaim_std_file_cache(_file: &std::fs::File, _reclaim_len: usize, _reclaim_on_shutdown: bool) {}
 
-static PRESYNCED_TMP_DATA_DIRS: std::sync::LazyLock<ParkingLotMutex<HashSet<PathBuf>>> =
-    std::sync::LazyLock::new(|| ParkingLotMutex::new(HashSet::new()));
+const PRESYNCED_TMP_DATA_DIRS_PER_SHARD: usize = 64;
+const PRESYNCED_TMP_FILES_PER_SHARD: usize = 128;
 
-static PRESYNCED_TMP_FILES: std::sync::LazyLock<ParkingLotMutex<HashSet<PathBuf>>> =
-    std::sync::LazyLock::new(|| ParkingLotMutex::new(HashSet::new()));
+static PRESYNCED_TMP_DATA_DIRS: std::sync::LazyLock<os::ShardedPathCache<()>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PRESYNCED_TMP_DATA_DIRS_PER_SHARD));
 
-fn mark_presynced_tmp_data_dir(dir: PathBuf) {
-    let mut set = PRESYNCED_TMP_DATA_DIRS.lock();
-    if set.len() >= 4096 {
-        set.clear();
-    }
-    set.insert(dir);
+static PRESYNCED_TMP_FILES: std::sync::LazyLock<os::ShardedPathCache<()>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PRESYNCED_TMP_FILES_PER_SHARD));
+
+fn mark_presynced_tmp_data_dir(dir: impl AsRef<Path>) {
+    PRESYNCED_TMP_DATA_DIRS.insert(dir.as_ref(), ());
 }
 
 fn unmark_presynced_tmp_data_dir(dir: &Path) -> bool {
-    PRESYNCED_TMP_DATA_DIRS.lock().remove(dir)
+    PRESYNCED_TMP_DATA_DIRS.remove(dir).is_some()
 }
 
 #[cfg(target_os = "linux")]
 pub(in crate::disk) fn is_presynced_tmp_data_dir(dir: &Path) -> bool {
-    PRESYNCED_TMP_DATA_DIRS.lock().contains(dir)
+    PRESYNCED_TMP_DATA_DIRS.with_get(dir, |()| ()).is_some()
 }
 
 #[cfg(target_os = "linux")]
 pub(in crate::disk) fn take_presynced_tmp_data_dir(dir: &Path) -> bool {
-    PRESYNCED_TMP_DATA_DIRS.lock().remove(dir)
+    PRESYNCED_TMP_DATA_DIRS.remove(dir).is_some()
 }
 
-fn mark_presynced_tmp_file(file: PathBuf) {
-    let mut set = PRESYNCED_TMP_FILES.lock();
-    if set.len() >= 8192 {
-        set.clear();
-    }
-    set.insert(file);
+fn mark_presynced_tmp_file(file: impl AsRef<Path>) {
+    PRESYNCED_TMP_FILES.insert(file.as_ref(), ());
 }
 
 fn unmark_presynced_tmp_file(file: &Path) {
-    PRESYNCED_TMP_FILES.lock().remove(file);
+    let _ = PRESYNCED_TMP_FILES.remove(file);
 }
 
 fn take_presynced_tmp_file(file: &Path) -> bool {
-    PRESYNCED_TMP_FILES.lock().remove(file)
+    PRESYNCED_TMP_FILES.remove(file).is_some()
 }
 
 fn open_lazy_local_file_std(file_path: &Path, volume_dir: &Path, _reclaim_on_shutdown: bool) -> std::io::Result<std::fs::File> {
@@ -4325,7 +4322,7 @@ impl LazyLocalFileWriter {
                         drop(file);
                         if sync_on_close {
                             if mark_dir_presynced && let Some(parent) = file_path.parent() {
-                                mark_presynced_tmp_data_dir(parent.to_path_buf());
+                                mark_presynced_tmp_data_dir(parent);
                             }
                             mark_presynced_tmp_file(file_path);
                         }
@@ -4347,7 +4344,7 @@ impl LazyLocalFileWriter {
                         drop(file);
                         if sync_on_close {
                             if mark_dir_presynced && let Some(parent) = file_path.parent() {
-                                mark_presynced_tmp_data_dir(parent.to_path_buf());
+                                mark_presynced_tmp_data_dir(parent);
                             }
                             mark_presynced_tmp_file(file_path);
                         }
@@ -4475,7 +4472,7 @@ impl AsyncWrite for LazyLocalFileWriter {
                         drop(file);
                         if sync_on_close {
                             if mark_dir_presynced && let Some(parent) = file_path.parent() {
-                                mark_presynced_tmp_data_dir(parent.to_path_buf());
+                                mark_presynced_tmp_data_dir(parent);
                             }
                             mark_presynced_tmp_file(file_path);
                         }
@@ -9145,11 +9142,15 @@ impl LocalDisk {
         opts: DeleteOptions,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> Vec<Option<DiskError>> {
-        let payload = super::GuardedDeleteVersionsPayload { versions, opts };
+        let payload = super::GuardedDeleteVersionsPayload {
+            versions,
+            opts,
+            bounded_recovery_protocol: None,
+        };
         if let Err(error) = payload.validate() {
             return vec![Some(error); payload.versions.len()];
         }
-        let super::GuardedDeleteVersionsPayload { versions, mut opts } = payload;
+        let super::GuardedDeleteVersionsPayload { versions, mut opts, .. } = payload;
         opts.recoverable_delete = true;
         let mut errors = Vec::with_capacity(versions.len());
         for version in &versions {
@@ -22183,7 +22184,7 @@ mod test {
                 .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}"))
                 .expect("staged shard directory should resolve");
             assert!(
-                !PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir),
+                PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_none(),
                 "an unknown destination must defer payload sync until commit"
             );
 
@@ -22236,7 +22237,10 @@ mod test {
             let staged_dir = disk
                 .get_object_path_for_io(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}"))
                 .expect("staged shard directory should resolve");
-            assert!(PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir), "the writer must pre-sync the shard");
+            assert!(
+                PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_some(),
+                "the writer must pre-sync the shard"
+            );
             assert!(!os::fsync_dir_recorder::was_fsynced(&staged_dir));
 
             let mut fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None);
@@ -22247,7 +22251,7 @@ mod test {
                 .await
                 .expect("rename_data should commit");
             assert!(
-                !PRESYNCED_TMP_DATA_DIRS.lock().contains(&staged_dir),
+                PRESYNCED_TMP_DATA_DIRS.with_get(&staged_dir, |()| ()).is_none(),
                 "the fused commit must consume the pre-sync marker"
             );
             assert!(
@@ -22438,6 +22442,141 @@ mod test {
         assert_eq!(resp.old_current_size, Some(OldCurrentSize::Present(1)));
     }
 
+    #[cfg(target_os = "linux")]
+    #[tokio::test]
+    #[allow(clippy::await_holding_lock)]
+    async fn test_rename_data_fused_overwrite_and_rollback_double_failure() {
+        const CHILD: &str = "RUSTFS_TEST_FUSED_OVERWRITE_CHILD";
+        const TEST: &str = "disk::local::test::test_rename_data_fused_overwrite_and_rollback_double_failure";
+
+        // Other tests install process-wide hooks that deliberately force rename_data off the fused path.
+        if std::env::var_os(CHILD).is_none() {
+            let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+                .args(["--exact", TEST, "--nocapture", "--test-threads=1"])
+                .env(CHILD, "1")
+                .env("RUSTFS_EXPERIMENTAL_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE", "false")
+                .env("RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE", "false")
+                .output()
+                .expect("fused overwrite subprocess");
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            assert!(output.status.success(), "{stdout}{stderr}");
+            assert!(stdout.contains("FUSED_OVERWRITE_OK"), "fused overwrite test did not run: {stdout}");
+            return;
+        }
+
+        assert!(!os::dst_dir_fsync_group_commit_enabled());
+        assert!(!os::file_fdatasync_group_commit_enabled());
+        let _mode = durability_mode_override::set(DurabilityMode::Strict);
+        let dir = tempfile::tempdir().expect("temp dir");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("utf8")).expect("endpoint");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
+        let (bucket, object, version_id) = ("fused-overwrite-bucket", "fused-object", Uuid::nil());
+        let (data_dir_v1, data_dir_v2) = (Uuid::new_v4(), Uuid::new_v4());
+        ensure_test_volume(&disk, bucket).await;
+        ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+
+        for (tmp, data_dir, payload) in [("tmp-v1", data_dir_v1, b"v1-d"), ("tmp-v2", data_dir_v2, b"v2-d")] {
+            let mut w = disk
+                .create_file(bucket, RUSTFS_META_TMP_BUCKET, &format!("{tmp}/{data_dir}/part.1"), 4)
+                .await
+                .expect("create shard");
+            w.write_all(payload).await.expect("write");
+            w.shutdown().await.expect("close");
+        }
+        let mut fi_v1 = test_file_info(object, version_id, Some(data_dir_v1), None);
+        (fi_v1.size, fi_v1.parts[0].size, fi_v1.parts[0].actual_size) = (4, 4, 4);
+        let resp1 = disk
+            .rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v1", fi_v1, bucket, object)
+            .await
+            .expect("commit v1");
+        assert_eq!((resp1.old_data_dir, resp1.rollback_data_dir), (None, None));
+        let dst_object_dir = disk.io_get_object_path(bucket, object).expect("dst object path");
+        let v1_meta_bytes = fs::read(dst_object_dir.join(STORAGE_FORMAT_FILE))
+            .await
+            .expect("read v1 xl.meta");
+
+        let mut fi_v2 = test_file_info(object, version_id, Some(data_dir_v2), None);
+        (fi_v2.size, fi_v2.parts[0].size, fi_v2.parts[0].actual_size) = (4, 4, 4);
+        assert!(!has_rename_data_test_failpoints(object));
+        let resp2 = disk
+            .rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v2", fi_v2, bucket, object)
+            .await
+            .expect("fused overwrite v2");
+        assert_eq!(
+            (resp2.old_data_dir, resp2.rollback_data_dir, resp2.cleanup_data_dir),
+            (Some(data_dir_v1), Some(data_dir_v1), Some(data_dir_v1))
+        );
+        assert_eq!(
+            fs::read(dst_object_dir.join(data_dir_v1.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))
+                .await
+                .expect("v1 backup"),
+            v1_meta_bytes
+        );
+
+        fs::create_dir_all(dir.path().join(RUSTFS_META_TMP_BUCKET).join("tmp-v3"))
+            .await
+            .expect("create tmp-v3");
+        let fi_v3 = test_file_info(object, version_id, None, Some(Bytes::from_static(b"v3-inline")));
+        let resp3 = disk
+            .rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v3", fi_v3, bucket, object)
+            .await
+            .expect("fused inline overwrite v3");
+        assert_eq!(
+            (resp3.old_data_dir, resp3.rollback_data_dir, resp3.cleanup_data_dir),
+            (Some(data_dir_v2), Some(data_dir_v2), Some(data_dir_v2))
+        );
+        assert!(
+            dst_object_dir
+                .join(data_dir_v2.to_string())
+                .join(STORAGE_FORMAT_FILE_BACKUP)
+                .is_file()
+        );
+
+        let dst_meta_path = dst_object_dir.join(STORAGE_FORMAT_FILE);
+        let dst_vol_dir = disk.io_get_bucket_path(bucket).expect("bucket path");
+        let lease = os::acquire_namespace_mutation_lease_with_owner(&dst_meta_path, None).await;
+        let guard =
+            commit::lock_rename_commit_directories(&dst_object_dir, &dst_object_dir, &dst_vol_dir, &disk.publication_root, lease)
+                .await
+                .expect("lock commit guard");
+        let replacement_dir = dst_object_dir.join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&replacement_dir).await.expect("create replacement_dir");
+        fs::write(replacement_dir.join("part.1"), b"live").await.expect("write part");
+
+        os::fsync_dir_recorder::set_failure(&dst_object_dir, std::io::ErrorKind::Other);
+        assert!(
+            commit::fsync_inline_commit_std(
+                &dst_meta_path,
+                &dst_vol_dir,
+                false,
+                Some(Uuid::new_v4()),
+                None,
+                Some(&replacement_dir),
+                &guard,
+            )
+            .is_err()
+        );
+        assert!(replacement_dir.join("part.1").is_file());
+
+        os::fsync_dir_recorder::set_failure(&dst_object_dir, std::io::ErrorKind::Other);
+        assert!(
+            commit::fsync_inline_commit_std(
+                &dst_meta_path,
+                &dst_vol_dir,
+                false,
+                Some(data_dir_v1),
+                None,
+                Some(&replacement_dir),
+                &guard,
+            )
+            .is_err()
+        );
+        assert!(!replacement_dir.exists());
+        assert_eq!(fs::read(&dst_meta_path).await.expect("restored"), v1_meta_bytes);
+        println!("FUSED_OVERWRITE_OK");
+    }
+
     /// rustfs/backlog#1009: `observe_old_current_size` must mirror the pre-PUT
     /// `get_object_info` semantics bit for bit — latest version's
     /// `ObjectInfo.size` (0 for a delete-marker latest, which that lookup
@@ -30309,6 +30448,7 @@ mod test {
             expected_delete_marker: None,
             delete_recovery_bucket_incarnation: None,
             recoverable_delete: false,
+            requires_bounded_delete_recovery: false,
             orphan_data_dir: false,
         };
         disk.delete("test-volume", "test-file.txt", delete_opts)

crates/ecstore/src/disk/local/commit.rs

@@ -15,6 +15,8 @@
 //! Single-disk object rename publication and rollback. The shared execution core
 //! retains instrumentation, mutation leases, and commit guards through the syscall.
 
+#[cfg(target_os = "linux")]
+use super::has_rename_data_test_failpoints;
 #[cfg(all(test, windows))]
 use super::run_destination_commit_directory_preparation;
 #[cfg(any(not(windows), test))]
@@ -96,7 +98,7 @@ struct RenameCommitContext<'a> {
     data_paths: Option<&'a (PathBuf, PathBuf)>,
 }
 
-struct StagedRollbackBackup(PathBuf);
+pub(super) struct StagedRollbackBackup(PathBuf);
 
 impl StagedRollbackBackup {
     fn path(&self) -> &Path {
@@ -233,11 +235,13 @@ fn rollback_inline_metadata_std(
     }
 }
 
-fn fsync_inline_commit_std(
+pub(super) fn fsync_inline_commit_std(
     dst: &Path,
     dst_volume_dir: &Path,
     destination_was_absent: bool,
+    rollback_data_dir: Option<Uuid>,
     local_backup: Option<&StagedRollbackBackup>,
+    new_data_dir: Option<&Path>,
     commit_guard: &os::RenameCommitGuard,
 ) -> std::io::Result<()> {
     if let Some(dst_parent) = dst.parent() {
@@ -247,7 +251,10 @@ fn fsync_inline_commit_std(
                 rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
                 fsync_started,
             );
-            rollback_inline_metadata_std(dst, None, local_backup)?;
+            rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
+            if let Some(new_data_path) = new_data_dir {
+                remove_dir_all_if_exists(new_data_path)?;
+            }
             return Err(err);
         }
         rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -268,7 +275,10 @@ fn fsync_inline_commit_std(
                     rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
                     fsync_started,
                 );
-                rollback_inline_metadata_std(dst, None, local_backup)?;
+                rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
+                if let Some(new_data_path) = new_data_dir {
+                    remove_dir_all_if_exists(new_data_path)?;
+                }
                 return Err(err);
             }
             rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -827,7 +837,7 @@ impl LocalDisk {
                         None
                     }
                 });
-                if rollback_data_dir.is_some() {
+                if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
                     return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 #[cfg(test)]
@@ -850,6 +860,9 @@ impl LocalDisk {
                     &dst_path_str,
                     version_id,
                 );
+                if let Some(old_data_dir) = has_old_data_dir.as_ref() {
+                    let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
+                }
                 xlmeta.add_version(fi_moved)?;
                 let version_signature = rename_data_versions_signature(&xlmeta);
                 let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
@@ -869,8 +882,9 @@ impl LocalDisk {
                 }
 
                 let mut staged_rollback_path = None;
-                if let Some(ref old_metadata) = has_dst_buf
-                    && (sync_commit_metadata || cfg!(test))
+                if (!no_inline || rollback_data_dir.is_none())
+                    && let Some(ref old_metadata) = has_dst_buf
+                    && (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
                 {
                     let backup_path = create_local_inline_rollback_backup(
                         &dst_file_path_owned,
@@ -971,9 +985,52 @@ impl LocalDisk {
                         }
                         return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
                     }
+                    if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
+                        let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
+                        if let Err(err) =
+                            os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
+                        {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
+                        }
+                        let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
+                            Ok(g) => g,
+                            Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
+                        };
+                        let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
+                        if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
+                        }
+                        #[cfg(test)]
+                        run_owned_file_before_open(&backup_path);
+                        if let Err(err) = backup_path_guard.write_file_for_path_access(
+                            &backup_path,
+                            dst_buf.as_ref(),
+                            sync_commit_metadata,
+                            sync_commit_metadata,
+                        ) {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
+                        }
+                    }
                     if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
                         return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
+                } else if let (Some(old_data_dir), Some(staged_backup)) =
+                    (rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
+                {
+                    let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
+                    let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
+                    if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
+                        staged_backup,
+                        &backup_path,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                        None,
+                        sync_commit_metadata,
+                    ) {
+                        let _ = remove_file_if_exists(staged_backup);
+                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
+                    }
+                    staged_rollback_path = None;
                 }
 
                 let rename_res = if should_fail_commit_rename(&dst_path_str) {
@@ -1001,15 +1058,16 @@ impl LocalDisk {
 
                 if should_fail_after_metadata_commit(&dst_path_str) {
                     if no_inline {
-                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
+                        rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())?;
                         if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
                             remove_dir_all_if_exists(new_data_path)?;
                         }
                         return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
-                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
-                        .err()
-                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                    let err =
+                        rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())
+                            .err()
+                            .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
                 }
 
@@ -1022,14 +1080,16 @@ impl LocalDisk {
                         &dst_file_path_owned,
                         &dst_volume_dir_owned,
                         destination_was_absent,
+                        rollback_data_dir,
                         staged_rollback_path.as_ref(),
+                        has_data_dir_path_owned
+                            .as_ref()
+                            .filter(|_| no_inline)
+                            .map(|(_, new_data_path)| new_data_path.as_path()),
                         &guard,
                     )
                 {
                     if no_inline {
-                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
-                            remove_dir_all_if_exists(new_data_path)?;
-                        }
                         return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
@@ -1045,9 +1105,9 @@ impl LocalDisk {
 
                 Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
                     multipart_source: None,
-                    old_data_dir: None,
-                    rollback_data_dir: None,
-                    cleanup_data_dir: None,
+                    old_data_dir: has_old_data_dir,
+                    rollback_data_dir,
+                    cleanup_data_dir: has_old_data_dir,
                     sign: version_signature,
                     old_current_size,
                     old_current_source_checked,
@@ -1536,8 +1596,9 @@ impl LocalDisk {
                     let dst_file_path = dst_file_path.clone();
                     let owned_directory_guard = commit.directory_guard.clone();
                     let owned_mutation_lease = commit.mutation_lease.clone();
+                    let sync = tmp_meta_sync != SyncMode::None;
                     async move {
-                        os::run_blocking_namespace_operation(owned_mutation_lease, move || {
+                        os::run_blocking_namespace_operation_with_sync(owned_mutation_lease, sync, move || {
                             #[cfg(test)]
                             run_owned_file_before_open(&src_file_path);
                             let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
@@ -1545,7 +1606,7 @@ impl LocalDisk {
                                 &dst_file_path,
                                 &owned_directory_guard,
                             )?;
-                            prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
+                            prepared_metadata_source.write_all(&new_dst_buf, sync)?;
                             Ok(prepared_metadata_source)
                         })
                         .await
@@ -1708,13 +1769,15 @@ impl LocalDisk {
                 // sync finish. A detached spawn_blocking writer could survive
                 // cancellation and later truncate a newer transaction's
                 // deterministic rollback backup.
-                let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
+                let lease = commit.mutation_lease.clone();
+                let sync = backup_sync != SyncMode::None;
+                let write_result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
                     #[cfg(test)]
                     run_owned_file_before_open(&backup_path);
                     backup_path_guard.write_file_for_path_access(
                         &backup_path,
                         backup_bytes.as_ref(),
-                        backup_sync != SyncMode::None,
+                        sync,
                         backup_sync == SyncMode::FileAndDir,
                     )
                 })
@@ -2154,7 +2217,9 @@ impl LocalDisk {
                             &dst,
                             &dst_volume_dir_for_commit,
                             destination_was_absent,
+                            rollback_data_dir,
                             staged_rollback_path.as_ref(),
+                            None,
                             &rename_commit_guard_for_preparation,
                         )
                     {

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -271,6 +271,9 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
         compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
         compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
     ] {
+        if canonical.metadata.is_empty() {
+            break;
+        }
         if !compat::contains_key_str(&canonical.metadata, suffix) {
             continue;
         }
@@ -301,22 +304,24 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
         }
         compat::insert_str(&mut canonical.metadata, suffix, value);
     }
-    fn ordered(value: serde_json::Value) -> serde_json::Value {
+    fn sort_json_in_place(value: &mut serde_json::Value) {
         match value {
             serde_json::Value::Object(map) => {
-                let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect();
-                serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect())
+                map.sort_keys();
+                map.values_mut().for_each(sort_json_in_place);
             }
-            serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()),
-            other => other,
+            serde_json::Value::Array(values) => values.iter_mut().for_each(sort_json_in_place),
+            _ => {}
         }
     }
+    let mut json_value = serde_json::to_value(&canonical)?;
+    sort_json_in_place(&mut json_value);
     Ok(crate::disk::DeleteRecoveryVersionIdentity {
         version_id: canonical.version_id,
         mod_time: canonical.mod_time,
         data_dir: canonical.data_dir,
         deleted: canonical.deleted,
-        metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?),
+        metadata_digest: digest(&serde_json::to_vec(&json_value)?),
     })
 }
 
@@ -427,6 +432,13 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
     if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 {
         return Err(DiskError::FileCorrupt);
     }
+    // MessagePack encoding of `DeleteRecoveryRpcReply::Inspected` is more compact
+    // than JSON; only construct and serialize `projected` when the JSON size plus
+    // `current_versions` and fixed envelope could approach the RPC payload limit.
+    let rpc_budget = rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES.saturating_sub(4096);
+    if bytes.len().saturating_mul(2).saturating_add(4096) <= rpc_budget {
+        return Ok(bytes);
+    }
     let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection {
         protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
         id: DeleteTransactionId {
@@ -443,7 +455,7 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
         explicit_delete_safe: false,
         fingerprint: [0; 32],
     });
-    if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 {
+    if rmp_serde::to_vec_named(&projected)?.len() > rpc_budget {
         return Err(DiskError::FileCorrupt);
     }
     Ok(bytes)
@@ -7168,6 +7180,86 @@ mod tests {
         assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
         assert!(!root.exists(), "empty object directory must be pruned");
     }
+
+    #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+    #[tokio::test]
+    async fn delete_recovery_receipt_formatting_and_post_finished_unlink_interruption() {
+        let (_temp, disk, info, _) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object path");
+
+        let mut v1 = info.clone();
+        v1.version_id = Some(Uuid::new_v4());
+        let mut v2 = info.clone();
+        v2.version_id = Some(Uuid::new_v4());
+        let mut meta = FileMeta::new();
+        meta.add_version(v1.clone()).expect("add v1");
+        meta.add_version(v2.clone()).expect("add v2");
+        fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
+            .await
+            .expect("write xl.meta");
+
+        let mut rev_info = v1.clone();
+        rev_info
+            .metadata
+            .extend([("x-amz-meta-z".into(), "1".into()), ("x-amz-meta-a".into(), "2".into())]);
+        let mut fwd_info = v1.clone();
+        fwd_info
+            .metadata
+            .extend([("x-amz-meta-a".into(), "2".into()), ("x-amz-meta-z".into(), "1".into())]);
+        assert_eq!(
+            generation(rev_info).expect("rev").metadata_digest,
+            generation(fwd_info).expect("fwd").metadata_digest
+        );
+
+        let transaction = Uuid::new_v4();
+        let opts = DeleteOptions {
+            recoverable_delete: true,
+            ..Default::default()
+        };
+        assert_eq!(
+            disk.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
+                .await
+                .expect("fused versioned delete preparation"),
+            Some(())
+        );
+        let (journal, sole_transaction, target_satisfied) = disk
+            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
+            .await
+            .expect("local publish")
+            .expect("versioned target satisfied");
+        assert!(sole_transaction && target_satisfied && journal.target_digest.is_some());
+
+        let rollback = root.join(transaction.to_string());
+        // Real finalizer failure at the second fsync_dir_std(&rollback) (after [TARGET, PREPARED, FINISHED] unlink)
+        // must leave a durable finished=true receipt that inspects as CommitFinished.
+        let rollback_for_hook = rollback.clone();
+        let _hook = os::prepared_publication_test_hooks::install_at(
+            os::prepared_publication_test_hooks::Stage::Remove,
+            &rollback.join(FINISHED),
+            move || {
+                os::fsync_dir_recorder::set_failure(&rollback_for_hook, std::io::ErrorKind::Other);
+            },
+        );
+        assert!(
+            disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+                .await
+                .is_err()
+        );
+        assert!(!rollback.join(FINISHED).exists());
+        assert_eq!(
+            disk.inspect_delete_transaction("recovery", "object", transaction)
+                .await
+                .expect("post-FINISHED-unlink failure")
+                .state,
+            DeleteRecoveryState::CommitFinished
+        );
+        assert!(
+            disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+                .await
+                .expect("finalize")
+        );
+        assert!(!rollback.exists());
+    }
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -398,7 +398,9 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        let sync_lane = sync_directory && incoming.data.is_none() && !incoming.is_remote();
+        let lease = Arc::clone(&staging.lease);
+        let fresh_head = os::run_cancellable_blocking_namespace_operation_with_sync(lease, sync_lane, move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }

crates/ecstore/src/disk/mod.rs

@@ -2125,10 +2125,19 @@ pub struct GuardedFileInfoVersions {
 pub struct GuardedDeleteVersionsPayload {
     pub versions: Vec<GuardedFileInfoVersions>,
     pub opts: DeleteOptions,
+    // The strict legacy decoder rejects this field before applying any change.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub bounded_recovery_protocol: Option<u32>,
 }
 
 impl GuardedDeleteVersionsPayload {
     pub fn validate(&self) -> Result<()> {
+        if self
+            .bounded_recovery_protocol
+            .is_some_and(|version| version != BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION)
+        {
+            return Err(DiskError::UnsupportedDisk);
+        }
         validate_delete_recovery_options(&self.opts)?;
         if self.versions.is_empty()
             || self.versions.len() > 1000
@@ -2306,6 +2315,9 @@ pub struct DeleteOptions {
     pub delete_recovery_bucket_incarnation: Option<Uuid>,
     #[serde(default)]
     pub recoverable_delete: bool,
+    /// Transport requirement for preparing a delete that will use bounded settlement.
+    #[serde(skip)]
+    pub requires_bounded_delete_recovery: bool,
     /// Reclaim only plain erasure-part leaves, preserving every other entry.
     /// Keep `recursive` false so an older peer cannot widen this operation.
     #[serde(default)]
@@ -2313,6 +2325,8 @@ pub struct DeleteOptions {
 }
 
 pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1;
+/// Accepted only for prepare RPCs. Legacy peers must refuse before staging a delete.
+pub const BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION: u32 = 2;
 pub(crate) fn get_delete_recovery_timeout() -> Result<std::time::Duration> {
     let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS");
     match value {
@@ -2909,6 +2923,7 @@ mod tests {
         insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32));
         let generation = CurrentDeleteGeneration::from_file_info(&fi).expect("capture complete remote identity");
         let payload = GuardedDeleteVersionsPayload {
+            bounded_recovery_protocol: None,
             versions: vec![GuardedFileInfoVersions {
                 group: FileInfoVersions {
                     name: "object".into(),
@@ -2963,6 +2978,7 @@ mod tests {
             expected_current: CurrentDeleteGeneration::Absent,
         };
         let mut payload = GuardedDeleteVersionsPayload {
+            bounded_recovery_protocol: None,
             versions: vec![group],
             opts: DeleteOptions {
                 old_data_dir: Some(Uuid::new_v4()),
@@ -2974,6 +2990,25 @@ mod tests {
         let decoded: GuardedDeleteVersionsPayload =
             rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode guarded batch");
         decoded.validate().expect("round-trip guarded batch remains valid");
+        #[derive(Debug, Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyGuardedDeletePayload {
+            #[serde(rename = "versions")]
+            _versions: Vec<GuardedFileInfoVersions>,
+            #[serde(rename = "opts")]
+            _opts: DeleteOptions,
+        }
+        rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&encoded)
+            .expect("legacy reader accepts legacy prepare");
+        payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION);
+        let bounded = rmp_serde::to_vec_named(&payload).expect("serialize bounded guarded prepare");
+        assert!(rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&bounded).is_err());
+        let bounded: GuardedDeleteVersionsPayload =
+            rustfs_protos::decode_delete_recovery_payload(&bounded).expect("current reader accepts bounded prepare");
+        bounded.validate().expect("valid bounded guarded prepare");
+        payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION + 1);
+        assert_eq!(payload.validate(), Err(DiskError::UnsupportedDisk));
+        payload.bounded_recovery_protocol = None;
         let mut trailing = encoded;
         trailing.push(0xc0);
         assert!(rustfs_protos::decode_delete_recovery_payload::<GuardedDeleteVersionsPayload>(&trailing).is_err());
@@ -3168,9 +3203,15 @@ mod tests {
             expected_delete_marker: None,
             delete_recovery_bucket_incarnation: None,
             recoverable_delete: false,
+            requires_bounded_delete_recovery: true,
             orphan_data_dir: false,
         };
 
+        let wire = serde_json::to_value(&opts).expect("encode legacy wire options");
+        assert!(wire.get("requires_bounded_delete_recovery").is_none());
+        let decoded: DeleteOptions = serde_json::from_value(wire).expect("decode legacy wire options");
+        assert!(!decoded.requires_bounded_delete_recovery);
+
         assert!(opts.recursive);
         assert!(!opts.immediate);
         assert!(opts.undo_write);

crates/ecstore/src/disk/multipart_source.rs

@@ -1149,6 +1149,8 @@ fn normalized_info(fi: &FileInfo, completion: bool) -> FileInfo {
             part.checksums = None;
             part.index = None;
             part.integrity = None;
+            // The completed object carries the aggregate proof in metadata.
+            part.encrypted_etag = false;
         }
     }
     fi
@@ -1739,6 +1741,12 @@ mod multipart_proof_authority_tests {
         completed.mod_time = Some(original.mod_time.unwrap() + time::Duration::seconds(1));
         completed.size = 15;
         completed.add_object_part(1, "etag".into(), 15, completed.mod_time, 15, None, None);
+        completed.parts[0].encrypted_etag = true;
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut completed.metadata,
+            rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
+            "v1".to_owned(),
+        );
         let publication = condition.attach(&mut completed).unwrap();
         assert_eq!(publication.version_id, None);
         assert_eq!(
@@ -1748,6 +1756,11 @@ mod multipart_proof_authority_tests {
         );
         let mut read_completed = persist_null_fixture(&completed);
         assert_eq!(read_completed.version_id, None);
+        assert!(!read_completed.parts[0].encrypted_etag, "xl.meta stores the aggregate proof");
+        assert_eq!(
+            rustfs_utils::http::get_consistent_str(&read_completed.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+            Some("v1")
+        );
         let reread_publication = reread_condition.attach(&mut read_completed).unwrap();
         assert_eq!(
             publication, reread_publication,

crates/ecstore/src/disk/os.rs

@@ -3808,7 +3808,7 @@ pub(crate) async fn rename_reconciled_metadata(
     authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
 ) -> Result<()> {
     let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
-    run_blocking_namespace_operation(lease, move || {
+    run_blocking_namespace_operation_with_sync(lease, true, move || {
         let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
         #[cfg(all(any(test, feature = "test-util"), not(windows)))]
         prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
@@ -4808,6 +4808,23 @@ pub(crate) async fn run_cancellable_blocking_namespace_operation<T: Send + 'stat
         .await
 }
 
+pub(crate) async fn run_cancellable_blocking_namespace_operation_with_sync<T: Send + 'static>(
+    lease: Arc<NamespaceMutationLease>,
+    sync: bool,
+    operation: impl FnOnce(&dyn Fn() -> bool) -> io::Result<T> + Send + 'static,
+) -> io::Result<T> {
+    if !sync {
+        return run_cancellable_blocking_namespace_operation(lease, operation).await;
+    }
+    let queue = Arc::clone(&lease.blocking_queue);
+    queue
+        .run_fsync_cancellable(move |is_cancelled| {
+            let _lease = lease;
+            operation(is_cancelled)
+        })
+        .await
+}
+
 /// Admit one strict inline commit under the disk sync limit. The caller already
 /// owns the namespace lease, establishing namespace -> disk ordering. Holding
 /// admission across adjacent durability barriers prevents one transaction from

crates/ecstore/src/erasure/coding/encode.rs

@@ -326,6 +326,40 @@ impl Default for WriteProgressPolicy {
     }
 }
 
+/// Caller-specific completion behavior for an erasure write.
+///
+/// A multipart upload must not discard a different healthy shard from each
+/// part merely because another quorum finished first: the final object needs
+/// a quorum of disks carrying every part. Pending writers still obey the
+/// existing progress and absolute deadlines and never count as completed.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ShardWritePolicy {
+    quorum: usize,
+    post_quorum: PostQuorumPolicy,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PostQuorumPolicy {
+    Grace,
+    ExistingBudget,
+}
+
+impl ShardWritePolicy {
+    pub(crate) fn object(quorum: usize) -> Self {
+        Self {
+            quorum,
+            post_quorum: PostQuorumPolicy::Grace,
+        }
+    }
+
+    pub(crate) fn multipart(quorum: usize) -> Self {
+        Self {
+            quorum,
+            post_quorum: PostQuorumPolicy::ExistingBudget,
+        }
+    }
+}
+
 /// After write quorum is already in hand, how long a slower shard may still
 /// finish before it is dropped. The per-shard stall budget (30s by default)
 /// is what a black-hole peer is allowed to consume *before* quorum exists.
@@ -338,13 +372,15 @@ fn post_quorum_straggler_grace(stall_timeout: Option<std::time::Duration>) -> Op
     stall_timeout.map(|stall| stall.min(WRITE_QUORUM_STRAGGLER_GRACE))
 }
 
-/// Drive shard ops until write quorum has succeeded and stragglers have had
-/// [`post_quorum_straggler_grace`] to finish. Returns `true` for each slot
-/// whose future completed; `false` means the caller must fail that slot.
+/// Drive shard operations within their existing deadlines. Ordinary object
+/// writes additionally stop after the post-quorum grace; multipart writes keep
+/// polling all candidates within the same operation budget. A `false` slot
+/// never completed and must be excluded from commit.
 async fn await_write_quorum<F>(
     write_quorum: usize,
     stall_timeout: Option<std::time::Duration>,
     slot_count: usize,
+    post_quorum: PostQuorumPolicy,
     mut futures: FuturesUnordered<F>,
 ) -> Vec<bool>
 where
@@ -376,6 +412,7 @@ where
         if succeeded {
             successes = successes.saturating_add(1);
             if successes >= write_quorum
+                && post_quorum == PostQuorumPolicy::Grace
                 && grace.is_none()
                 && let Some(delay) = post_quorum_straggler_grace(stall_timeout)
             {
@@ -404,6 +441,7 @@ pub(crate) struct MultiWriter<'a> {
     writers: &'a mut [Option<BitrotWriterWrapper>],
     integrity: Option<&'a mut IntegrityBuilder>,
     write_quorum: usize,
+    post_quorum: PostQuorumPolicy,
     errs: Vec<Option<Error>>,
     policy: WriteProgressPolicy,
     /// Absolute deadline for the whole object, armed lazily on the first
@@ -412,16 +450,21 @@ pub(crate) struct MultiWriter<'a> {
 }
 
 impl<'a> MultiWriter<'a> {
-    pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
-        Self::with_policy(writers, write_quorum, WriteProgressPolicy::default())
+    pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_policy: ShardWritePolicy) -> Self {
+        Self::with_policy(writers, write_policy, WriteProgressPolicy::default())
     }
 
-    pub fn with_policy(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize, policy: WriteProgressPolicy) -> Self {
+    pub fn with_policy(
+        writers: &'a mut [Option<BitrotWriterWrapper>],
+        write_policy: ShardWritePolicy,
+        policy: WriteProgressPolicy,
+    ) -> Self {
         let length = writers.len();
         MultiWriter {
             writers,
             integrity: None,
-            write_quorum,
+            write_quorum: write_policy.quorum,
+            post_quorum: write_policy.post_quorum,
             errs: vec![None; length],
             policy,
             absolute_deadline: None,
@@ -506,9 +549,8 @@ impl<'a> MultiWriter<'a> {
                 // failed and its disk dropped, so a stalled peer cannot pin an
                 // otherwise-healthy write quorum (rustfs/backlog#1319). `budget`
                 // is recomputed per block, so it bounds a stall — not the total
-                // transfer time — and a slow-but-honest writer is never killed
-                // while quorum is still open. Once quorum has succeeded, a
-                // straggler only gets `WRITE_QUORUM_STRAGGLER_GRACE` more time.
+                // transfer time. Only ordinary object writes add the shorter
+                // post-quorum grace; multipart preserves this operation budget.
                 futures.push(async move {
                     match budget {
                         Some(budget) => match tokio::time::timeout(budget, Self::write_shard(writer_opt, err, shard)).await {
@@ -524,7 +566,7 @@ impl<'a> MultiWriter<'a> {
                     (index, succeeded)
                 });
             }
-            await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
+            await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
         };
         abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
 
@@ -587,8 +629,8 @@ impl<'a> MultiWriter<'a> {
                 // here forever for a small object whose bytes were fully buffered
                 // (so `write` never blocked). Bound it with the same progress
                 // budget and drop the stalled writer before the quorum check.
-                // After quorum has shut down, do not keep waiting out the rest of
-                // that budget for a peer that has stopped answering.
+                // Ordinary object writes also apply their post-quorum grace;
+                // multipart waits for all candidates within this same budget.
                 futures.push(async move {
                     match budget {
                         Some(budget) => match tokio::time::timeout(budget, Self::shutdown_writer(writer_opt, err)).await {
@@ -604,7 +646,7 @@ impl<'a> MultiWriter<'a> {
                     (index, succeeded)
                 });
             }
-            await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
+            await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
         };
         abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
 
@@ -639,7 +681,7 @@ impl Erasure {
         self: Arc<Self>,
         reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         part_number: usize,
         mode: IntegrityEncodeMode,
         protected: bool,
@@ -668,19 +710,19 @@ impl Erasure {
             }
             IntegrityEncodeMode::SingleBlock(hint) => {
                 let (reader, size) = self
-                    .encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
+                    .encode_small_direct(reader, writers, write_policy, true, hint, integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
             IntegrityEncodeMode::Streaming => {
                 let (reader, size) = self
-                    .encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
+                    .encode_with_ingest_mode(reader, writers, write_policy, use_bytesmut_ingest(), integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
             IntegrityEncodeMode::Batched => {
                 let (reader, size) = self
-                    .encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
+                    .encode_batched_with_integrity(reader, writers, write_policy, integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
@@ -745,7 +787,7 @@ impl Erasure {
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         require_single_block: bool,
         size_hint: usize,
         integrity: Option<&mut IntegrityBuilder>,
@@ -780,7 +822,7 @@ impl Erasure {
         }
 
         let block = self.encode_data_owned_block(buf)?;
-        let mut mw = MultiWriter::new(writers, quorum);
+        let mut mw = MultiWriter::new(writers, write_policy);
         mw.integrity = integrity;
         mw.write_block(&block).await?;
         mw.shutdown().await?;
@@ -847,7 +889,7 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
         let use_bytesmut_ingest = use_bytesmut_ingest();
-        self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
+        self.encode_with_ingest_mode(reader, writers, ShardWritePolicy::object(quorum), use_bytesmut_ingest, None)
             .await
     }
 
@@ -859,7 +901,7 @@ impl Erasure {
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         use_bytesmut_ingest: bool,
         integrity: Option<&mut IntegrityBuilder>,
     ) -> std::io::Result<(R, usize)>
@@ -961,7 +1003,7 @@ impl Erasure {
             Ok((reader, total))
         }));
 
-        let mut writers = MultiWriter::new(writers, quorum);
+        let mut writers = MultiWriter::new(writers, write_policy);
         writers.integrity = integrity;
 
         let mut write_err = None;
@@ -1017,14 +1059,15 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
-        self.encode_batched_with_integrity(reader, writers, quorum, None).await
+        self.encode_batched_with_integrity(reader, writers, ShardWritePolicy::object(quorum), None)
+            .await
     }
 
     async fn encode_batched_with_integrity<R>(
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         integrity: Option<&mut IntegrityBuilder>,
     ) -> std::io::Result<(R, usize)>
     where
@@ -1105,7 +1148,7 @@ impl Erasure {
             Ok((reader, total))
         }));
 
-        let mut writers = MultiWriter::new(writers, quorum);
+        let mut writers = MultiWriter::new(writers, write_policy);
         writers.integrity = integrity;
         let mut write_err = None;
 
@@ -1165,7 +1208,7 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin,
     {
         let size_hint = self.block_size;
-        self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
             .await
     }
 
@@ -1182,7 +1225,7 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin,
     {
-        self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
             .await
     }
 
@@ -1199,7 +1242,8 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin,
     {
         let size_hint = self.block_size;
-        self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
+            .await
     }
 
     /// Size-aware single-block fast path. `size_hint` only controls the bounded
@@ -1215,7 +1259,8 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin,
     {
-        self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
+            .await
     }
 }
 
@@ -1632,6 +1677,219 @@ mod tests {
         }
     }
 
+    #[derive(Clone, Copy, Debug)]
+    enum DelayedPhase {
+        Write,
+        Flush,
+        Shutdown,
+    }
+
+    struct PhaseDelayedWriter {
+        inner: DeferredCommitWriter,
+        phase: DelayedPhase,
+        delay: Duration,
+        timer: Option<Pin<Box<tokio::time::Sleep>>>,
+        ready: bool,
+    }
+
+    impl PhaseDelayedWriter {
+        fn poll_delay(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+            if self.ready {
+                return Poll::Ready(());
+            }
+            let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep(self.delay)));
+            match timer.as_mut().poll(cx) {
+                Poll::Pending => Poll::Pending,
+                Poll::Ready(()) => {
+                    self.ready = true;
+                    Poll::Ready(())
+                }
+            }
+        }
+    }
+
+    impl AsyncWrite for PhaseDelayedWriter {
+        fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
+            if matches!(self.phase, DelayedPhase::Write) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_write(cx, bytes)
+        }
+
+        fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            if matches!(self.phase, DelayedPhase::Flush) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_flush(cx)
+        }
+
+        fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            if matches!(self.phase, DelayedPhase::Shutdown) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_shutdown(cx)
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_keeps_slow_writers_in_every_encode_mode() {
+        const BLOCK: usize = 64;
+        for make_mode in [
+            || IntegrityEncodeMode::SingleBlock(BLOCK),
+            || IntegrityEncodeMode::Streaming,
+            || IntegrityEncodeMode::Batched,
+        ] {
+            for phase in [DelayedPhase::Write, DelayedPhase::Flush, DelayedPhase::Shutdown] {
+                for delay in [Duration::from_secs(2), Duration::from_secs(5)] {
+                    let committed: Vec<_> = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
+                    let mut writers = committed
+                        .iter()
+                        .enumerate()
+                        .map(|(slot, data)| {
+                            if slot == 0 {
+                                Some(bitrot_writer(
+                                    PhaseDelayedWriter {
+                                        inner: DeferredCommitWriter::new(data.clone()),
+                                        phase,
+                                        delay,
+                                        timer: None,
+                                        ready: false,
+                                    },
+                                    BLOCK / 2,
+                                ))
+                            } else {
+                                Some(bitrot_writer(DeferredCommitWriter::new(data.clone()), BLOCK / 2))
+                            }
+                        })
+                        .collect::<Vec<_>>();
+                    let payload = vec![0x5a; BLOCK];
+                    let erasure = Arc::new(Erasure::new(2, 2, BLOCK));
+                    let started = tokio::time::Instant::now();
+                    erasure
+                        .encode_with_shard_integrity(
+                            Cursor::new(payload.clone()),
+                            &mut writers,
+                            ShardWritePolicy::multipart(3),
+                            1,
+                            make_mode(),
+                            false,
+                        )
+                        .await
+                        .expect("multipart must keep finite slow writers within the existing budget");
+                    assert!(started.elapsed() >= delay, "must await the slow phase: {phase:?}");
+                    assert!(writers.iter().all(Option::is_some), "all four completed shards must survive");
+                    for data in &committed {
+                        let bytes = data.lock().expect("committed bytes").clone();
+                        crate::erasure::coding::bitrot_verify(
+                            Cursor::new(bytes.clone()),
+                            bytes.len(),
+                            BLOCK / 2,
+                            HashAlgorithm::HighwayHash256S,
+                            BLOCK / 2,
+                        )
+                        .await
+                        .expect("each completed shard must retain valid bitrot framing");
+                    }
+                }
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_expires_blackholes_without_restarting_deadlines() {
+        for shutdown in [false, true] {
+            for blackholes in [1, 2] {
+                for absolute in [Duration::ZERO, Duration::from_secs(3)] {
+                    let mut writers = (0..4)
+                        .map(|slot| {
+                            if slot < blackholes {
+                                Some(bitrot_writer_plain(
+                                    if shutdown {
+                                        StallingWriter::stalls_on_shutdown()
+                                    } else {
+                                        StallingWriter::stalls_on_write(0)
+                                    },
+                                    64,
+                                ))
+                            } else {
+                                Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))
+                            }
+                        })
+                        .collect::<Vec<_>>();
+                    let started = tokio::time::Instant::now();
+                    let result = {
+                        let mut writer = MultiWriter::with_policy(
+                            &mut writers,
+                            ShardWritePolicy::multipart(3),
+                            WriteProgressPolicy::new(Duration::from_secs(7), absolute),
+                        );
+                        if shutdown {
+                            writer.write(four_shards()).await.expect("buffered writes should complete");
+                            tokio::time::sleep(Duration::from_secs(1)).await;
+                            writer.shutdown().await
+                        } else {
+                            writer.write(four_shards()).await
+                        }
+                    };
+                    assert_eq!(result.is_ok(), blackholes == 1, "only a confirmed quorum may succeed");
+                    let expected = if absolute.is_zero() {
+                        Duration::from_secs(7 + u64::from(shutdown))
+                    } else {
+                        absolute
+                    };
+                    assert_eq!(started.elapsed(), expected, "budget must not restart at quorum or shutdown");
+                    assert!(writers[..blackholes].iter().all(Option::is_none));
+                    assert!(writers[blackholes..].iter().all(Option::is_some));
+                }
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_cancellation_drops_pending_writers() {
+        struct DropSignal(Option<oneshot::Sender<()>>);
+        impl Drop for DropSignal {
+            fn drop(&mut self) {
+                if let Some(tx) = self.0.take() {
+                    let _ = tx.send(());
+                }
+            }
+        }
+        impl AsyncWrite for DropSignal {
+            fn poll_write(self: Pin<&mut Self>, _: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
+                Poll::Ready(Ok(bytes.len()))
+            }
+            fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+                Poll::Ready(Ok(()))
+            }
+            fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+                Poll::Pending
+            }
+        }
+        let (tx, rx) = oneshot::channel();
+        let (entered_tx, entered) = oneshot::channel();
+        let task = tokio::spawn(async move {
+            let mut writers = vec![Some(bitrot_writer_plain(DropSignal(Some(tx)), 64))];
+            writers.extend(
+                (0..3).map(|_| Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))),
+            );
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::multipart(3));
+            writer.write(four_shards()).await.expect("buffered writes should succeed");
+            let _ = entered_tx.send(());
+            writer.shutdown().await
+        });
+        entered.await.expect("write must reach shutdown");
+        tokio::task::yield_now().await;
+        assert!(!task.is_finished());
+        task.abort();
+        assert!(
+            task.await
+                .expect_err("caller cancellation should cancel the task")
+                .is_cancelled()
+        );
+        rx.await.expect("pending writer must be dropped with its owning task");
+    }
+
     /// A writer that always makes forward progress but each `poll_write` first
     /// waits `delay` (on the runtime timer). Under a paused virtual clock this
     /// deterministically models a slow-but-honest peer: it must never be killed
@@ -1704,12 +1962,16 @@ mod tests {
         let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
 
         let encode = match pipeline {
-            EncodePipeline::Vec => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
-            }
-            EncodePipeline::BytesMut => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
-            }
+            EncodePipeline::Vec => tokio::spawn(async move {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }),
+            EncodePipeline::BytesMut => tokio::spawn(async move {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }),
             EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
         };
 
@@ -1767,8 +2029,16 @@ mod tests {
         ))];
 
         let result = match pipeline {
-            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
-            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
+            EncodePipeline::Vec => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }
+            EncodePipeline::BytesMut => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
         };
 
@@ -1841,8 +2111,16 @@ mod tests {
         let mut writers = vec![Some(bitrot_writer(FailAfterReaderErrorWriter { reader_failed }, BLOCK_SIZE))];
 
         let result = match pipeline {
-            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
-            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
+            EncodePipeline::Vec => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }
+            EncodePipeline::BytesMut => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
         };
 
@@ -1879,7 +2157,7 @@ mod tests {
         let erasure_for_task = erasure.clone();
         let encode = tokio::spawn(async move {
             erasure_for_task
-                .encode_with_ingest_mode(reader, &mut writers, 1, false, None)
+                .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
                 .await
         });
 
@@ -2048,8 +2326,16 @@ mod tests {
             let erasure = Arc::new(Erasure::new(2, 2, 64));
             let result = match path {
                 "direct" => erasure.encode_single_block_non_inline(reader, &mut writers, 2).await,
-                "vec" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, false, None).await,
-                "bytesmut" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, true, None).await,
+                "vec" => {
+                    erasure
+                        .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), false, None)
+                        .await
+                }
+                "bytesmut" => {
+                    erasure
+                        .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), true, None)
+                        .await
+                }
                 "batched" => erasure.encode_batched(reader, &mut writers, 2).await,
                 _ => unreachable!(),
             };
@@ -2094,7 +2380,7 @@ mod tests {
     async fn multi_writer_short_write_fails_before_shutdown() {
         let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 32))];
         let err = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             writer
                 .write(vec![Bytes::from_static(b"short-write payload")])
                 .await
@@ -2111,7 +2397,7 @@ mod tests {
     async fn multi_writer_reports_fallback_summary_when_only_offline_writers_remain() {
         let mut writers = vec![None, None];
         let (err, summary) = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             let err = writer
                 .write(vec![Bytes::from_static(b"offline-a"), Bytes::from_static(b"offline-b")])
                 .await
@@ -2131,7 +2417,7 @@ mod tests {
         assert!(summary.contains("required=1"));
 
         let (shutdown_err, summary) = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             let err = writer
                 .shutdown()
                 .await
@@ -2155,7 +2441,7 @@ mod tests {
     async fn multi_writer_reports_quorum_failure_when_quorum_exceeds_writer_count() {
         let committed = Arc::new(Mutex::new(Vec::new()));
         let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 32))];
-        let mut writer = MultiWriter::new(&mut writers, 2);
+        let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(2));
 
         let err = writer
             .write(vec![Bytes::from_static(b"quorum impossible")])
@@ -2244,7 +2530,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("one stalled writer must still satisfy a 3/4 write quorum without hanging");
@@ -2271,7 +2557,7 @@ mod tests {
         let started = tokio::time::Instant::now();
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("quorum must succeed without waiting out the black-hole stall");
@@ -2304,7 +2590,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("a straggler inside the grace window must still satisfy the write");
@@ -2329,7 +2615,7 @@ mod tests {
         ];
 
         let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-        let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+        let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
         let err = mw
             .write(four_shards())
             .await
@@ -2358,7 +2644,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards()).await.expect("all writers accept the buffered shard");
             mw.shutdown()
                 .await
@@ -2379,7 +2665,7 @@ mod tests {
         ];
 
         let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-        let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+        let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
         mw.write(four_shards()).await.expect("all writers accept the buffered shard");
         let err = mw
             .shutdown()
@@ -2408,7 +2694,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             for _ in 0..8 {
                 mw.write(four_full_shards(64))
                     .await
@@ -2438,7 +2724,7 @@ mod tests {
             // stall (10s) never fires for an 8s-per-block writer, but the 15s
             // absolute cap does once its aggregate engagement crosses the cap.
             let policy = WriteProgressPolicy::new(Duration::from_secs(10), Duration::from_secs(15));
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             for _ in 0..4 {
                 mw.write(four_full_shards(64))
                     .await
@@ -2583,7 +2869,7 @@ mod tests {
         let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
         let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
         let (_reader, written) = erasure
-            .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
+            .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
             .await
             .expect("BytesMut ingest path should encode the streaming payload");
 
@@ -2612,7 +2898,9 @@ mod tests {
         let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
 
         rustfs_io_metrics::set_put_stage_metrics_enabled(true);
-        let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
+        let result = erasure
+            .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+            .await;
         rustfs_io_metrics::set_put_stage_metrics_enabled(false);
 
         let (_reader, written) = result.expect("bytesmut streaming encode should complete");
@@ -3230,12 +3518,12 @@ mod tests {
         let (_reader, total) = match pipeline {
             EncodePipeline::Vec => {
                 erasure
-                    .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), false, None)
                     .await
             }
             EncodePipeline::BytesMut => {
                 erasure
-                    .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
                     .await
             }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,

crates/ecstore/src/erasure/coding/heal.rs

@@ -205,7 +205,7 @@ impl super::Erasure {
         // the survivors. Reconstruction correctness is still guaranteed by the
         // read-side quorum check and parity cross-checks below, which are unaffected.
         let write_quorum = 1;
-        let mut writers = MultiWriter::new(writers, write_quorum);
+        let mut writers = MultiWriter::new(writers, super::encode::ShardWritePolicy::object(write_quorum));
         let read_timeout = get_object_disk_read_timeout();
         let shard_file_size = self.shard_file_size(total_length as i64) as usize;
 

crates/ecstore/src/object_api/types.rs

@@ -478,6 +478,7 @@ struct ScannerPublicationCommitScopeInner {
 #[derive(Clone)]
 pub struct ScannerPublicationCommitScope {
     inner: Arc<ScannerPublicationCommitScopeInner>,
+    owns_completion: bool,
 }
 
 /// RAII fallback for storage paths that return before their commit closure
@@ -569,6 +570,16 @@ impl ScannerPublicationCommitScope {
                 movement_permit: Mutex::new(Some(movement_permit)),
                 lease_release_safe,
             }),
+            owns_completion: true,
+        }
+    }
+
+    /// A pool may use the shared fence, but only the enclosing operation may
+    /// report completion and authorize release of its leases.
+    pub(crate) fn pool_delete_participant(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+            owns_completion: false,
         }
     }
 
@@ -607,7 +618,9 @@ impl ScannerPublicationCommitScope {
     /// storage mutation owner. Once set, dropping a scanner waiter leaves the
     /// scope in-flight until that owner reports committed or indeterminate.
     pub fn attach_mutation_owner(&self) {
-        self.inner.owner_attached.store(true, Ordering::Release);
+        if self.owns_completion {
+            self.inner.owner_attached.store(true, Ordering::Release);
+        }
     }
 
     fn owner_attached(&self) -> bool {
@@ -654,16 +667,17 @@ impl ScannerPublicationCommitScope {
     }
 
     pub fn mark_aborted_before_commit(&self) -> bool {
-        if self
-            .inner
-            .state
-            .compare_exchange(
-                SCANNER_PUBLICATION_SCOPE_ADMITTED,
-                SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
-                Ordering::AcqRel,
-                Ordering::Acquire,
-            )
-            .is_ok()
+        if self.owns_completion
+            && self
+                .inner
+                .state
+                .compare_exchange(
+                    SCANNER_PUBLICATION_SCOPE_ADMITTED,
+                    SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
+                    Ordering::AcqRel,
+                    Ordering::Acquire,
+                )
+                .is_ok()
         {
             self.inner.lease_release_safe.store(true, Ordering::Release);
             self.inner.completed.notify_waiters();
@@ -677,6 +691,9 @@ impl ScannerPublicationCommitScope {
     }
 
     fn mark_terminal(&self, terminal: ScannerPublicationCommitState) -> bool {
+        if !self.owns_completion {
+            return false;
+        }
         self.inner
             .state
             .compare_exchange(SCANNER_PUBLICATION_SCOPE_IN_FLIGHT, terminal.as_u8(), Ordering::AcqRel, Ordering::Acquire)
@@ -709,7 +726,7 @@ impl ScannerPublicationCommitScope {
     /// Release the storage-owned movement permit only after a known-safe
     /// terminal result. Returns `false` for in-flight or indeterminate work.
     pub async fn release_movement_permit(&self) -> bool {
-        if !self.state().permits_lease_release() {
+        if !self.owns_completion || !self.state().permits_lease_release() {
             return false;
         }
         self.inner.movement_permit.lock().await.take().is_some()

crates/ecstore/src/services/rebalance/entry.rs

@@ -2250,6 +2250,195 @@ mod tests {
             .expect("large-stack rebalance test thread should complete");
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn real_rebalance_preserves_encrypted_multipart_etag_proof() {
+        run_large_stack_async_test("rebalance-encrypted-multipart-etag", || async {
+            use crate::io_support::rio::{HashReader, WriteEncryption, WritePlan};
+            use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SUFFIX_OPAQUE_ENCRYPTED_ETAG, etag_can_be_listed};
+
+            const REBALANCE_ID: &str = "rebalance-encrypted-multipart-etag";
+            for (object, versioned, protected) in [
+                ("protected-version", true, true),
+                ("protected-null", false, true),
+                ("legacy-version", true, false),
+                ("legacy-null", false, false),
+            ] {
+                // Each case needs a fresh progress counter: the fixture's small
+                // capacity can reach its rebalance goal after one encrypted object.
+                let (_temp_dirs, store, _peer) =
+                    crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
+                let bucket = "rebalance-encrypted-etag";
+                Box::pin(store.make_bucket(
+                    bucket,
+                    &MakeBucketOptions {
+                        versioning_enabled: versioned,
+                        ..Default::default()
+                    },
+                ))
+                .await
+                .expect("create the migration bucket");
+                prepare_rebalance_test_volumes(&store).await;
+                let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+                let rebalance_meta = active_rebalance_meta(REBALANCE_ID);
+                rebalance_meta
+                    .save(Arc::clone(&store.pools[0]))
+                    .await
+                    .expect("persist active rebalance");
+                *store.rebalance_meta.write().await = Some(rebalance_meta);
+
+                let source_set = store.pools[0].get_disks_by_key(object);
+                let target_set = store.pools[1].get_disks_by_key(object);
+                let write_opts = ObjectOptions {
+                    versioned,
+                    user_defined: std::collections::HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
+                    ..Default::default()
+                };
+                let upload = Box::pin(source_set.new_multipart_upload(bucket, object, &write_opts))
+                    .await
+                    .expect("create source encrypted multipart upload");
+                let plaintext = b"encrypted multipart bytes must survive pool migration";
+                let size = i64::try_from(plaintext.len()).expect("small fixture length");
+                let source = HashReader::from_stream(std::io::Cursor::new(plaintext.to_vec()), size, size, None, None, false)
+                    .expect("plaintext source reader");
+                let encrypted = WritePlan::new()
+                    .with_encryption(WriteEncryption::multipart([7; 32], [9; 12], 1))
+                    .apply(source, size)
+                    .expect("encrypt the source part through the production writer");
+                let mut part_opts = write_opts.clone();
+                if !protected {
+                    // Model a retained legacy fingerprint without producer proof.
+                    part_opts.preserve_etag = Some("0123456789abcdef0123456789abcdef".to_owned());
+                }
+                let part = Box::pin(source_set.put_object_part(
+                    bucket,
+                    object,
+                    &upload.upload_id,
+                    1,
+                    &mut PutObjReader::new(encrypted),
+                    &part_opts,
+                ))
+                .await
+                .expect("persist source encrypted part");
+                let written = Box::pin(Arc::clone(&source_set).complete_multipart_upload(
+                    bucket,
+                    object,
+                    &upload.upload_id,
+                    vec![CompletePart {
+                        part_num: part.part_num,
+                        etag: part.etag,
+                        ..Default::default()
+                    }],
+                    &write_opts,
+                ))
+                .await
+                .expect("complete source encrypted multipart upload");
+                assert_eq!(written.version_id.is_some(), versioned);
+                assert!(written.is_multipart(), "the actual migration must use its multipart path");
+                assert_eq!(etag_can_be_listed(&written.user_defined), protected);
+                let read_opts = ObjectOptions {
+                    version_id: Some(data_movement::data_movement_exact_version_id(written.version_id)),
+                    data_movement: true,
+                    raw_data_movement_read: true,
+                    ..Default::default()
+                };
+                let source_bytes = {
+                    let mut reader = Box::pin(source_set.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts))
+                        .await
+                        .expect("read source ciphertext");
+                    let mut bytes = Vec::new();
+                    reader
+                        .stream
+                        .read_to_end(&mut bytes)
+                        .await
+                        .expect("collect source ciphertext");
+                    bytes
+                };
+                assert_ne!(source_bytes.as_slice(), plaintext, "the fixture must contain encrypted bytes");
+                let entry = metacache_entry_from_source(&source_set, bucket, object).await;
+                let outcome = tokio::time::timeout(
+                    StdDuration::from_secs(60),
+                    Box::pin(Arc::clone(&store).rebalance_entry(
+                        RebalanceEntryTarget {
+                            bucket: bucket.to_owned(),
+                            pool_index: 0,
+                        },
+                        entry,
+                        Arc::clone(&source_set),
+                        Arc::new(RebalanceBucketConfigs {
+                            bucket_incarnation_id: Some(incarnation),
+                            ..Default::default()
+                        }),
+                        Arc::from(REBALANCE_ID),
+                        CancellationToken::new(),
+                    )),
+                )
+                .await
+                .expect("rebalance must finish within the test deadline")
+                .expect("rebalance source entry");
+                let target = match target_set.get_object_info(bucket, object, &read_opts).await {
+                    Ok(target) => target,
+                    Err(err) => {
+                        let target_versions = target_set
+                            .load_file_info_versions_exact(bucket, object)
+                            .await
+                            .expect("inspect target after metadata read failure")
+                            .map(|versions| {
+                                versions
+                                    .versions
+                                    .into_iter()
+                                    .map(|version| version.version_id)
+                                    .collect::<Vec<_>>()
+                            });
+                        let source_versions = source_set
+                            .load_file_info_versions_exact(bucket, object)
+                            .await
+                            .expect("inspect source after metadata read failure")
+                            .map(|versions| {
+                                versions
+                                    .versions
+                                    .into_iter()
+                                    .map(|version| version.version_id)
+                                    .collect::<Vec<_>>()
+                            });
+                        panic!(
+                            "read migrated {object}: {err}; outcome={outcome:?}, expected_version={:?}, target_versions={target_versions:?}, source_versions={source_versions:?}",
+                            written.version_id
+                        );
+                    }
+                };
+                assert_eq!(target.version_id, written.version_id, "{object}");
+                assert_eq!(target.etag, written.etag, "{object}");
+                assert_eq!(
+                    rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+                    protected.then_some("v1"),
+                    "aggregate ETag proof must follow the exact source version for {object}"
+                );
+                assert_eq!(etag_can_be_listed(&target.user_defined), protected, "{object}");
+                assert_eq!(outcome, RebalanceEntryOutcome::Completed, "{object}");
+                let mut target_reader =
+                    Box::pin(target_set.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts))
+                        .await
+                        .expect("read target ciphertext");
+                let mut target_bytes = Vec::new();
+                target_reader
+                    .stream
+                    .read_to_end(&mut target_bytes)
+                    .await
+                    .expect("collect target ciphertext");
+                assert_eq!(target_bytes, source_bytes, "rebalance preserves ciphertext bytes for {object}");
+                assert!(
+                    source_set
+                        .load_file_info_versions_exact(bucket, object)
+                        .await
+                        .expect("inspect source after migration")
+                        .is_none(),
+                    "source cleanup follows a verified target for {object}"
+                );
+            }
+        });
+    }
+
     const TRANSITIONED_REBALANCE_ID: &str = "rebalance-transitioned-source-cleanup";
 
     /// One transitioned source version on pool 0 of a two-pool store with an

crates/ecstore/src/services/tier/mod.rs

@@ -26,6 +26,7 @@ pub(crate) mod tier_probe_intent;
 pub mod warm_backend;
 pub mod warm_backend_aliyun;
 pub mod warm_backend_azure;
+pub mod warm_backend_azure_blob;
 #[cfg(feature = "gcs")]
 pub mod warm_backend_gcs;
 pub mod warm_backend_huaweicloud;

crates/ecstore/src/services/tier/tier.rs

@@ -327,6 +327,7 @@ pub const TIER_CONFIG_VERSION: u16 = 2;
 const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
 const EXTERNAL_TIER_TYPE_S3: i32 = 1;
 const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
+const EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL: &str = "rustfs-azure-blob-v1:";
 const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
 const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
 const EXTERNAL_TIER_VERSION_WASABI_V1: &str = "rustfs-wasabi-v1";
@@ -2147,6 +2148,7 @@ fn build_coordinator_tier_mutation_intent(
         return Ok(None);
     }
     Ok(Some(TierMutationIntent {
+        requires_native_azure: candidate.tiers.values().any(TierConfig::is_native_azure),
         mutation_id: uuid::Uuid::new_v4(),
         revision: 1,
         kind,
@@ -2515,11 +2517,11 @@ fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
         }),
         TierType::Azure => config.azure.as_ref().map(|value| {
             (
-                "azure",
+                if config.is_native_azure() { "azure-blob-v1" } else { "azure" },
                 value.endpoint.as_str(),
                 value.bucket.as_str(),
                 value.prefix.as_str(),
-                value.region.as_str(),
+                if config.is_native_azure() { "" } else { value.region.as_str() },
                 value.access_key.as_str(),
             )
         }),
@@ -2592,7 +2594,7 @@ fn normalized_tier_prefix<'a>(tier_type: &TierType, prefix: &'a str) -> &'a str
 }
 
 fn tier_exact_get_delete(config: &TierConfig) -> bool {
-    ProviderVersionCapabilities::for_tier_type(&config.tier_type.as_lowercase()).exact_get_delete
+    config.is_native_azure() || ProviderVersionCapabilities::for_tier_type(&config.tier_type.as_lowercase()).exact_get_delete
 }
 
 struct TierDriverGeneration {
@@ -3130,7 +3132,15 @@ fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<External
                 .ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
             out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
             out.azure = Some(ExternalTierAzure {
-                endpoint: az.endpoint.clone(),
+                // Keep the v2 Azure payload readable and rewriteable by older
+                // servers, but make their S3 constructor reject its endpoint.
+                // An unknown numeric type would discard the Azure payload on
+                // an unrelated configuration rewrite by an older server.
+                endpoint: if tier.is_native_azure() {
+                    format!("{EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL}{}", az.endpoint)
+                } else {
+                    az.endpoint.clone()
+                },
                 account_name: az.access_key.clone(),
                 account_key: az.secret_key.clone(),
                 bucket: az.bucket.clone(),
@@ -3289,6 +3299,15 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
         ..Default::default()
     };
 
+    let azure_blob = ext.version == super::tier_config::AZURE_BLOB_TIER_VERSION;
+    let native_endpoint = ext
+        .azure
+        .as_ref()
+        .is_some_and(|azure| azure.endpoint.starts_with(EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL));
+    if azure_blob != native_endpoint || azure_blob && (ext.tier_type != EXTERNAL_TIER_TYPE_AZURE || ext.tier_type_hint.is_some())
+    {
+        return Err(io::Error::other("native Azure Blob tier has inconsistent protocol discriminators"));
+    }
     let wasabi_hint = ext.tier_type_hint.as_deref() == Some("wasabi");
     let wasabi_version = ext.version == EXTERNAL_TIER_VERSION_WASABI_V1;
     if wasabi_hint && !wasabi_version {
@@ -3309,7 +3328,9 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
             cfg.name
         )));
     }
-    let tier_type = if wasabi_version {
+    let tier_type = if azure_blob {
+        TierType::Azure
+    } else if wasabi_version {
         TierType::Wasabi
     } else {
         tier_type_from_hint(ext.tier_type_hint.as_deref()).unwrap_or(match ext.tier_type {
@@ -3387,7 +3408,15 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul
                 .ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
             cfg.azure = Some(crate::services::tier::tier_config::TierAzure {
                 name: cfg.name.clone(),
-                endpoint: az.endpoint.clone(),
+                endpoint: if azure_blob {
+                    az.endpoint
+                        .strip_prefix(EXTERNAL_TIER_AZURE_BLOB_ENDPOINT_SENTINEL)
+                        .filter(|endpoint| !endpoint.is_empty())
+                        .ok_or_else(|| io::Error::other("native Azure Blob endpoint marker is missing"))?
+                        .to_string()
+                } else {
+                    az.endpoint.clone()
+                },
                 access_key: az.account_name.clone(),
                 secret_key: az.account_key.clone(),
                 bucket: az.bucket.clone(),
@@ -3558,6 +3587,9 @@ impl TierConfigMgr {
     }
 
     pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
+        if data.starts_with(&TIER_CONFIG_FORMAT.to_le_bytes()) {
+            return decode_external_tiering_config_blob(data);
+        }
         let mut cfg: TierConfigMgr = serde_json::from_slice(data)?;
         for (name, tier) in &mut cfg.tiers {
             if matches!(&tier.tier_type, TierType::Wasabi) {
@@ -3575,6 +3607,11 @@ impl TierConfigMgr {
     }
 
     pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
+        // The legacy JSON shape omits the driver protocol version. Native
+        // Azure must retain the binary discriminator even through this helper.
+        if self.tiers.values().any(TierConfig::is_native_azure) {
+            return encode_external_tiering_config_blob(self);
+        }
         let data = serde_json::to_vec(&self)?;
         let mut data = Bytes::from(data);
 
@@ -3608,6 +3645,17 @@ impl TierConfigMgr {
         deadline: Option<Instant>,
     ) -> std::result::Result<(), AdminError> {
         normalize_s3_gcs_add_tier_name(&mut tier_config)?;
+        if tier_config.tier_type == TierType::Azure {
+            tier_config.version = super::tier_config::AZURE_BLOB_TIER_VERSION.to_string();
+            if let Some(azure) = tier_config.azure.as_mut() {
+                if azure.storage_class.eq_ignore_ascii_case("STANDARD") {
+                    azure.storage_class.clear();
+                }
+                if azure.endpoint.is_empty() {
+                    azure.endpoint = format!("https://{}.blob.core.windows.net", azure.access_key);
+                }
+            }
+        }
         self.ensure_generation_is_idle(&tier_config.name)?;
         let tier_name = tier_config.name.clone();
         if tier_name != tier_name.to_uppercase() {
@@ -9536,6 +9584,98 @@ mod tests {
         assert_eq!(az.sp_auth.client_secret, "secret");
     }
 
+    #[tokio::test]
+    async fn native_azure_add_preserves_protocol_through_restart_and_requires_peer_support() {
+        let mut manager = empty_mgr();
+        let mut tier = build_azure_tier("account");
+        let name = tier.name.clone();
+        let azure = tier.azure.as_mut().expect("Azure payload should exist");
+        azure.endpoint.clear();
+        azure.storage_class = "STANDARD".to_string();
+        azure.secret_key = base64_simd::STANDARD.encode_to_string([7; 32]);
+        TIER_DRIVER_TEST_FACTORY
+            .scope(healthy_driver_factory(), manager.add(tier, true))
+            .await
+            .expect("current Console Azure payload should be admitted");
+        let created = &manager.tiers[&name];
+        assert!(created.is_native_azure());
+        assert!(tier_exact_get_delete(created));
+        let azure = created.azure.as_ref().expect("Azure payload should survive");
+        assert_eq!(azure.endpoint, "https://account.blob.core.windows.net");
+        assert!(azure.storage_class.is_empty());
+
+        let bytes = manager
+            .marshal()
+            .expect("native configuration should serialize without losing its protocol");
+        let external: ExternalTierConfigMgr =
+            rmp_serde::from_slice(&bytes[4..]).expect("native config should use the binary shape");
+        assert_eq!(external.tiers[&name].tier_type, EXTERNAL_TIER_TYPE_AZURE);
+        let stored = external.tiers[&name]
+            .azure
+            .as_ref()
+            .expect("native storage keeps the existing Azure payload");
+        assert_eq!(stored.endpoint, "rustfs-azure-blob-v1:https://account.blob.core.windows.net");
+        let legacy_backend_config = crate::services::tier::tier_config::TierAzure {
+            endpoint: stored.endpoint.clone(),
+            access_key: stored.account_name.clone(),
+            secret_key: stored.account_key.clone(),
+            bucket: stored.bucket.clone(),
+            ..Default::default()
+        };
+        assert!(
+            crate::services::tier::warm_backend_azure::WarmBackendAzure::new(&legacy_backend_config, &name)
+                .await
+                .is_err(),
+            "the unchanged legacy S3 constructor must reject the native endpoint before any request"
+        );
+        assert!(
+            external.tiers[&name].tier_type_hint.is_none(),
+            "the protocol marker must not be overridden by an extended provider hint"
+        );
+        let restarted = TierConfigMgr::unmarshal(&bytes).expect("native configuration should reload");
+        assert!(restarted.tiers[&name].is_native_azure());
+        assert_eq!(
+            tier_backend_identity(created).expect("created identity"),
+            tier_backend_identity(&restarted.tiers[&name]).expect("reloaded identity")
+        );
+        let targets = vec![TierMutationIntentTarget {
+            tier_name: name,
+            old_backend_identity: None,
+            new_backend_identity: Some(tier_backend_identity(created).expect("native destination identity")),
+        }];
+        let intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &manager, targets)
+            .expect("native intent should build")
+            .expect("native Add must use peer prepare");
+        assert!(intent.requires_native_azure);
+    }
+
+    #[test]
+    fn native_azure_protocol_is_distinct_from_legacy_and_rejects_corrupt_discriminators() {
+        let legacy = build_azure_tier("account");
+        let mut native = legacy.clone_with_credentials();
+        native.version = super::super::tier_config::AZURE_BLOB_TIER_VERSION.to_string();
+        assert_ne!(
+            tier_backend_identity(&legacy).expect("legacy identity"),
+            tier_backend_identity(&native).expect("native identity"),
+            "native Blob must never acquire a legacy S3 destination's cleanup authority"
+        );
+        let stored = to_external_tier_config(&legacy.name, &legacy).expect("legacy encoding");
+        assert_eq!(stored.tier_type, EXTERNAL_TIER_TYPE_AZURE);
+        let decoded = from_external_tier_config(legacy.name, stored).expect("legacy decoding");
+        assert!(!decoded.is_native_azure());
+        assert!(!tier_exact_get_delete(&decoded));
+        let stored = to_external_tier_config(&native.name, &native).expect("native encoding");
+        for mutate in [
+            |config: &mut ExternalTierConfig| config.version = "v1".to_string(),
+            |config: &mut ExternalTierConfig| config.tier_type = EXTERNAL_TIER_TYPE_S3,
+            |config: &mut ExternalTierConfig| config.tier_type_hint = Some("azure".to_string()),
+        ] {
+            let mut corrupt = stored.clone();
+            mutate(&mut corrupt);
+            assert!(from_external_tier_config(native.name.clone(), corrupt).is_err());
+        }
+    }
+
     #[test]
     fn test_external_blob_gcs_writer_stays_raw_for_v2_old_readers() {
         const RAW_JSON: &str = r#"{"type":"service_account","project_id":"tier-௿"}"#;
@@ -9944,6 +10084,7 @@ mod tests {
 
     fn prepared_remove_intent(tier_name: &str, mutation_id: uuid::Uuid) -> TierMutationIntent {
         TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Remove,

crates/ecstore/src/services/tier/tier_config.rs

@@ -139,6 +139,7 @@ impl TierType {
 }
 
 pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
+pub(crate) const AZURE_BLOB_TIER_VERSION: &str = "rustfs-azure-blob-v1";
 
 #[derive(Default, Serialize, Deserialize)]
 #[serde(default)]
@@ -178,6 +179,10 @@ impl Clone for TierConfig {
 }
 
 impl TierConfig {
+    pub(crate) fn is_native_azure(&self) -> bool {
+        self.tier_type == TierType::Azure && self.version == AZURE_BLOB_TIER_VERSION
+    }
+
     pub(crate) fn redacted(&self) -> Self {
         let mut redacted = Self {
             version: self.version.clone(),

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -127,6 +127,10 @@ impl TierMutationIntentTarget {
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TierMutationIntent {
+    // Old peers deny unknown intent fields before installing a prepare block.
+    // Omit this extension for existing drivers to retain their wire checksum.
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub requires_native_azure: bool,
     pub mutation_id: Uuid,
     pub revision: u64,
     pub kind: TierMutationIntentKind,
@@ -282,7 +286,8 @@ impl TierMutationIntent {
     }
 
     pub(crate) fn same_identity_as(&self, other: &Self) -> bool {
-        self.mutation_id == other.mutation_id
+        self.requires_native_azure == other.requires_native_azure
+            && self.mutation_id == other.mutation_id
             && self.kind == other.kind
             && self.old_config_etag == other.old_config_etag
             && self.candidate_digest == other.candidate_digest
@@ -825,6 +830,10 @@ fn identity_is_empty(identity: &TierDestinationId) -> bool {
     identity.iter().all(|byte| *byte == 0)
 }
 
+fn is_false(value: &bool) -> bool {
+    !value
+}
+
 fn digest_is_empty(digest: &TierMutationDigest) -> bool {
     digest.iter().all(|byte| *byte == 0)
 }
@@ -840,6 +849,7 @@ mod tests {
 
     fn prepared_intent() -> TierMutationIntent {
         TierMutationIntent {
+            requires_native_azure: false,
             mutation_id: Uuid::new_v4(),
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -867,6 +877,49 @@ mod tests {
         assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
     }
 
+    #[test]
+    fn native_azure_requirement_is_identity_bound_and_rejected_by_the_previous_wire_reader() {
+        // Preserve the previous strict wire shape as an independent reader.
+        #[derive(Serialize, Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyIntent {
+            mutation_id: Uuid,
+            revision: u64,
+            kind: TierMutationIntentKind,
+            state: TierMutationIntentState,
+            #[serde(default, skip_serializing_if = "Option::is_none")]
+            old_config_etag: Option<String>,
+            #[serde(default, skip_serializing_if = "Option::is_none")]
+            committed_config_etag: Option<String>,
+            candidate_digest: TierMutationDigest,
+            affected_targets: Vec<TierMutationIntentTarget>,
+            expires_at_unix_nanos: i64,
+        }
+        let legacy = prepared_intent();
+        let legacy_bytes = serde_json::to_vec(&legacy).expect("legacy intent should encode");
+        let old_reader: LegacyIntent =
+            serde_json::from_slice(&legacy_bytes).expect("ordinary mutations remain readable by old peers");
+        assert_eq!(
+            serde_json::to_vec(&old_reader).expect("old encoding"),
+            legacy_bytes,
+            "ordinary mutation checksums must stay unchanged"
+        );
+
+        let mut native = legacy.clone();
+        native.requires_native_azure = true;
+        let native_bytes = serde_json::to_vec(&native).expect("native intent should encode");
+        assert!(
+            serde_json::from_slice::<LegacyIntent>(&native_bytes).is_err(),
+            "an old peer must reject native Azure before prepare"
+        );
+        assert!(!legacy.same_identity_as(&native), "an abort cannot drop the native-driver requirement");
+        let encoded = native.encode().expect("native durable intent should encode");
+        assert_eq!(
+            TierMutationIntent::decode(native.mutation_id, &encoded).expect("new peer should decode"),
+            native
+        );
+    }
+
     #[tokio::test]
     async fn mutation_mutex_serializes_the_same_id() {
         let mutation_id = Uuid::new_v4();

crates/ecstore/src/services/tier/warm_backend.rs

@@ -25,6 +25,7 @@ use crate::services::tier::{
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
     warm_backend_azure::WarmBackendAzure,
+    warm_backend_azure_blob::{AzureBlobError, WarmBackendAzureBlob},
     warm_backend_huaweicloud::WarmBackendHuaweicloud,
     warm_backend_minio::WarmBackendMinIO,
     warm_backend_r2::WarmBackendR2,
@@ -309,10 +310,10 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
 
 /// Connection parameters every S3-compatible warm backend provider supplies.
 ///
-/// The Aliyun, Azure, Huaweicloud, Tencent, MinIO, R2, and RustFS backends all
+/// The Aliyun, legacy Azure, Huaweicloud, Tencent, MinIO, R2, and RustFS backends all
 /// wrap [`WarmBackendS3`] around a statically-credentialed [`TransitionClient`]
 /// built from exactly these values. `bucket_lookup` is a parameter rather than a
-/// constant because the providers split into two families: Aliyun, Azure,
+/// constant because the providers split into two families: Aliyun, legacy Azure,
 /// Huaweicloud, and Tencent pin [`BucketLookupType::BucketLookupDNS`], while
 /// MinIO, R2, and RustFS leave it at [`BucketLookupType::BucketLookupAuto`].
 pub(crate) struct S3CompatibleWarmBackendParams<'a> {
@@ -673,6 +674,9 @@ fn probe_cleanup_incomplete_error() -> AdminError {
 }
 
 fn is_remote_tier_auth_rejection(err: &std::io::Error) -> bool {
+    if AzureBlobError::from_io(err).is_some() && err.kind() == std::io::ErrorKind::PermissionDenied {
+        return true;
+    }
     let Some(response) = err.get_ref().and_then(|err| err.downcast_ref::<ErrorResponse>()) else {
         return false;
     };
@@ -706,10 +710,27 @@ async fn check_warm_backend_with_deadlines(
         Ok(Err(err)) if is_remote_tier_auth_rejection(&err) => {
             return Err(ERR_TIER_INVALID_CREDENTIALS.clone());
         }
-        Ok(Err(_)) => {
+        Ok(Err(err)) => {
+            if let Some(rejection) = AzureBlobError::from_io(&err).filter(|error| error.rejects_write()) {
+                let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                admin_err.message = rejection.to_string();
+                return Err(admin_err);
+            }
             return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
-                Ok(()) => ERR_TIER_PERM_ERR.clone(),
-                Err(_) => probe_cleanup_incomplete_error(),
+                Ok(()) => {
+                    let mut admin_err = ERR_TIER_PERM_ERR.clone();
+                    if let Some(original) = AzureBlobError::from_io(&err) {
+                        admin_err.message = original.to_string();
+                    }
+                    admin_err
+                }
+                Err(_) => {
+                    let mut admin_err = probe_cleanup_incomplete_error();
+                    if let Some(original) = AzureBlobError::from_io(&err) {
+                        admin_err.message.push_str(&format!(": {original}"));
+                    }
+                    admin_err
+                }
             });
         }
         Err(_) => {
@@ -951,24 +972,23 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
             }
         }
         TierType::Azure => {
-            if let Some(azure_config) = tier.azure.as_ref() {
-                let dd = WarmBackendAzure::new(azure_config, &tier.name).await;
-                if let Err(err) = dd {
-                    warn!("{}", err);
-                    return Err(AdminError {
-                        code: "XRustFSAdminTierInvalidConfig".to_string(),
-                        message: format!("Unable to setup remote tier, check tier configuration: {err}"),
-                        status_code: StatusCode::BAD_REQUEST,
-                    });
-                }
-                d = Some(Box::new(dd.expect("Failed to create Azure backend")));
+            let azure = tier.azure.as_ref().ok_or_else(|| AdminError {
+                code: "XRustFSAdminTierInvalidConfig".to_string(),
+                message: "Azure tier configuration not found".to_string(),
+                status_code: StatusCode::BAD_REQUEST,
+            })?;
+            let driver = if tier.is_native_azure() {
+                WarmBackendAzureBlob::new(azure).map(|driver| -> WarmBackendImpl { Box::new(driver) })
             } else {
-                return Err(AdminError {
-                    code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: "Azure tier configuration not found".to_string(),
-                    status_code: StatusCode::BAD_REQUEST,
-                });
-            }
+                WarmBackendAzure::new(azure, &tier.name)
+                    .await
+                    .map(|driver| -> WarmBackendImpl { Box::new(driver) })
+            };
+            d = Some(driver.map_err(|err| AdminError {
+                code: "XRustFSAdminTierInvalidConfig".to_string(),
+                message: format!("Unable to setup remote tier, check tier configuration: {err}"),
+                status_code: StatusCode::BAD_REQUEST,
+            })?);
         }
         #[cfg(not(feature = "gcs"))]
         TierType::GCS => {
@@ -1042,6 +1062,13 @@ pub(crate) async fn new_transition_candidate_reconciler(
     tier: &TierConfig,
 ) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
     let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
+        TierType::Azure if tier.is_native_azure() => Box::new(
+            WarmBackendAzureBlob::new(tier.azure.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?).map_err(|err| {
+                let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                admin_err.message = err.to_string();
+                admin_err
+            })?,
+        ),
         TierType::S3 => Box::new(
             WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
                 .await

crates/ecstore/src/services/tier/warm_backend_azure.rs

@@ -88,7 +88,7 @@ impl WarmBackend for WarmBackendAzure {
     }
 
     async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
-        // Azure currently uses the shared S3/SigV4 transport, but its normal
+        // Legacy Azure uses the shared S3/SigV4 transport, but its normal
         // object path cannot persist exact remote versions across mixed
         // RustFS releases. The mutation probe may still detect and precisely
         // remove a versioned test object before rejecting that configuration.

crates/ecstore/src/services/tier/warm_backend_azure_blob.rs

@@ -0,0 +1,881 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Native Azure Blob tier transport, including Azure Stack Hub 2301 and later.
+//!
+//! Writes use one conditional commit and never replay a failed HTTP request.
+//! Larger bodies are staged in bounded blocks with a unique upload namespace;
+//! an interrupted staging operation leaves only Azure's expiring uncommitted
+//! blocks. Lifecycle uploads use the existing transaction journal after a lost
+//! commit acknowledgement; validation probes retain in-process compensation.
+//! Versioned GET/DELETE address the persisted version. Current-root cleanup
+//! may demote that matching version under a strong ETag before exact deletion.
+//! Unversioned DELETE also requires a strong ETag to refuse replacement.
+
+use super::tier_config::TierAzure;
+use super::warm_backend::{
+    TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
+    build_transition_put_options, transition_client_timeouts_from_env,
+};
+use super::warm_backend_s3::transition_candidate_metadata_matches;
+use crate::bucket::lifecycle::transition_transaction::canonical_transition_remote_object;
+use bytes::Bytes;
+use futures_util::stream;
+use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode};
+use quick_xml::{Reader, events::Event};
+use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
+use rustfs_signer::azure_shared_key_signature;
+use rustfs_utils::crypto::hex;
+use rustfs_utils::egress::validate_outbound_url;
+use rustfs_utils::hash::FastMd5;
+use rustfs_utils::http::metadata_compat::{
+    MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, internal_key_rustfs,
+};
+use std::collections::HashMap;
+use std::io::{self, Cursor, ErrorKind};
+use std::time::Duration;
+use time::OffsetDateTime;
+use tokio::io::{AsyncRead, AsyncReadExt};
+use url::Url;
+
+const API_VERSION: &str = "2020-10-02";
+const MIN_BLOCK_BYTES: u64 = 16 * 1024 * 1024;
+const MAX_BLOCK_BYTES: u64 = 100 * 1024 * 1024;
+const MAX_BLOCKS: u64 = 50_000;
+const MAX_CONTROL_BYTES: usize = 64 * 1024;
+const MAX_METADATA_BYTES: usize = 8 * 1024;
+const METADATA_PREFIX: &str = "x-ms-meta-rustfs_";
+const HTTP_DATE: &[time::format_description::BorrowedFormatItem<'static>] =
+    time::macros::format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
+
+/// A bounded diagnostic that never contains credentials, URLs or response bodies.
+#[derive(Debug, thiserror::Error)]
+#[error("Azure Blob {operation} failed (HTTP {status}, code {code}, request ID {request_id})")]
+pub(crate) struct AzureBlobError {
+    operation: &'static str,
+    status: StatusCode,
+    code: String,
+    request_id: String,
+}
+
+impl AzureBlobError {
+    pub(super) fn from_io(error: &io::Error) -> Option<&Self> {
+        error.get_ref()?.downcast_ref()
+    }
+
+    pub(super) fn rejects_write(&self) -> bool {
+        matches!(
+            self.status,
+            StatusCode::BAD_REQUEST | StatusCode::NOT_FOUND | StatusCode::CONFLICT | StatusCode::PRECONDITION_FAILED
+        )
+    }
+}
+
+pub struct WarmBackendAzureBlob {
+    client: reqwest::Client,
+    endpoint: Url,
+    account: String,
+    key: Vec<u8>,
+    container: String,
+    prefix: String,
+    request_timeout: Duration,
+}
+
+impl WarmBackendAzureBlob {
+    /// Build a native Blob backend without performing remote I/O.
+    ///
+    /// # Errors
+    ///
+    /// Rejects unsupported credentials or storage classes, an invalid account,
+    /// container, prefix, endpoint or account key, and TLS/client setup failures.
+    pub fn new(conf: &TierAzure) -> io::Result<Self> {
+        validate_config(conf)?;
+        let endpoint = if conf.endpoint.is_empty() {
+            format!("https://{}.blob.core.windows.net", conf.access_key)
+        } else {
+            conf.endpoint.clone()
+        };
+        let endpoint = Url::parse(&endpoint).map_err(|_| invalid_input("Azure Blob endpoint is not a valid URL"))?;
+        if !matches!(endpoint.scheme(), "https" | "http")
+            || endpoint.host_str().is_none()
+            || !endpoint.username().is_empty()
+            || endpoint.password().is_some()
+            || !matches!(endpoint.path(), "" | "/")
+            || endpoint.query().is_some()
+            || endpoint.fragment().is_some()
+        {
+            return Err(invalid_input(
+                "Azure Blob endpoint must be an HTTP(S) origin without credentials, path or query",
+            ));
+        }
+        validate_outbound_url(&endpoint).map_err(|_| invalid_input("Azure Blob endpoint is not allowed"))?;
+        let key = decode_key(&conf.secret_key)?;
+        rustfs_tls_runtime::ensure_crypto_provider().map_err(io::Error::other)?;
+        let timeouts = transition_client_timeouts_from_env();
+        let client = reqwest::Client::builder()
+            .connect_timeout(timeouts.connect_timeout)
+            .read_timeout(timeouts.response_body_idle_timeout)
+            .redirect(reqwest::redirect::Policy::none())
+            .retry(reqwest::retry::never())
+            .no_gzip()
+            .no_brotli()
+            .no_zstd()
+            .no_deflate()
+            .build()
+            .map_err(transport_error)?;
+        Ok(Self {
+            client,
+            endpoint,
+            account: conf.access_key.clone(),
+            key,
+            container: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix('/').unwrap_or(&conf.prefix).to_string(),
+            request_timeout: timeouts.request_timeout,
+        })
+    }
+
+    fn object_url(&self, object: &str, version: &str) -> io::Result<Url> {
+        validate_version(version)?;
+        let name = if self.prefix.is_empty() {
+            object.to_string()
+        } else {
+            format!("{}/{object}", self.prefix)
+        };
+        validate_object_name(&name)?;
+        let mut url = self.endpoint.clone();
+        {
+            let mut segments = url
+                .path_segments_mut()
+                .map_err(|_| invalid_input("Azure Blob endpoint has no path"))?;
+            segments.clear().push(&self.container);
+            for segment in name.split('/') {
+                segments.push(segment);
+            }
+        }
+        if !version.is_empty() {
+            url.query_pairs_mut().append_pair("versionid", version);
+        }
+        Ok(url)
+    }
+
+    fn container_url(&self) -> io::Result<Url> {
+        let mut url = self.endpoint.clone();
+        url.path_segments_mut()
+            .map_err(|_| invalid_input("Azure Blob endpoint has no path"))?
+            .clear()
+            .push(&self.container);
+        Ok(url)
+    }
+
+    async fn send(&self, method: Method, url: Url, mut headers: HeaderMap, body: Option<Bytes>) -> io::Result<reqwest::Response> {
+        if let Some(body) = &body {
+            insert_header(&mut headers, "content-length", &body.len().to_string())?;
+            let bytes = body.clone();
+            let digest = tokio::task::spawn_blocking(move || FastMd5::digest(&bytes))
+                .await
+                .map_err(|_| io::Error::other("Azure Blob content checksum task failed"))?;
+            insert_header(&mut headers, "content-md5", &base64_simd::STANDARD.encode_to_string(digest))?;
+        }
+        headers.insert("x-ms-version", HeaderValue::from_static(API_VERSION));
+        headers.insert("accept-encoding", HeaderValue::from_static("identity"));
+        let date = OffsetDateTime::now_utc()
+            .format(HTTP_DATE)
+            .map_err(|_| io::Error::other("Azure Blob request date is not representable"))?;
+        insert_header(&mut headers, "x-ms-date", &date)?;
+        let signature = azure_shared_key_signature(&self.key, &self.account, method.as_str(), &url, &headers)
+            .map_err(|err| io::Error::new(ErrorKind::InvalidInput, err))?;
+        insert_header(&mut headers, "authorization", &format!("SharedKey {}:{signature}", self.account))?;
+        let mut request = reqwest::Request::new(method, url);
+        *request.headers_mut() = headers;
+        *request.body_mut() = body.map(reqwest::Body::from);
+        tokio::time::timeout(self.request_timeout, self.client.execute(request))
+            .await
+            .map_err(|_| io::Error::new(ErrorKind::TimedOut, "Azure Blob request timed out"))?
+            .map_err(transport_error)
+    }
+
+    async fn expect_status(
+        &self,
+        response: reqwest::Response,
+        expected: StatusCode,
+        operation: &'static str,
+    ) -> io::Result<reqwest::Response> {
+        if response.status() == expected {
+            return Ok(response);
+        }
+        Err(service_error(response, operation, self.request_timeout).await)
+    }
+
+    async fn head(&self, object: &str, version: &str) -> io::Result<Option<HeaderMap>> {
+        let response = self
+            .send(Method::HEAD, self.object_url(object, version)?, HeaderMap::new(), None)
+            .await?;
+        if response.status() == StatusCode::OK {
+            if !version.is_empty() {
+                validate_response_version(response.headers(), version)?;
+            }
+            return Ok(Some(response.headers().clone()));
+        }
+        let err = service_error(response, "HEAD", self.request_timeout).await;
+        if err.kind() == ErrorKind::NotFound {
+            return Ok(None);
+        }
+        Err(err)
+    }
+
+    async fn delete(&self, object: &str, version: &str, headers: HeaderMap) -> io::Result<()> {
+        let response = self
+            .send(Method::DELETE, self.object_url(object, version)?, headers, None)
+            .await?;
+        if response.status() == StatusCode::ACCEPTED {
+            return Ok(());
+        }
+        let error = service_error(response, "DELETE", self.request_timeout).await;
+        if error.kind() == ErrorKind::NotFound {
+            return Ok(());
+        }
+        Err(error)
+    }
+
+    async fn write(
+        &self,
+        object: &str,
+        reader: ReaderImpl,
+        length: i64,
+        metadata: HashMap<String, String>,
+    ) -> io::Result<String> {
+        let size = u64::try_from(length).map_err(|_| invalid_input("Azure Blob upload requires a known, non-negative length"))?;
+        let block_size = block_size(size)?;
+        let url = self.object_url(object, "")?;
+        let mut headers = metadata_headers(metadata)?;
+        headers.insert("if-none-match", HeaderValue::from_static("*"));
+        let mut reader: Box<dyn AsyncRead + Send + Sync + Unpin> = match reader {
+            ReaderImpl::Body(bytes) => Box::new(Cursor::new(bytes)),
+            ReaderImpl::ObjectBody(reader) => Box::new(reader),
+        };
+        if size <= block_size {
+            let body = read_part(&mut reader, size).await?;
+            ensure_eof(&mut reader).await?;
+            headers.insert("x-ms-blob-type", HeaderValue::from_static("BlockBlob"));
+            let response = self.send(Method::PUT, url, headers, Some(body)).await?;
+            let response = self.expect_status(response, StatusCode::CREATED, "PUT").await?;
+            return response_version(response.headers());
+        }
+
+        let upload = uuid::Uuid::new_v4().simple().to_string();
+        let mut blocks = Vec::new();
+        let mut remaining = size;
+        while remaining > 0 {
+            let count = remaining.min(block_size);
+            let body = read_part(&mut reader, count).await?;
+            let block_id = base64_simd::STANDARD.encode_to_string(format!("{upload}-{:05}", blocks.len()));
+            let mut block_url = url.clone();
+            block_url
+                .query_pairs_mut()
+                .append_pair("comp", "block")
+                .append_pair("blockid", &block_id);
+            let response = self.send(Method::PUT, block_url, HeaderMap::new(), Some(body)).await?;
+            self.expect_status(response, StatusCode::CREATED, "PUT block").await?;
+            blocks.push(block_id);
+            remaining -= count;
+        }
+        ensure_eof(&mut reader).await?;
+        let mut xml = String::from("<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>");
+        for block in blocks {
+            // IDs are generated base64, so they contain no XML delimiters.
+            xml.push_str("<Uncommitted>");
+            xml.push_str(&block);
+            xml.push_str("</Uncommitted>");
+        }
+        xml.push_str("</BlockList>");
+        let mut commit_url = url;
+        commit_url.query_pairs_mut().append_pair("comp", "blocklist");
+        headers.insert("content-type", HeaderValue::from_static("application/xml"));
+        let response = self.send(Method::PUT, commit_url, headers, Some(Bytes::from(xml))).await?;
+        let response = self.expect_status(response, StatusCode::CREATED, "PUT block list").await?;
+        response_version(response.headers())
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendAzureBlob {
+    fn validate_remote_version_id(&self, version: &str) -> io::Result<()> {
+        validate_version(version)
+    }
+
+    async fn put(&self, object: &str, reader: ReaderImpl, length: i64) -> io::Result<String> {
+        self.write(object, reader, length, HashMap::new()).await
+    }
+
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        reader: ReaderImpl,
+        length: i64,
+        metadata: HashMap<String, String>,
+    ) -> io::Result<String> {
+        self.write(object, reader, length, metadata).await
+    }
+
+    async fn get(&self, object: &str, version: &str, opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
+        let range = requested_range(&opts)?;
+        let mut headers = HeaderMap::new();
+        if let Some((start, end)) = range {
+            let value = end.map_or_else(|| format!("bytes={start}-"), |end| format!("bytes={start}-{end}"));
+            insert_header(&mut headers, "range", &value)?;
+        }
+        let response = self
+            .send(Method::GET, self.object_url(object, version)?, headers, None)
+            .await?;
+        let expected = if range.is_some() {
+            StatusCode::PARTIAL_CONTENT
+        } else {
+            StatusCode::OK
+        };
+        let response = self.expect_status(response, expected, "GET").await?;
+        validate_response_version(response.headers(), version)?;
+        let length = response
+            .headers()
+            .get("content-length")
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<u64>().ok())
+            .ok_or_else(|| invalid_data("Azure Blob GET omitted a valid Content-Length"))?;
+        if let Some(range) = range {
+            validate_content_range(response.headers(), range, length)?;
+        }
+        let body = stream::try_unfold((response, length), |(mut response, remaining)| async move {
+            loop {
+                match response.chunk().await.map_err(transport_error)? {
+                    Some(chunk) if chunk.is_empty() => continue,
+                    Some(chunk) => {
+                        let count =
+                            u64::try_from(chunk.len()).map_err(|_| invalid_data("Azure Blob response length overflow"))?;
+                        let remaining = remaining
+                            .checked_sub(count)
+                            .ok_or_else(|| invalid_data("Azure Blob returned more bytes than declared"))?;
+                        return Ok::<_, io::Error>(Some((chunk, (response, remaining))));
+                    }
+                    None if remaining == 0 => return Ok(None),
+                    None => {
+                        return Err(io::Error::new(
+                            ErrorKind::UnexpectedEof,
+                            "Azure Blob response ended before its declared length",
+                        ));
+                    }
+                }
+            }
+        });
+        Ok(ReadCloser::new(tokio_util::io::StreamReader::new(Box::pin(body))))
+    }
+
+    async fn remove(&self, object: &str, version: &str) -> io::Result<()> {
+        let mut headers = HeaderMap::new();
+        if version.is_empty() {
+            let Some(current) = self.head(object, "").await? else {
+                return Ok(());
+            };
+            if !response_version(&current)?.is_empty() {
+                return Err(invalid_data("Azure Blob became versioned; refusing an unversioned delete"));
+            }
+            headers = delete_preconditions(&current)?;
+        }
+        let Err(error) = self.delete(object, version, headers).await else {
+            return Ok(());
+        };
+        if version.is_empty()
+            || !AzureBlobError::from_io(&error).is_some_and(|azure| {
+                matches!(azure.status, StatusCode::FORBIDDEN | StatusCode::CONFLICT)
+                    && azure.code == "OperationNotAllowedOnRootBlob"
+            })
+        {
+            return Err(error);
+        }
+
+        // Azure may require the current version to become a previous version
+        // before exact deletion. Only its matching generation can be demoted.
+        if let Some(current) = self.head(object, "").await? {
+            validate_response_version(&current, version)?;
+            self.delete(object, "", delete_preconditions(&current)?).await?;
+        }
+        // A successful base delete alone does not complete version cleanup.
+        // An error or lost acknowledgement remains retryable by the caller.
+        self.delete(object, version, HeaderMap::new()).await
+    }
+
+    async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
+        Ok(match self.head(object, "").await? {
+            None => TransitionCandidateProbe::Missing,
+            Some(headers) => classify_head(&headers)?,
+        })
+    }
+
+    async fn probe_transition_version(&self, object: &str, version: &str) -> io::Result<TransitionCandidateProbe> {
+        if version.is_empty() {
+            return Err(invalid_input("Azure Blob exact probe requires a version ID"));
+        }
+        Ok(match self.head(object, version).await? {
+            None => TransitionCandidateProbe::Missing,
+            Some(_) => TransitionCandidateProbe::VersionedPresent(version.to_string()),
+        })
+    }
+
+    async fn in_use(&self) -> io::Result<bool> {
+        let mut url = self.container_url()?;
+        let prefix = if self.prefix.is_empty() {
+            String::new()
+        } else {
+            format!("{}/", self.prefix)
+        };
+        url.query_pairs_mut()
+            .append_pair("restype", "container")
+            .append_pair("comp", "list")
+            .append_pair("prefix", &prefix)
+            .append_pair("maxresults", "1");
+        let response = self.send(Method::GET, url, HeaderMap::new(), None).await?;
+        let response = self.expect_status(response, StatusCode::OK, "LIST").await?;
+        list_has_objects(&read_control_body(response, self.request_timeout).await?)
+    }
+}
+
+#[async_trait::async_trait]
+impl TransitionCandidateReconciler for WarmBackendAzureBlob {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> io::Result<TransitionCandidateProbe> {
+        let Some(headers) = self.head(object, "").await? else {
+            return Ok(TransitionCandidateProbe::Missing);
+        };
+        let mut metadata = HashMap::new();
+        for suffix in [SUFFIX_TRANSITION_TRANSACTION_ID, SUFFIX_TRANSITION_TIER_DESTINATION_ID] {
+            for key in [internal_key_rustfs(suffix), format!("{MINIO_INTERNAL_PREFIX}{suffix}")] {
+                if let Some(value) = headers.get(metadata_header_name(&key)) {
+                    let decoded = base64_simd::STANDARD
+                        .decode_to_vec(value.as_bytes())
+                        .map_err(|_| invalid_data("Azure Blob candidate metadata is not valid base64"))?;
+                    let value =
+                        String::from_utf8(decoded).map_err(|_| invalid_data("Azure Blob candidate metadata is not UTF-8"))?;
+                    metadata.insert(key, value);
+                }
+            }
+        }
+        if !transition_candidate_metadata_matches(&metadata, identity)? {
+            return Ok(TransitionCandidateProbe::Ambiguous);
+        }
+        classify_head(&headers)
+    }
+}
+
+fn validate_config(conf: &TierAzure) -> io::Result<()> {
+    if !(3..=24).contains(&conf.access_key.len())
+        || !conf
+            .access_key
+            .bytes()
+            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
+    {
+        return Err(invalid_input("Azure Blob account name must contain 3 to 24 lowercase letters or digits"));
+    }
+    let container = conf.bucket.as_bytes();
+    if !(3..=63).contains(&container.len())
+        || !container
+            .iter()
+            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-')
+        || container.first() == Some(&b'-')
+        || container.last() == Some(&b'-')
+        || conf.bucket.contains("--")
+    {
+        return Err(invalid_input("Azure Blob container name is invalid"));
+    }
+    if !(conf.storage_class.is_empty() || conf.storage_class == "STANDARD")
+        || !conf.sp_auth.tenant_id.is_empty()
+        || !conf.sp_auth.client_id.is_empty()
+        || !conf.sp_auth.client_secret.is_empty()
+    {
+        return Err(invalid_input(
+            "Azure Blob tiers currently require an account key and the default storage class",
+        ));
+    }
+    let prefix = conf.prefix.strip_suffix('/').unwrap_or(&conf.prefix);
+    if !prefix.is_empty() {
+        validate_object_name(prefix)?;
+        // Canonical suffix length is independent of the bucket and UUID values.
+        let object = canonical_transition_remote_object(
+            uuid::Uuid::from_u128(1),
+            &conf.bucket,
+            uuid::Uuid::from_u128(2),
+            uuid::Uuid::from_u128(3),
+        )
+        .map_err(io::Error::other)?;
+        validate_object_name(&format!("{prefix}/{object}"))?;
+    }
+    Ok(())
+}
+
+fn decode_key(key: &str) -> io::Result<Vec<u8>> {
+    let key = base64_simd::STANDARD
+        .decode_to_vec(key.trim().as_bytes())
+        .map_err(|_| invalid_input("Azure Blob account key must be base64"))?;
+    if key.is_empty() {
+        return Err(invalid_input("Azure Blob account key must not be empty"));
+    }
+    Ok(key)
+}
+
+fn validate_object_name(name: &str) -> io::Result<()> {
+    if name.is_empty()
+        || name.len() > 1760
+        || name.chars().count() > 880
+        || name.split('/').any(|segment| matches!(segment, "." | ".."))
+    {
+        return Err(invalid_input(
+            "Azure Blob tier name exceeds Stack Hub limits or contains a standalone dot segment",
+        ));
+    }
+    Ok(())
+}
+
+fn validate_version(version: &str) -> io::Result<()> {
+    if version.len() > 1024 || version.bytes().any(|byte| !byte.is_ascii_graphic()) {
+        return Err(invalid_input("Azure Blob version ID is invalid"));
+    }
+    Ok(())
+}
+
+fn response_version(headers: &HeaderMap) -> io::Result<String> {
+    let Some(version) = headers.get("x-ms-version-id") else {
+        return Ok(String::new());
+    };
+    let version = version
+        .to_str()
+        .map_err(|_| invalid_data("Azure Blob version ID is not ASCII"))?;
+    validate_version(version)?;
+    Ok(version.to_string())
+}
+
+fn validate_response_version(headers: &HeaderMap, requested: &str) -> io::Result<()> {
+    if response_version(headers)? != requested {
+        return Err(invalid_data("Azure Blob did not return the requested exact version"));
+    }
+    Ok(())
+}
+
+fn delete_preconditions(current: &HeaderMap) -> io::Result<HeaderMap> {
+    let etag = current
+        .get("etag")
+        .ok_or_else(|| invalid_data("Azure Blob HEAD omitted the delete precondition"))?;
+    let bytes = etag.as_bytes();
+    // A wildcard, weak tag or list cannot identify a single matching generation.
+    if bytes.len() < 3
+        || bytes.first() != Some(&b'"')
+        || bytes.last() != Some(&b'"')
+        || !bytes[1..bytes.len() - 1]
+            .iter()
+            .all(|byte| *byte == b'!' || (b'#'..=b'~').contains(byte))
+    {
+        return Err(invalid_data("Azure Blob HEAD omitted a single strong ETag"));
+    }
+    let mut headers = HeaderMap::new();
+    headers.insert("if-match", etag.clone());
+    Ok(headers)
+}
+
+fn classify_head(headers: &HeaderMap) -> io::Result<TransitionCandidateProbe> {
+    let version = response_version(headers)?;
+    Ok(if version.is_empty() {
+        TransitionCandidateProbe::UnversionedPresent
+    } else {
+        TransitionCandidateProbe::VersionedPresent(version)
+    })
+}
+
+fn block_size(size: u64) -> io::Result<u64> {
+    if size > MAX_BLOCK_BYTES * MAX_BLOCKS {
+        return Err(invalid_input("Azure Blob exceeds the Stack Hub block count or object size limit"));
+    }
+    Ok(size.div_ceil(MAX_BLOCKS).max(MIN_BLOCK_BYTES))
+}
+
+async fn read_part(reader: &mut (dyn AsyncRead + Send + Sync + Unpin), length: u64) -> io::Result<Bytes> {
+    let length = usize::try_from(length).map_err(|_| invalid_input("Azure Blob block length is not representable"))?;
+    let mut bytes = vec![0; length];
+    reader.read_exact(&mut bytes).await?;
+    Ok(Bytes::from(bytes))
+}
+
+async fn ensure_eof(reader: &mut (dyn AsyncRead + Send + Sync + Unpin)) -> io::Result<()> {
+    if reader.read(&mut [0_u8; 1]).await? != 0 {
+        return Err(invalid_input("Azure Blob upload body exceeds its declared length"));
+    }
+    Ok(())
+}
+
+fn metadata_header_name(key: &str) -> String {
+    format!("{METADATA_PREFIX}{}", hex(key.as_bytes()))
+}
+
+fn metadata_headers(metadata: HashMap<String, String>) -> io::Result<HeaderMap> {
+    let opts = build_transition_put_options(String::new(), metadata);
+    let mut headers = HeaderMap::new();
+    for (name, value) in [
+        ("x-ms-blob-content-type", opts.content_type),
+        ("x-ms-blob-content-encoding", opts.content_encoding),
+        ("x-ms-blob-content-language", opts.content_language),
+        ("x-ms-blob-content-disposition", opts.content_disposition),
+        ("x-ms-blob-cache-control", opts.cache_control),
+    ] {
+        if !value.is_empty() {
+            insert_header(&mut headers, name, &value)?;
+        }
+    }
+    let mut size = 0_usize;
+    for (key, value) in opts.user_metadata {
+        let key = key.to_ascii_lowercase();
+        let key = key.strip_prefix("x-amz-meta-").unwrap_or(&key);
+        let name = metadata_header_name(key);
+        let value = base64_simd::STANDARD.encode_to_string(value.as_bytes());
+        size = size.saturating_add(name.len()).saturating_add(value.len());
+        if size > MAX_METADATA_BYTES {
+            return Err(invalid_input("Azure Blob encoded metadata exceeds 8 KiB"));
+        }
+        if headers.contains_key(&name) {
+            return Err(invalid_input("Azure Blob metadata contains conflicting names"));
+        }
+        insert_header(&mut headers, &name, &value)?;
+    }
+    Ok(headers)
+}
+
+fn requested_range(opts: &WarmBackendGetOpts) -> io::Result<Option<(u64, Option<u64>)>> {
+    let start = u64::try_from(opts.start_offset).map_err(|_| invalid_input("Azure Blob range starts before zero"))?;
+    let length = u64::try_from(opts.length).map_err(|_| invalid_input("Azure Blob range length is negative"))?;
+    if length > 0 {
+        let end = start
+            .checked_add(length - 1)
+            .ok_or_else(|| invalid_input("Azure Blob range overflows"))?;
+        return Ok(Some((start, Some(end))));
+    }
+    Ok((start > 0).then_some((start, None)))
+}
+
+fn validate_content_range(headers: &HeaderMap, requested: (u64, Option<u64>), length: u64) -> io::Result<()> {
+    let value = headers
+        .get("content-range")
+        .and_then(|value| value.to_str().ok())
+        .unwrap_or_default();
+    let range = value
+        .strip_prefix("bytes ")
+        .and_then(|value| value.split_once('/'))
+        .and_then(|(range, total)| {
+            let (start, end) = range.split_once('-')?;
+            Some((start.parse::<u64>().ok()?, end.parse::<u64>().ok()?, total.parse::<u64>().ok()?))
+        });
+    let Some((start, end, total)) = range else {
+        return Err(invalid_data("Azure Blob returned an invalid Content-Range"));
+    };
+    if start != requested.0
+        || end < start
+        || end >= total
+        || end != requested.1.unwrap_or(total - 1).min(total - 1)
+        || end.checked_sub(start).and_then(|size| size.checked_add(1)) != Some(length)
+    {
+        return Err(invalid_data("Azure Blob returned a different range from the one requested"));
+    }
+    Ok(())
+}
+
+async fn read_control_body(mut response: reqwest::Response, timeout: Duration) -> io::Result<Vec<u8>> {
+    tokio::time::timeout(timeout, async move {
+        let mut body = Vec::new();
+        while let Some(chunk) = response.chunk().await.map_err(transport_error)? {
+            if body.len().saturating_add(chunk.len()) > MAX_CONTROL_BYTES {
+                return Err(invalid_data("Azure Blob control response exceeds 64 KiB"));
+            }
+            body.extend_from_slice(&chunk);
+        }
+        Ok(body)
+    })
+    .await
+    .map_err(|_| io::Error::new(ErrorKind::TimedOut, "Azure Blob control response timed out"))?
+}
+
+async fn service_error(response: reqwest::Response, operation: &'static str, timeout: Duration) -> io::Error {
+    let status = response.status();
+    let request_id = diagnostic_token(
+        response
+            .headers()
+            .get("x-ms-request-id")
+            .and_then(|value| value.to_str().ok()),
+    );
+    let mut code = diagnostic_token(
+        response
+            .headers()
+            .get("x-ms-error-code")
+            .and_then(|value| value.to_str().ok()),
+    );
+    if code == "unknown"
+        && let Ok(body) = read_control_body(response, timeout).await
+        && let Some(xml_code) = xml_error_code(&body)
+    {
+        code = diagnostic_token(Some(&xml_code));
+    }
+    let kind = if status == StatusCode::NOT_FOUND && code == "BlobNotFound" {
+        ErrorKind::NotFound
+    } else if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
+        ErrorKind::PermissionDenied
+    } else if status == StatusCode::PRECONDITION_FAILED {
+        ErrorKind::AlreadyExists
+    } else {
+        ErrorKind::Other
+    };
+    io::Error::new(
+        kind,
+        AzureBlobError {
+            operation,
+            status,
+            code,
+            request_id,
+        },
+    )
+}
+
+fn diagnostic_token(value: Option<&str>) -> String {
+    value
+        .filter(|value| {
+            !value.is_empty() && value.len() <= 128 && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
+        })
+        .unwrap_or("unknown")
+        .to_string()
+}
+
+fn xml_error_code(body: &[u8]) -> Option<String> {
+    let mut reader = Reader::from_reader(body);
+    let mut depth = 0_u32;
+    let mut root = false;
+    let mut code = None;
+    loop {
+        match reader.read_event().ok()? {
+            Event::Start(start) => {
+                if depth == 0 {
+                    if root || start.local_name().as_ref() != "Error" {
+                        return None;
+                    }
+                    root = true;
+                }
+                if depth == 1 && start.local_name().as_ref() == "Code" {
+                    if code.is_some() {
+                        return None;
+                    }
+                    let text = reader.read_text(start.name()).ok()?;
+                    code = Some(quick_xml::escape::unescape(text.as_ref()).ok()?.into_owned());
+                    continue;
+                }
+                depth += 1;
+                if depth > 16 {
+                    return None;
+                }
+            }
+            Event::End(_) => depth = depth.checked_sub(1)?,
+            Event::Text(text) if depth == 0 && !text.as_ref().trim().is_empty() => return None,
+            Event::CData(_) if depth == 0 => return None,
+            Event::Eof => return (root && depth == 0).then_some(code).flatten(),
+            Event::DocType(_) => return None,
+            _ => {}
+        }
+    }
+}
+
+fn list_has_objects(body: &[u8]) -> io::Result<bool> {
+    let mut reader = Reader::from_reader(body);
+    let mut depth = 0_u32;
+    let mut root = false;
+    let mut blobs = false;
+    let mut found = false;
+    loop {
+        let event = reader
+            .read_event()
+            .map_err(|_| invalid_data("Azure Blob listing is not valid XML"))?;
+        let empty = matches!(event, Event::Empty(_));
+        match event {
+            Event::Start(start) | Event::Empty(start) => {
+                if depth == 0 && (root || start.local_name().as_ref() != "EnumerationResults") {
+                    return Err(invalid_data("Azure Blob listing has an unexpected root"));
+                }
+                root |= start.local_name().as_ref() == "EnumerationResults";
+                if start.local_name().as_ref() == "Blobs" {
+                    if blobs || depth != 1 {
+                        return Err(invalid_data("Azure Blob listing has an invalid Blobs element"));
+                    }
+                    blobs = true;
+                }
+                found |= matches!(start.local_name().as_ref(), "Blob" | "BlobPrefix");
+                if !empty && start.local_name().as_ref() == "NextMarker" {
+                    let marker = reader
+                        .read_text(start.name())
+                        .map_err(|_| invalid_data("Azure Blob listing marker is invalid"))?;
+                    found |= !marker.is_empty();
+                    continue;
+                }
+                if !empty {
+                    depth += 1;
+                    if depth > 16 {
+                        return Err(invalid_data("Azure Blob listing exceeds the XML depth limit"));
+                    }
+                }
+            }
+            Event::End(_) => {
+                depth = depth
+                    .checked_sub(1)
+                    .ok_or_else(|| invalid_data("Azure Blob listing has an invalid closing tag"))?;
+            }
+            Event::DocType(_) => return Err(invalid_data("Azure Blob listing must not contain a document type")),
+            Event::Text(text) if depth == 0 && !text.as_ref().trim().is_empty() => {
+                return Err(invalid_data("Azure Blob listing contains text outside its root"));
+            }
+            Event::CData(_) if depth == 0 => return Err(invalid_data("Azure Blob listing contains text outside its root")),
+            Event::Eof if root && blobs && depth == 0 => return Ok(found),
+            Event::Eof => return Err(invalid_data("Azure Blob listing is incomplete")),
+            _ => {}
+        }
+    }
+}
+
+fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> io::Result<()> {
+    let name =
+        HeaderName::from_bytes(name.as_bytes()).map_err(|_| invalid_input("Azure Blob metadata header name is invalid"))?;
+    let value = HeaderValue::from_str(value).map_err(|_| invalid_input("Azure Blob metadata header value is invalid"))?;
+    headers.insert(name, value);
+    Ok(())
+}
+
+fn invalid_input(message: &'static str) -> io::Error {
+    io::Error::new(ErrorKind::InvalidInput, message)
+}
+
+fn invalid_data(message: &'static str) -> io::Error {
+    io::Error::new(ErrorKind::InvalidData, message)
+}
+
+fn transport_error(err: reqwest::Error) -> io::Error {
+    let kind = if err.is_timeout() {
+        ErrorKind::TimedOut
+    } else {
+        ErrorKind::Other
+    };
+    io::Error::new(kind, err.without_url())
+}
+
+#[cfg(test)]
+#[path = "warm_backend_azure_blob_tests.rs"]
+mod tests;

crates/ecstore/src/services/tier/warm_backend_azure_blob_tests.rs

@@ -0,0 +1,929 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::services::tier::warm_backend::{WarmBackendImpl, check_warm_backend};
+use rustfs_utils::http::metadata_compat::insert_str;
+use tokio::io::AsyncWriteExt;
+use tokio::net::{TcpListener, TcpStream};
+use tokio::task::JoinHandle;
+
+struct Reply {
+    status: StatusCode,
+    headers: HeaderMap,
+    body: Bytes,
+    disconnect: bool,
+}
+
+fn reply(status: StatusCode, headers: &[(&str, &str)], body: &[u8]) -> Reply {
+    let mut map = HeaderMap::new();
+    for (name, value) in headers {
+        insert_header(&mut map, name, value).expect("fixture headers should be valid");
+    }
+    Reply {
+        status,
+        headers: map,
+        body: Bytes::copy_from_slice(body),
+        disconnect: false,
+    }
+}
+
+fn present(version: Option<&str>) -> Reply {
+    let mut response = reply(StatusCode::OK, &[("content-length", "6"), ("etag", "\"original\"")], b"");
+    if let Some(version) = version {
+        insert_header(&mut response.headers, "x-ms-version-id", version).expect("fixture version should be valid");
+    }
+    response
+}
+
+fn missing() -> Reply {
+    reply(StatusCode::NOT_FOUND, &[("x-ms-error-code", "BlobNotFound")], b"")
+}
+
+#[derive(Debug)]
+struct Captured {
+    method: String,
+    target: String,
+    headers: HeaderMap,
+    body: Vec<u8>,
+}
+
+async fn capture(stream: &mut TcpStream) -> Captured {
+    let mut bytes = Vec::new();
+    let boundary = loop {
+        let mut buffer = [0_u8; 4096];
+        let count = stream.read(&mut buffer).await.expect("fixture request should be readable");
+        assert_ne!(count, 0, "request must include a complete HTTP head");
+        bytes.extend_from_slice(&buffer[..count]);
+        if let Some(index) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
+            break index + 4;
+        }
+        assert!(bytes.len() <= 32 * 1024, "request headers must remain bounded");
+    };
+    let text = std::str::from_utf8(&bytes[..boundary]).expect("request headers should be ASCII");
+    let mut lines = text.split("\r\n");
+    let mut request_line = lines.next().expect("request line").split_whitespace();
+    let method = request_line.next().expect("request method").to_string();
+    let target = request_line.next().expect("request target").to_string();
+    let mut headers = HeaderMap::new();
+    for line in lines.filter(|line| !line.is_empty()) {
+        let (name, value) = line.split_once(':').expect("request header should include a colon");
+        insert_header(&mut headers, name, value.trim()).expect("request headers should parse");
+    }
+    let length = headers
+        .get("content-length")
+        .map(|value| {
+            value
+                .to_str()
+                .expect("length should be ASCII")
+                .parse::<usize>()
+                .expect("length should be numeric")
+        })
+        .unwrap_or_default();
+    assert!(length <= 17 * 1024 * 1024, "fixture only accepts its bounded test bodies");
+    let mut body = bytes[boundary..].to_vec();
+    let received = body.len();
+    body.resize(length, 0);
+    stream
+        .read_exact(&mut body[received..])
+        .await
+        .expect("request body should match its declared length");
+    Captured {
+        method,
+        target,
+        headers,
+        body,
+    }
+}
+
+async fn fixture(responses: Vec<Reply>) -> (WarmBackendAzureBlob, JoinHandle<Vec<Captured>>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("fixture listener should bind");
+    let address = listener.local_addr().expect("fixture address");
+    let task = tokio::spawn(async move {
+        let mut requests = Vec::new();
+        for mut response in responses {
+            let (mut stream, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept())
+                .await
+                .expect("expected request must arrive")
+                .expect("fixture should accept");
+            requests.push(capture(&mut stream).await);
+            if response.disconnect {
+                continue;
+            }
+            if !response.headers.contains_key("content-length") {
+                insert_header(&mut response.headers, "content-length", &response.body.len().to_string())
+                    .expect("fixture length should be valid");
+            }
+            let mut wire = format!(
+                "HTTP/1.1 {} {}\r\nConnection: close\r\n",
+                response.status.as_u16(),
+                response.status.canonical_reason().unwrap_or("")
+            );
+            for (name, value) in &response.headers {
+                wire.push_str(&format!("{name}: {}\r\n", value.to_str().expect("fixture header is ASCII")));
+            }
+            wire.push_str("\r\n");
+            stream.write_all(wire.as_bytes()).await.expect("fixture should send headers");
+            stream.write_all(&response.body).await.expect("fixture should send body");
+        }
+        requests
+    });
+    rustfs_tls_runtime::ensure_crypto_provider().expect("fixture TLS provider should initialize");
+    let client = reqwest::Client::builder()
+        .no_proxy()
+        .redirect(reqwest::redirect::Policy::none())
+        .retry(reqwest::retry::never())
+        .read_timeout(Duration::from_secs(5))
+        .build()
+        .expect("fixture client should build");
+    (
+        WarmBackendAzureBlob {
+            client,
+            endpoint: Url::parse(&format!("http://{address}")).expect("fixture endpoint"),
+            account: "account".to_string(),
+            key: vec![7; 32],
+            container: "container".to_string(),
+            prefix: "cold".to_string(),
+            request_timeout: Duration::from_secs(5),
+        },
+        task,
+    )
+}
+
+#[tokio::test]
+async fn mutation_probe_uses_blob_shared_key_and_cleans_up_with_an_etag() {
+    let responses = vec![
+        reply(StatusCode::CREATED, &[], b""),
+        present(None),
+        reply(StatusCode::PARTIAL_CONTENT, &[("content-range", "bytes 0-5/6")], b"RustFS"),
+        present(None),
+        reply(StatusCode::ACCEPTED, &[], b""),
+        missing(),
+    ];
+    let (backend, task) = fixture(responses).await;
+    let backend: WarmBackendImpl = Box::new(backend);
+    check_warm_backend(Some(&backend))
+        .await
+        .expect("native Blob mutation probe should succeed");
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(
+        requests.iter().map(|request| request.method.as_str()).collect::<Vec<_>>(),
+        ["PUT", "HEAD", "GET", "HEAD", "DELETE", "HEAD"]
+    );
+    assert!(requests[0].target.starts_with("/container/cold/rustfs-tier-probe-"));
+    assert_eq!(requests[0].body, b"RustFS");
+    assert_eq!(requests[0].headers["x-ms-blob-type"], "BlockBlob");
+    assert_eq!(requests[0].headers["if-none-match"], "*");
+    assert_eq!(requests[4].headers["if-match"], "\"original\"");
+    for request in &requests {
+        assert_eq!(request.target, requests[0].target, "the whole probe must address the same blob");
+        assert_eq!(request.headers["x-ms-version"], API_VERSION);
+        assert!(
+            request.headers["authorization"]
+                .to_str()
+                .expect("authorization is ASCII")
+                .starts_with("SharedKey account:")
+        );
+        assert!(!request.headers.contains_key("x-amz-date"));
+        assert!(!request.target.contains("location"));
+        assert!(!request.target.contains("versionid"));
+    }
+}
+
+#[tokio::test]
+async fn versioned_mutation_probe_cleans_up_after_forbidden_current_version_delete() {
+    let version = "2026-11-06T01:02:03.0000000Z";
+    let (backend, task) = fixture(vec![
+        reply(StatusCode::CREATED, &[("x-ms-version-id", version)], b""),
+        present(Some(version)),
+        reply(
+            StatusCode::PARTIAL_CONTENT,
+            &[("content-range", "bytes 0-5/6"), ("x-ms-version-id", version)],
+            b"RustFS",
+        ),
+        reply(StatusCode::FORBIDDEN, &[], b"<Error><Code>OperationNotAllowedOnRootBlob</Code></Error>"),
+        present(Some(version)),
+        reply(StatusCode::ACCEPTED, &[], b""),
+        reply(StatusCode::ACCEPTED, &[], b""),
+        missing(),
+    ])
+    .await;
+    let backend: WarmBackendImpl = Box::new(backend);
+    check_warm_backend(Some(&backend))
+        .await
+        .expect("a versioned Add/Verify probe must finish cleanup after the specific 403");
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(
+        requests.iter().map(|request| request.method.as_str()).collect::<Vec<_>>(),
+        ["PUT", "HEAD", "GET", "DELETE", "HEAD", "DELETE", "DELETE", "HEAD"]
+    );
+    let base = &requests[0].target;
+    assert!(base.starts_with("/container/cold/rustfs-tier-probe-"));
+    for index in [1, 4, 5, 7] {
+        assert_eq!(&requests[index].target, base, "base operations must retain the probe identity");
+    }
+    for index in [2, 3, 6] {
+        let url = Url::parse(&format!("http://fixture{}", requests[index].target)).expect("captured URL");
+        assert_eq!(url.path(), base);
+        assert_eq!(
+            url.query_pairs().collect::<Vec<_>>(),
+            [("versionid".into(), version.into())],
+            "exact reads and deletes must retain the timestamp version"
+        );
+    }
+    assert_eq!(requests[5].headers["if-match"], "\"original\"");
+    for request in &requests {
+        assert_eq!(request.headers["x-ms-version"], "2020-10-02");
+    }
+}
+
+#[tokio::test]
+async fn rejected_put_preserves_azure_error_and_never_deletes_an_existing_blob() {
+    for (status, code) in [
+        (StatusCode::BAD_REQUEST, "InvalidHeaderValue"),
+        (StatusCode::PRECONDITION_FAILED, "ConditionNotMet"),
+    ] {
+        let (backend, task) = fixture(vec![reply(
+            status,
+            &[("x-ms-error-code", code), ("x-ms-request-id", "abc-123")],
+            b"<Error><Code>PRIVATE-RESPONSE-BODY</Code></Error>",
+        )])
+        .await;
+        let backend: WarmBackendImpl = Box::new(backend);
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("deterministic rejected write must fail");
+        assert!(err.message.contains(code));
+        assert!(err.message.contains("abc-123"));
+        assert!(!err.message.contains("PRIVATE-RESPONSE-BODY"));
+        assert!(!err.message.contains("cleanup is incomplete"));
+        let requests = task.await.expect("fixture should join");
+        assert_eq!(requests.len(), 1, "a rejected conditional PUT must not trigger cleanup");
+    }
+}
+
+#[tokio::test]
+async fn empty_version_headers_use_the_unversioned_read_and_delete_paths() {
+    let mut head = present(Some(""));
+    head.headers.insert("content-length", HeaderValue::from_static("0"));
+    let (backend, task) = fixture(vec![
+        reply(StatusCode::CREATED, &[("x-ms-version-id", "")], b""),
+        reply(StatusCode::OK, &[("x-ms-version-id", "")], b""),
+        head,
+        reply(StatusCode::ACCEPTED, &[], b""),
+    ])
+    .await;
+    let version = backend
+        .put("empty", ReaderImpl::Body(Bytes::new()), 0)
+        .await
+        .expect("empty version is unversioned");
+    assert!(version.is_empty());
+    let mut reader = backend
+        .get("empty", &version, WarmBackendGetOpts::default())
+        .await
+        .expect("unversioned read");
+    let mut body = Vec::new();
+    reader.read_to_end(&mut body).await.expect("empty blob should read");
+    assert!(body.is_empty());
+    backend
+        .remove("empty", &version)
+        .await
+        .expect("unversioned conditional delete");
+    for request in task.await.expect("fixture should join") {
+        assert_eq!(request.target, "/container/cold/empty");
+    }
+}
+
+#[tokio::test]
+async fn rejected_credentials_are_reported_without_an_uncertain_cleanup() {
+    let (backend, task) = fixture(vec![reply(
+        StatusCode::FORBIDDEN,
+        &[("x-ms-error-code", "AuthenticationFailed")],
+        b"",
+    )])
+    .await;
+    let backend: WarmBackendImpl = Box::new(backend);
+    let err = check_warm_backend(Some(&backend))
+        .await
+        .expect_err("invalid credentials must fail");
+    assert_eq!(err.code, crate::services::tier::tier_handlers::ERR_TIER_INVALID_CREDENTIALS.code);
+    assert_eq!(task.await.expect("fixture should join").len(), 1);
+}
+
+#[tokio::test]
+async fn lost_put_response_is_not_replayed_and_its_candidate_is_cleaned() {
+    let mut lost = reply(StatusCode::CREATED, &[], b"");
+    lost.disconnect = true;
+    let (backend, task) = fixture(vec![
+        lost,
+        present(None),
+        present(None),
+        reply(StatusCode::ACCEPTED, &[], b""),
+        missing(),
+        missing(),
+    ])
+    .await;
+    let backend: WarmBackendImpl = Box::new(backend);
+    let err = check_warm_backend(Some(&backend))
+        .await
+        .expect_err("a lost write acknowledgement must remain a failure");
+    assert!(!err.message.contains("cleanup is incomplete"));
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests.iter().filter(|request| request.method == "PUT").count(), 1);
+    assert_eq!(requests.iter().filter(|request| request.method == "DELETE").count(), 1);
+}
+
+#[tokio::test]
+async fn block_upload_commits_only_the_complete_declared_body() {
+    let (backend, task) = fixture(vec![
+        reply(StatusCode::CREATED, &[], b""),
+        reply(StatusCode::CREATED, &[], b""),
+        reply(StatusCode::CREATED, &[], b""),
+    ])
+    .await;
+    let length = usize::try_from(MIN_BLOCK_BYTES + 5).expect("fixture size");
+    let bytes = Bytes::from(vec![42; length]);
+    assert_eq!(
+        backend
+            .put(
+                "large",
+                ReaderImpl::ObjectBody(ReadCloser::new(Cursor::new(bytes.clone()))),
+                i64::try_from(length).expect("fixture length")
+            )
+            .await
+            .expect("block upload"),
+        ""
+    );
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests.len(), 3);
+    let mut block_ids = Vec::new();
+    let mut body = Vec::new();
+    for request in &requests[..2] {
+        let url = Url::parse(&format!("http://fixture{}", request.target)).expect("captured URL");
+        let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
+        assert_eq!(query["comp"], "block");
+        block_ids.push(query["blockid"].clone());
+        assert!(request.headers.contains_key("content-md5"));
+        body.extend_from_slice(&request.body);
+    }
+    assert_eq!(body, bytes);
+    assert_ne!(block_ids[0], block_ids[1]);
+    assert!(requests[2].target.ends_with("?comp=blocklist"));
+    assert_eq!(requests[2].headers["if-none-match"], "*");
+    let expected = format!(
+        "<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList><Uncommitted>{}</Uncommitted><Uncommitted>{}</Uncommitted></BlockList>",
+        block_ids[0], block_ids[1],
+    );
+    assert_eq!(requests[2].body, expected.as_bytes());
+}
+
+#[tokio::test]
+async fn server_error_remains_visible_after_successful_probe_compensation() {
+    let (backend, task) = fixture(vec![
+        reply(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            &[("x-ms-error-code", "InternalError"), ("x-ms-request-id", "error-123")],
+            b"",
+        ),
+        present(None),
+        present(None),
+        reply(StatusCode::ACCEPTED, &[], b""),
+        missing(),
+        missing(),
+    ])
+    .await;
+    let backend: WarmBackendImpl = Box::new(backend);
+    let error = check_warm_backend(Some(&backend))
+        .await
+        .expect_err("failed upload remains an error after cleanup");
+    assert!(error.message.contains("500"));
+    assert!(error.message.contains("InternalError"));
+    assert!(error.message.contains("error-123"));
+    assert!(!error.message.contains("cleanup is incomplete"));
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests.iter().filter(|request| request.method == "PUT").count(), 1);
+    assert_eq!(requests.iter().filter(|request| request.method == "DELETE").count(), 1);
+}
+
+#[tokio::test]
+async fn empty_blob_and_declared_length_failures_are_handled_before_commit() {
+    let (backend, task) = fixture(vec![reply(StatusCode::CREATED, &[], b"")]).await;
+    backend
+        .put("empty", ReaderImpl::Body(Bytes::new()), 0)
+        .await
+        .expect("empty upload");
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests[0].headers["content-length"], "0");
+    assert_eq!(requests[0].headers["content-md5"], "1B2M2Y8AsgTpgAmY7PhCfg==");
+    for (body, kind) in [
+        (b"short".as_slice(), ErrorKind::UnexpectedEof),
+        (b"too-long".as_slice(), ErrorKind::InvalidInput),
+    ] {
+        let (backend, task) = fixture(vec![]).await;
+        let err = backend
+            .put("bad", ReaderImpl::Body(Bytes::copy_from_slice(body)), 6)
+            .await
+            .expect_err("length mismatch");
+        assert_eq!(err.kind(), kind);
+        assert!(task.await.expect("fixture should join").is_empty());
+    }
+    let (backend, task) = fixture(vec![reply(StatusCode::CREATED, &[], b"")]).await;
+    let body = Bytes::from(vec![0; usize::try_from(MIN_BLOCK_BYTES + 1).expect("fixture size")]);
+    let err = backend
+        .put(
+            "short-block",
+            ReaderImpl::Body(body),
+            i64::try_from(MIN_BLOCK_BYTES + 2).expect("fixture length"),
+        )
+        .await
+        .expect_err("short final block");
+    assert_eq!(err.kind(), ErrorKind::UnexpectedEof);
+    assert_eq!(
+        task.await.expect("fixture should join").len(),
+        1,
+        "short input must never commit its staged blocks"
+    );
+}
+
+#[tokio::test]
+async fn range_get_preserves_key_encoding_and_exact_version_for_delete() {
+    let version = "2026-11-06T01:02:03.0000000Z";
+    let (backend, task) = fixture(vec![
+        reply(
+            StatusCode::PARTIAL_CONTENT,
+            &[("content-range", "bytes 1-3/6"), ("x-ms-version-id", version)],
+            b"ust",
+        ),
+        reply(StatusCode::ACCEPTED, &[], b""),
+    ])
+    .await;
+    let object = "dir/a b+?#%2F.bin";
+    let mut reader = backend
+        .get(
+            object,
+            version,
+            WarmBackendGetOpts {
+                start_offset: 1,
+                length: 3,
+            },
+        )
+        .await
+        .expect("versioned range read");
+    let mut body = Vec::new();
+    reader.read_to_end(&mut body).await.expect("range body");
+    assert_eq!(body, b"ust");
+    backend.remove_exact(object, version).await.expect("exact delete");
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests.len(), 2, "exact delete needs no current-version HEAD");
+    for request in requests {
+        let url = Url::parse(&format!("http://fixture{}", request.target)).expect("captured URL");
+        assert_eq!(
+            urlencoding::decode(url.path()).expect("decoded path"),
+            format!("/container/cold/{object}")
+        );
+        let query: Vec<_> = url.query_pairs().into_owned().collect();
+        assert_eq!(query, [("versionid".to_string(), version.to_string())]);
+    }
+}
+
+#[tokio::test]
+async fn current_version_delete_demotes_only_the_matching_root_before_exact_cleanup() {
+    for status in [StatusCode::FORBIDDEN, StatusCode::CONFLICT] {
+        let version = "target-version";
+        let (backend, task) = fixture(vec![
+            reply(status, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
+            present(Some(version)),
+            reply(StatusCode::ACCEPTED, &[], b""),
+            reply(StatusCode::ACCEPTED, &[], b""),
+        ])
+        .await;
+        backend
+            .remove_exact("current", version)
+            .await
+            .expect("current version should be removable");
+        let requests = task.await.expect("fixture should join");
+        assert_eq!(requests.len(), 4);
+        assert_eq!(requests[0].target, "/container/cold/current?versionid=target-version");
+        assert_eq!(requests[1].method, "HEAD");
+        assert_eq!(requests[1].target, "/container/cold/current");
+        assert_eq!(requests[2].method, "DELETE");
+        assert_eq!(requests[2].target, "/container/cold/current");
+        assert_eq!(requests[2].headers["if-match"], "\"original\"");
+        assert_eq!(requests[3].method, "DELETE");
+        assert_eq!(requests[3].target, requests[0].target);
+    }
+}
+
+#[tokio::test]
+async fn current_version_delete_refuses_replacements_before_and_after_head() {
+    for replaced_after_head in [false, true] {
+        let mut responses = vec![
+            reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
+            present(Some(if replaced_after_head {
+                "target-version"
+            } else {
+                "replacement-version"
+            })),
+        ];
+        if replaced_after_head {
+            responses.push(reply(StatusCode::PRECONDITION_FAILED, &[("x-ms-error-code", "ConditionNotMet")], b""));
+        }
+        let (backend, task) = fixture(responses).await;
+        assert!(backend.remove_exact("current", "target-version").await.is_err());
+        let requests = task.await.expect("fixture should join");
+        if replaced_after_head {
+            assert_eq!(requests.len(), 3, "a failed condition must stop exact cleanup");
+            assert_eq!(requests[2].target, "/container/cold/current");
+            assert_eq!(requests[2].headers["if-match"], "\"original\"");
+        } else {
+            assert_eq!(requests.len(), 2, "a different current version must never authorize a base delete");
+        }
+    }
+}
+
+#[tokio::test]
+async fn current_version_delete_retries_the_exact_version_when_the_root_is_already_absent() {
+    let (backend, task) = fixture(vec![
+        reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
+        missing(),
+        reply(StatusCode::ACCEPTED, &[], b""),
+    ])
+    .await;
+    backend
+        .remove_exact("current", "target-version")
+        .await
+        .expect("previous version cleanup");
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests.len(), 3);
+    assert_eq!(requests[2].target, "/container/cold/current?versionid=target-version");
+    assert!(
+        requests
+            .iter()
+            .all(|request| request.method != "DELETE" || request.target.contains("versionid="))
+    );
+}
+
+#[tokio::test]
+async fn current_version_delete_keeps_partial_or_uncertain_cleanup_retryable() {
+    for fails_during_exact_cleanup in [false, true] {
+        for lost_response in [false, true] {
+            let mut responses = vec![
+                reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
+                present(Some("target-version")),
+            ];
+            if fails_during_exact_cleanup {
+                responses.push(reply(StatusCode::ACCEPTED, &[], b""));
+            }
+            let mut failure = reply(StatusCode::INTERNAL_SERVER_ERROR, &[("x-ms-error-code", "InternalError")], b"");
+            failure.disconnect = lost_response;
+            responses.push(failure);
+            responses.push(reply(StatusCode::ACCEPTED, &[], b""));
+            let (backend, task) = fixture(responses).await;
+            backend
+                .remove_exact("current", "target-version")
+                .await
+                .expect_err("base removal alone or an uncertain response must not complete cleanup");
+            backend
+                .remove_exact("current", "target-version")
+                .await
+                .expect("retry exact cleanup");
+            let requests = task.await.expect("fixture should join");
+            let expected = if fails_during_exact_cleanup { 5 } else { 4 };
+            assert_eq!(requests.len(), expected, "uncertain destructive requests must not be replayed internally");
+            assert_eq!(requests[2].headers["if-match"], "\"original\"");
+            assert_eq!(requests[expected - 1].target, "/container/cold/current?versionid=target-version");
+            assert_eq!(
+                requests
+                    .iter()
+                    .filter(|request| request.method == "DELETE" && !request.target.contains('?'))
+                    .count(),
+                1,
+                "a retry must not delete the root again"
+            );
+        }
+    }
+}
+
+#[tokio::test]
+async fn delete_requires_a_single_strong_etag_and_an_exact_current_version() {
+    for versioned in [false, true] {
+        for etag in [
+            None,
+            Some(""),
+            Some("*"),
+            Some("W/\"weak\""),
+            Some("\"first\", \"second\""),
+            Some("\"\""),
+        ] {
+            let mut head = present(if versioned { Some("target-version") } else { None });
+            head.headers.remove("etag");
+            if let Some(etag) = etag {
+                insert_header(&mut head.headers, "etag", etag).expect("fixture etag should be representable");
+            }
+            let mut responses = Vec::new();
+            if versioned {
+                responses.push(reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""));
+            }
+            responses.push(head);
+            let (backend, task) = fixture(responses).await;
+            assert!(
+                backend
+                    .remove("current", if versioned { "target-version" } else { "" })
+                    .await
+                    .is_err()
+            );
+            let requests = task.await.expect("fixture should join");
+            assert!(
+                requests
+                    .iter()
+                    .all(|request| request.method != "DELETE" || request.target.contains("versionid=")),
+                "a missing, wildcard, weak or multi-value ETag cannot authorize root removal"
+            );
+        }
+    }
+    for current_version in [None, Some("")] {
+        let (backend, task) = fixture(vec![
+            reply(StatusCode::FORBIDDEN, &[("x-ms-error-code", "OperationNotAllowedOnRootBlob")], b""),
+            present(current_version),
+        ])
+        .await;
+        assert!(backend.remove_exact("current", "target-version").await.is_err());
+        assert_eq!(task.await.expect("fixture should join").len(), 2);
+    }
+}
+
+#[tokio::test]
+async fn exact_delete_does_not_demote_roots_for_other_service_errors() {
+    for (status, code) in [
+        (StatusCode::CONFLICT, "BlobImmutableDueToPolicy"),
+        (StatusCode::FORBIDDEN, "BlobImmutableDueToPolicy"),
+        (StatusCode::FORBIDDEN, "AuthenticationFailed"),
+        (StatusCode::FORBIDDEN, "AuthorizationFailure"),
+        (StatusCode::FORBIDDEN, "AuthorizationPermissionMismatch"),
+        (StatusCode::FORBIDDEN, "OperationNotAllowedOnRootBlobOther"),
+        (StatusCode::UNAUTHORIZED, "OperationNotAllowedOnRootBlob"),
+        (StatusCode::INTERNAL_SERVER_ERROR, "OperationNotAllowedOnRootBlob"),
+    ] {
+        let (backend, task) = fixture(vec![reply(status, &[("x-ms-error-code", code)], b"")]).await;
+        let error = backend
+            .remove_exact("current", "target-version")
+            .await
+            .expect_err("service rejection");
+        assert_eq!(AzureBlobError::from_io(&error).expect("typed service error").code, code);
+        assert_eq!(
+            task.await.expect("fixture should join").len(),
+            1,
+            "unrelated errors cannot trigger root removal"
+        );
+    }
+}
+
+#[test]
+fn prefix_reserves_the_complete_canonical_lifecycle_name_before_backend_creation() {
+    use crate::bucket::lifecycle::transition_transaction::canonical_transition_remote_object;
+    let mut config = TierAzure {
+        access_key: "account".to_string(),
+        secret_key: base64_simd::STANDARD.encode_to_string([7; 32]),
+        bucket: "container".to_string(),
+        prefix: "a".repeat(800),
+        ..Default::default()
+    };
+    assert!(
+        WarmBackendAzureBlob::new(&config).is_err(),
+        "a prefix that fits only the probe is unusable for lifecycle"
+    );
+    let object = canonical_transition_remote_object(
+        uuid::Uuid::from_u128(1),
+        "source-bucket",
+        uuid::Uuid::from_u128(2),
+        uuid::Uuid::from_u128(3),
+    )
+    .expect("canonical lifecycle key");
+    let max_ascii_prefix = 880 - object.chars().count() - 1;
+    config.prefix = "a".repeat(max_ascii_prefix);
+    let backend = WarmBackendAzureBlob::new(&config).expect("complete name at the character limit");
+    assert!(backend.object_url(&object, "").is_ok());
+    config.prefix.push('/');
+    let backend = WarmBackendAzureBlob::new(&config).expect("one trailing slash is normalized");
+    assert!(backend.object_url(&object, "").is_ok());
+    config.prefix = "a".repeat(max_ascii_prefix + 1);
+    assert!(WarmBackendAzureBlob::new(&config).is_err(), "reject one extra character before I/O");
+    let max_unicode_prefix = (1760 - object.len() - 1) / '界'.len_utf8();
+    config.prefix = "界".repeat(max_unicode_prefix);
+    let backend = WarmBackendAzureBlob::new(&config).expect("complete Unicode name within the byte limit");
+    assert!(backend.object_url(&object, "").is_ok());
+    config.prefix.push('界');
+    assert!(
+        WarmBackendAzureBlob::new(&config).is_err(),
+        "the byte limit must include the canonical suffix"
+    );
+}
+
+#[tokio::test]
+async fn version_drift_and_incorrect_ranges_fail_closed() {
+    let (backend, task) = fixture(vec![present(Some("new-version"))]).await;
+    let err = backend
+        .remove("changed", "")
+        .await
+        .expect_err("unversioned delete must reject version drift");
+    assert!(err.to_string().contains("became versioned"));
+    assert_eq!(task.await.expect("fixture should join").len(), 1);
+    for headers in [
+        vec![("content-range", "bytes 0-2/6")],
+        vec![("content-range", "bytes 1-3/6"), ("x-ms-version-id", "unexpected-version")],
+    ] {
+        let (backend, task) = fixture(vec![reply(StatusCode::PARTIAL_CONTENT, &headers, b"ust")]).await;
+        let result = backend
+            .get(
+                "object",
+                "",
+                WarmBackendGetOpts {
+                    start_offset: 1,
+                    length: 3,
+                },
+            )
+            .await;
+        assert!(result.is_err(), "wrong offset or an unexpected version cannot satisfy the read");
+        task.await.expect("fixture should join");
+    }
+}
+
+#[tokio::test]
+async fn unversioned_delete_keeps_the_head_precondition_when_the_blob_changes() {
+    let (backend, task) = fixture(vec![
+        present(None),
+        reply(StatusCode::PRECONDITION_FAILED, &[("x-ms-error-code", "ConditionNotMet")], b""),
+    ])
+    .await;
+    assert!(backend.remove("changed", "").await.is_err());
+    let requests = task.await.expect("fixture should join");
+    assert_eq!(requests[1].headers["if-match"], "\"original\"");
+}
+
+#[tokio::test]
+async fn absence_requires_blob_not_found_and_container_listing_is_bounded() {
+    for (response, missing_expected) in [
+        (missing(), true),
+        (reply(StatusCode::NOT_FOUND, &[("x-ms-error-code", "ContainerNotFound")], b""), false),
+        (reply(StatusCode::NOT_FOUND, &[], b""), false),
+    ] {
+        let (backend, task) = fixture(vec![response]).await;
+        let result = backend.probe_transition_candidate("object").await;
+        if missing_expected {
+            assert_eq!(result.expect("explicit blob absence"), TransitionCandidateProbe::Missing);
+        } else {
+            assert!(result.is_err(), "an absent container or an ambiguous HEAD is not blob absence");
+        }
+        task.await.expect("fixture should join");
+    }
+    for (body, in_use) in [
+        (b"<EnumerationResults><Blobs/><NextMarker/></EnumerationResults>".as_slice(), false),
+        (
+            b"<EnumerationResults><Blobs><Blob><Name>cold/object</Name></Blob></Blobs><NextMarker/></EnumerationResults>"
+                .as_slice(),
+            true,
+        ),
+        (
+            b"<EnumerationResults><Blobs/><NextMarker>next</NextMarker></EnumerationResults>".as_slice(),
+            true,
+        ),
+    ] {
+        let (backend, task) = fixture(vec![reply(StatusCode::OK, &[], body)]).await;
+        assert_eq!(backend.in_use().await.expect("native container listing"), in_use);
+        let requests = task.await.expect("fixture should join");
+        assert!(
+            requests[0]
+                .target
+                .contains("restype=container&comp=list&prefix=cold%2F&maxresults=1")
+        );
+    }
+    let oversized = vec![b' '; MAX_CONTROL_BYTES + 1];
+    let (backend, task) = fixture(vec![reply(StatusCode::OK, &[], &oversized)]).await;
+    assert!(backend.in_use().await.is_err(), "oversized listing cannot establish an empty destination");
+    task.await.expect("fixture should join");
+}
+
+#[tokio::test]
+async fn recovery_requires_the_uploaded_transaction_and_destination_identity() {
+    let identity = TransitionCandidateIdentity {
+        transaction_id: uuid::Uuid::new_v4(),
+        destination_id: [3; 32],
+    };
+    let mut metadata = HashMap::new();
+    insert_str(&mut metadata, SUFFIX_TRANSITION_TRANSACTION_ID, identity.transaction_id.to_string());
+    insert_str(&mut metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, hex(identity.destination_id));
+    let headers = metadata_headers(metadata).expect("identity metadata should encode for Azure");
+    assert_eq!(
+        headers
+            .keys()
+            .filter(|name| name.as_str().starts_with(METADATA_PREFIX))
+            .count(),
+        4
+    );
+    let mut owned = present(Some("owned-version"));
+    owned.headers.extend(headers);
+    let (backend, task) = fixture(vec![owned, present(None)]).await;
+    assert_eq!(
+        backend
+            .probe_transition_candidate_for("object", identity)
+            .await
+            .expect("owned candidate"),
+        TransitionCandidateProbe::VersionedPresent("owned-version".to_string())
+    );
+    assert_eq!(
+        backend
+            .probe_transition_candidate_for("object", identity)
+            .await
+            .expect("foreign candidate classification"),
+        TransitionCandidateProbe::Ambiguous
+    );
+    task.await.expect("fixture should join");
+}
+
+#[test]
+fn configuration_xml_and_size_limits_fail_before_network_io() {
+    let mut config = TierAzure {
+        access_key: "account".to_string(),
+        secret_key: base64_simd::STANDARD.encode_to_string([7; 32]),
+        bucket: "container".to_string(),
+        ..Default::default()
+    };
+    assert!(
+        WarmBackendAzureBlob::new(&config).is_ok(),
+        "public endpoint should be derived from the account"
+    );
+    config.endpoint = "http://127.0.0.1:10000".to_string();
+    assert!(
+        WarmBackendAzureBlob::new(&config).is_err(),
+        "production constructor must apply the outbound guard"
+    );
+    config.endpoint = "https://account.blob.example/path".to_string();
+    assert!(
+        WarmBackendAzureBlob::new(&config).is_err(),
+        "an endpoint path cannot rewrite object addressing"
+    );
+    for name in [".", "..", "a/../b", "a/./b"] {
+        assert!(validate_object_name(name).is_err());
+    }
+    assert_eq!(
+        block_size(MAX_BLOCK_BYTES * MAX_BLOCKS).expect("largest supported object"),
+        MAX_BLOCK_BYTES
+    );
+    assert!(block_size(MAX_BLOCK_BYTES * MAX_BLOCKS + 1).is_err());
+    for xml in [
+        b"<html>not a container</html>".as_slice(),
+        b"<EnumerationResults><Blobs>".as_slice(),
+        b"<!DOCTYPE x><EnumerationResults><Blobs/></EnumerationResults>".as_slice(),
+        b"<EnumerationResults><Other><Blobs/></Other></EnumerationResults>".as_slice(),
+        b"garbage<EnumerationResults><Blobs/></EnumerationResults>".as_slice(),
+    ] {
+        assert!(list_has_objects(xml).is_err());
+    }
+    assert_eq!(
+        xml_error_code(b"<Error><Code>BlobNotFound</Code></Error>").as_deref(),
+        Some("BlobNotFound")
+    );
+    assert!(xml_error_code(b"<Error><Code>BlobNotFound</Code>").is_none());
+    assert!(xml_error_code(b"<Error><Code>BlobNotFound</Code></Error>garbage").is_none());
+}
+
+#[tokio::test]
+async fn control_response_has_a_total_deadline_after_headers_arrive() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("control fixture should bind");
+    let address = listener.local_addr().expect("control fixture address");
+    let (release, wait) = tokio::sync::oneshot::channel();
+    let server = tokio::spawn(async move {
+        let (mut stream, _) = listener.accept().await.expect("control fixture should accept");
+        capture(&mut stream).await;
+        stream
+            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 16\r\n\r\n")
+            .await
+            .expect("control response headers");
+        let _ = wait.await;
+    });
+    let (mut backend, empty) = fixture(Vec::new()).await;
+    empty.await.expect("unused fixture should close");
+    backend.endpoint = Url::parse(&format!("http://{address}")).expect("control endpoint should parse");
+    let response = backend
+        .send(Method::GET, backend.container_url().expect("container URL"), HeaderMap::new(), None)
+        .await
+        .expect("headers should arrive before testing the body deadline");
+    let err = tokio::time::timeout(Duration::from_secs(1), read_control_body(response, Duration::from_millis(20)))
+        .await
+        .expect("body deadline must be shorter than the transport idle timeout")
+        .expect_err("an unfinished control response must time out");
+    assert_eq!(err.kind(), ErrorKind::TimedOut);
+    let _ = release.send(());
+    server.await.expect("control fixture should join");
+}

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -351,7 +351,7 @@ fn transition_candidate_metadata(info: ObjectInfo) -> HashMap<String, String> {
     metadata
 }
 
-fn transition_candidate_metadata_matches(
+pub(super) fn transition_candidate_metadata_matches(
     metadata: &HashMap<String, String>,
     identity: TransitionCandidateIdentity,
 ) -> Result<bool, std::io::Error> {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -284,8 +284,9 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
             present_count += 1;
             let part = &parts[part_idx];
             let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
-            let (count, _) = part_quorum.entry(key).or_insert((0, part));
+            let (count, protected_count, _) = part_quorum.entry(key).or_insert((0, 0, part));
             *count += 1;
+            *protected_count += usize::from(part.encrypted_etag);
             continue;
         }
 
@@ -296,12 +297,16 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
         }
     }
 
-    let max_part = part_quorum.values().max_by_key(|(count, _)| count);
-    let max_quorum = max_part.map_or(0, |(count, _)| *count).max(missing_count);
-    if let Some((count, part)) = max_part
+    let max_part = part_quorum.values().max_by_key(|(count, _, _)| count);
+    let max_quorum = max_part.map_or(0, |(count, _, _)| *count).max(missing_count);
+    if let Some((count, protected_count, part)) = max_part
         && *count >= read_quorum
     {
-        return Ok((*part).clone());
+        let mut part = (*part).clone();
+        // Older peers may omit the extension while agreeing on the part itself.
+        // Require independent proof quorum without reducing part availability.
+        part.encrypted_etag = *protected_count >= read_quorum;
+        return Ok(part);
     }
 
     if missing_count >= read_quorum {
@@ -13390,6 +13395,53 @@ mod tests {
         }
     }
 
+    #[test]
+    fn resolve_read_part_requires_matching_quorum_for_encrypted_etag_proof() {
+        let protected = ObjectPartInfo {
+            encrypted_etag: true,
+            ..read_part_test_part(1, "protected-etag")
+        };
+        let legacy = ObjectPartInfo {
+            encrypted_etag: false,
+            ..protected.clone()
+        };
+        for (parts, quorum, expected) in [
+            (vec![protected.clone(), legacy.clone(), legacy.clone()], 2, false),
+            (vec![legacy.clone(), protected.clone(), protected.clone()], 2, true),
+            (vec![protected.clone(), protected.clone(), legacy.clone(), legacy.clone()], 3, false),
+            (
+                vec![
+                    protected.clone(),
+                    ObjectPartInfo {
+                        etag: "other-generation".to_owned(),
+                        ..protected.clone()
+                    },
+                    legacy.clone(),
+                ],
+                2,
+                false,
+            ),
+            (
+                vec![
+                    protected.clone(),
+                    ObjectPartInfo {
+                        error: Some("read failed".to_owned()),
+                        ..protected.clone()
+                    },
+                    legacy,
+                ],
+                2,
+                false,
+            ),
+        ] {
+            let responses = parts.into_iter().map(|part| Some(vec![part])).collect::<Vec<_>>();
+            let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, quorum)
+                .expect("matching metadata remains readable across producer versions");
+            assert_eq!(part.etag, protected.etag);
+            assert_eq!(part.encrypted_etag, expected);
+        }
+    }
+
     #[test]
     fn resolve_read_part_diagnostic_branch_keeps_read_quorum_error() {
         temp_env::with_var("RUSTFS_ISSUE3031_DIAG_ENABLE", Some("true"), || {

crates/ecstore/src/set_disk/mod.rs

@@ -7980,21 +7980,23 @@ fn is_min_allowed_part_size(size: i64) -> bool {
 }
 
 fn get_complete_multipart_md5(parts: &[CompletePart]) -> String {
-    let mut buf = Vec::new();
-
-    for part in parts.iter() {
-        if let Some(etag) = &part.etag {
-            if let Ok(etag_bytes) = hex_simd::decode_to_vec(etag.as_bytes()) {
-                buf.extend(etag_bytes);
-            } else {
-                buf.extend(etag.bytes());
+    let mut hasher = Md5::new();
+    for etag in parts.iter().filter_map(|part| part.etag.as_deref()) {
+        // Native part ETags decode to one fixed-size digest. Preserve the
+        // historical hex-or-raw interpretation for imported part ETags.
+        let mut digest = [0u8; 16];
+        if etag.len() == digest.len() * 2 {
+            match hex_simd::decode(etag.as_bytes(), hex_simd::Out::from_slice(&mut digest)) {
+                Ok(decoded) => hasher.update(decoded),
+                Err(_) => hasher.update(etag.as_bytes()),
             }
+        } else if let Ok(decoded) = hex_simd::decode_to_vec(etag.as_bytes()) {
+            hasher.update(decoded);
+        } else {
+            hasher.update(etag.as_bytes());
         }
     }
 
-    let mut hasher = Md5::new();
-    hasher.update(&buf);
-
     let digest = hasher.finalize();
     let etag_hex = faster_hex::hex_string(digest.as_slice());
     format!("{}-{}", etag_hex, parts.len())
@@ -9682,50 +9684,59 @@ mod tests {
     }
 
     #[test]
-    fn test_get_complete_multipart_md5() {
-        // Test MD5 calculation for multipart upload
-        let parts = vec![
+    fn complete_multipart_md5_preserves_legacy_etag_encodings() {
+        fn part(etag: Option<&str>, part_num: usize) -> CompletePart {
             CompletePart {
-                part_num: 1,
-                etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
-                checksum_crc32: None,
-                checksum_crc32c: None,
-                checksum_sha1: None,
-                checksum_sha256: None,
-                checksum_crc64nvme: None,
-            },
-            CompletePart {
-                part_num: 2,
-                etag: Some("098f6bcd4621d373cade4e832627b4f6".to_string()),
-                checksum_crc32: None,
-                checksum_crc32c: None,
-                checksum_sha1: None,
-                checksum_sha256: None,
-                checksum_crc64nvme: None,
-            },
+                part_num,
+                etag: etag.map(str::to_owned),
+                ..Default::default()
+            }
+        }
+
+        fn buffered_digest(parts: &[CompletePart]) -> String {
+            let mut bytes = Vec::new();
+            for etag in parts.iter().filter_map(|part| part.etag.as_deref()) {
+                match hex_simd::decode_to_vec(etag.as_bytes()) {
+                    Ok(decoded) => bytes.extend(decoded),
+                    Err(_) => bytes.extend_from_slice(etag.as_bytes()),
+                }
+            }
+            format!("{}-{}", faster_hex::hex_string(&Md5::digest(bytes)), parts.len())
+        }
+
+        assert_eq!(get_complete_multipart_md5(&[]), "d41d8cd98f00b204e9800998ecf8427e-0");
+        assert_eq!(get_complete_multipart_md5(&[part(None, 1)]), "d41d8cd98f00b204e9800998ecf8427e-1");
+        assert_eq!(get_complete_multipart_md5(&[part(Some("00"), 1)]), "93b885adfe0da089cdf634904fd59f71-1");
+        assert_eq!(
+            get_complete_multipart_md5(&[
+                part(Some("d41d8cd98f00b204e9800998ecf8427e"), 1),
+                part(Some("098f6bcd4621d373cade4e832627b4f6"), 2),
+            ]),
+            "f666c301a43456b66b9f3a47f696d391-2"
+        );
+
+        let encodings = [
+            None,
+            Some(""),
+            Some("00"),
+            Some("0aBcFf"),
+            Some("0123456789abcdef0123456789abcdef"),
+            Some("0123456789ABCDEF0123456789ABCDEF"),
+            Some("0123456789abcdef0123456789abcdef00"),
+            Some("0123456789abcdef0123456789abcdeg"),
+            Some("0123456789abcdef0123456789abcdeg00"),
+            Some("f"),
+            Some("not-hex"),
+            Some("\"0123456789abcdef0123456789abcdef\""),
+            Some("0123456789abcdef0123456789abcdef-2"),
+            Some("café"),
         ];
-
-        let md5 = get_complete_multipart_md5(&parts);
-        assert!(md5.ends_with("-2")); // Should end with part count
-        assert!(md5.len() > 10); // Should have reasonable length
-
-        // Test with empty parts
-        let empty_parts = vec![];
-        let empty_result = get_complete_multipart_md5(&empty_parts);
-        assert!(empty_result.ends_with("-0"));
-
-        // Test with single part
-        let single_part = vec![CompletePart {
-            part_num: 1,
-            etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
-            checksum_crc32: None,
-            checksum_crc32c: None,
-            checksum_sha1: None,
-            checksum_sha256: None,
-            checksum_crc64nvme: None,
-        }];
-        let single_result = get_complete_multipart_md5(&single_part);
-        assert!(single_result.ends_with("-1"));
+        for first in encodings {
+            for second in encodings {
+                let parts = [part(first, 1), part(second, 2)];
+                assert_eq!(get_complete_multipart_md5(&parts), buffered_digest(&parts), "{first:?}, {second:?}");
+            }
+        }
     }
 
     #[test]

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -46,8 +46,8 @@ use super::super::{
     log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
     prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
     reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
-    restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
-    to_object_err, warn,
+    restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, should_persist_encryption_original_size,
+    strip_internal_multipart_metadata, to_object_err, warn,
 };
 use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
 #[cfg(test)]
@@ -112,6 +112,10 @@ use std::time::Duration;
 use tokio::io::AsyncReadExt;
 use tokio::task::JoinSet;
 
+#[cfg(test)]
+#[path = "multipart_quorum_tests.rs"]
+mod multipart_writer_test_hook;
+
 // The erasure-set width bounds fan-out. Await every opener so errors retain
 // their disk slots and every successful writer remains owned until quorum is checked.
 async fn create_part_writers(
@@ -124,6 +128,13 @@ async fn create_part_writers(
         let Some(disk) = disk else {
             return (None, Some(DiskError::DiskNotFound));
         };
+        #[cfg(test)]
+        if let Some(result) = multipart_writer_test_hook::create_writer(disk, path, length, shard_size).await {
+            return match result {
+                Ok(writer) => (Some(writer), None),
+                Err(error) => (None, Some(error)),
+            };
+        }
         match create_bitrot_writer_with_orig_volume(
             false,
             Some(disk),
@@ -2644,8 +2655,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 SmallWritePath::PipelineBatchedLarge => IntegrityEncodeMode::Batched,
                 SmallWritePath::Inline | SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
             };
+            let shard_write_policy = crate::erasure::coding::encode::ShardWritePolicy::multipart(write_quorum);
+            #[cfg(test)]
+            let shard_write_policy = multipart_writer_test_hook::write_policy(&shuffle_disks, write_quorum)
+                .unwrap_or(shard_write_policy);
             let (reader, w_size, _, integrity) = Arc::clone(&erasure)
-                .encode_with_shard_integrity(stream, &mut writers, write_quorum, part_id, mode, protected_upload)
+                .encode_with_shard_integrity(
+                    stream,
+                    &mut writers,
+                    shard_write_policy,
+                    part_id,
+                    mode,
+                    protected_upload,
+                )
                 .await?;
 
             if let Some(stage_start) = encode_stage_start {
@@ -2706,6 +2728,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 .map(crate::io_support::rio::compression_index_storage_bytes);
 
             let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
+            let encrypted_etag = data.stream.encrypted_etag
+                && !etag.is_empty()
+                && opts.preserve_etag.as_ref().is_none_or(|preserved| preserved == &etag);
 
             if let Some(ref tag) = opts.preserve_etag {
                 etag = tag.clone();
@@ -2726,6 +2751,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
             let part_info = ObjectPartInfo {
                 etag: etag.clone(),
+                encrypted_etag,
                 number: part_id,
                 size: w_size,
                 mod_time: Some(OffsetDateTime::now_utc()),
@@ -3625,6 +3651,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
         }
         let source_condition = capture_source_condition(bucket, object, &upload_id_path, &fi, Uuid::new_v4())?;
+        let preserved_encrypted_etag = preserves_staged_encrypted_etag_proof(&fi, opts, expected_restore_operation_id);
         let upload_data_dir = source_condition.lineage.original_data_dir.to_string();
         // The source generation remains in the captured precondition. Its
         // ACTIVE/recovery controls are not final user-object metadata.
@@ -4095,11 +4122,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         }
 
         // etag
-        let etag = resolve_complete_etag(opts, &uploaded_parts);
+        let (etag, encrypted_etag) = resolve_complete_etag(opts, &uploaded_parts, &fi.parts);
+        let encrypted_etag = encrypted_etag
+            || preserved_encrypted_etag
+                && fi.version_id.filter(|version| !version.is_nil()) == source_condition.lineage.original_version_id;
 
-        // An upload can contain parts from an older writer or a peer. Until
-        // every part's format is attested, keep its ETag out of keyless lists.
+        // New writes attest their selected parts. Same-version internal moves
+        // and restores can retain the proof from quorum-validated source metadata.
         rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
+        if encrypted_etag {
+            insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+        }
 
         fi.metadata.insert("etag".to_owned(), etag);
 
@@ -4926,19 +4959,57 @@ fn completed_multipart_upload_matches(
     Ok(true)
 }
 
+// Capture only the original staged proof, before completion applies metadata
+// updates. Publication must also retain the source lineage's version.
+fn preserves_staged_encrypted_etag_proof(fi: &FileInfo, opts: &ObjectOptions, restore_operation_id: Option<Uuid>) -> bool {
+    if opts.replication_request {
+        return false;
+    }
+    let movement = opts.data_movement
+        && opts.version_id.is_some()
+        && get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD)
+            .is_some_and(|identity| !identity.is_empty());
+    // Commit rechecks this operation against the live object under its lock.
+    let restore = restore_operation_id
+        .is_some_and(|expected| restore_operation_id_from_metadata(&fi.metadata).is_ok_and(|id| id == Some(expected)));
+    if !movement && !restore {
+        return false;
+    }
+    let Ok(version) = opts.version_id.as_deref().map(Uuid::parse_str).transpose() else {
+        return false;
+    };
+    let Some(etag) = opts.preserve_etag.as_deref().filter(|etag| !etag.is_empty()) else {
+        return false;
+    };
+    version.filter(|version| !version.is_nil()) == fi.version_id.filter(|version| !version.is_nil())
+        && fi.metadata.get("etag").map(String::as_str) == Some(etag)
+        && get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG) == Some("v1")
+}
+
 /// Final ETag for a completed multipart object. An authorized replication
 /// request preserves the source ETag so the replication HEAD comparison
 /// converges even when the source ETag is not derivable from the uploaded
 /// parts (foreign-origin objects, ciphertext-derived ETags); the internal
 /// metadata override comes next; otherwise the ETag is computed from parts.
-fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart]) -> String {
-    if let Some(etag) = opts.preserve_etag.as_ref().filter(|etag| !etag.is_empty()) {
-        return etag.clone();
+fn resolve_complete_etag(
+    opts: &ObjectOptions,
+    uploaded_parts: &[CompletePart],
+    stored_parts: &[ObjectPartInfo],
+) -> (String, bool) {
+    let preserved = opts
+        .preserve_etag
+        .as_ref()
+        .filter(|etag| !etag.is_empty())
+        .or_else(|| opts.user_defined.get("etag"));
+    let protected_parts = !stored_parts.is_empty() && stored_parts.iter().all(|part| part.encrypted_etag);
+    if let Some(etag) = preserved
+        && !protected_parts
+    {
+        return (etag.clone(), false);
     }
-    if let Some(etag) = opts.user_defined.get("etag") {
-        return etag.clone();
-    }
-    get_complete_multipart_md5(uploaded_parts)
+    let computed = get_complete_multipart_md5(uploaded_parts);
+    let encrypted_etag = protected_parts && preserved.is_none_or(|etag| etag == &computed);
+    (preserved.cloned().unwrap_or(computed), encrypted_etag)
 }
 
 /// Serialize the object-level checksum record that `complete_multipart_upload`
@@ -9071,6 +9142,26 @@ mod tests {
                 receipts.push(part.etag);
             }
         }
+        let (staged, _) = set
+            .check_upload_id_exists(bucket, "object", &upload.upload_id, false)
+            .await
+            .expect("read upload generation");
+        let upload_path = SetDisks::get_upload_id_dir(bucket, "object", &upload.upload_id);
+        let data_dir = staged.data_dir.expect("staging data directory");
+        for disk in &disks {
+            let metadata = disk
+                .read_parts(
+                    RUSTFS_META_MULTIPART_BUCKET,
+                    &[
+                        format!("{upload_path}/{data_dir}/part.1.meta"),
+                        format!("{upload_path}/{data_dir}/part.3.meta"),
+                    ],
+                )
+                .await
+                .expect("read persisted part proofs");
+            assert!(metadata[0].encrypted_etag, "rewritten protected part is attested");
+            assert!(!metadata[1].encrypted_etag, "an overridden part ETag has no producer proof");
+        }
         assert_ne!(receipts[0], receipts[1], "spare parts must not identify another part's plaintext");
         assert_ne!(receipts[0], receipts[2], "same-number retries get a new receipt");
         for _ in 0..2 {
@@ -9107,6 +9198,230 @@ mod tests {
             .await
             .expect("committed completion replay is stable");
         assert_eq!(replay.etag, committed.etag);
+        for completed in [committed, replay] {
+            assert!(
+                rustfs_utils::http::etag_can_be_listed(&completed.user_defined),
+                "the selected protected part retains its ETag after completion and replay"
+            );
+            assert_eq!(
+                rustfs_utils::http::get_consistent_str(&completed.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,),
+                Some("v1")
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn complete_encrypted_multipart_clears_unproven_upload_etag_marker() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "legacy-encrypted-etag";
+        make_bucket_on_all(&disks, bucket).await;
+        let mut opts = ObjectOptions {
+            user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
+            ..Default::default()
+        };
+        insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+        // Model an older writer whose part metadata has no protected-format proof.
+        let (upload_id, parts) = stage_upload_with_create_opts(&set, bucket, "object", b"legacy part", &opts).await;
+        let completed = set
+            .clone()
+            .complete_multipart_upload(bucket, "object", &upload_id, parts, &ObjectOptions::default())
+            .await
+            .expect("legacy part can still complete");
+        let reread = set
+            .get_object_info(bucket, "object", &ObjectOptions::default())
+            .await
+            .expect("read completed metadata");
+        assert_eq!(completed.etag, reread.etag);
+        for info in [completed, reread] {
+            assert!(!rustfs_utils::http::etag_can_be_listed(&info.user_defined));
+            assert!(!contains_key_str(&info.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG));
+        }
+    }
+
+    #[test]
+    fn data_movement_etag_proof_requires_verified_source_identity() {
+        let version = Uuid::new_v4();
+        let mut source = FileInfo {
+            version_id: Some(version),
+            metadata: HashMap::from([("etag".to_owned(), "protected-source-etag".to_owned())]),
+            ..Default::default()
+        };
+        insert_str(
+            &mut source.metadata,
+            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
+            "source-generation".to_owned(),
+        );
+        insert_str(&mut source.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+        let opts = ObjectOptions {
+            data_movement: true,
+            version_id: Some(version.to_string()),
+            preserve_etag: source.metadata.get("etag").cloned(),
+            ..Default::default()
+        };
+        assert!(preserves_staged_encrypted_etag_proof(&source, &opts, None));
+        for version_id in [None, Some(Uuid::nil())] {
+            let null_source = FileInfo {
+                version_id,
+                ..source.clone()
+            };
+            let null_opts = ObjectOptions {
+                version_id: Some(Uuid::nil().to_string()),
+                ..opts.clone()
+            };
+            assert!(preserves_staged_encrypted_etag_proof(&null_source, &null_opts, None));
+        }
+        for rejected in [
+            ObjectOptions {
+                data_movement: false,
+                ..opts.clone()
+            },
+            ObjectOptions {
+                replication_request: true,
+                ..opts.clone()
+            },
+            ObjectOptions {
+                version_id: None,
+                ..opts.clone()
+            },
+            ObjectOptions {
+                version_id: Some("invalid".to_owned()),
+                ..opts.clone()
+            },
+            ObjectOptions {
+                version_id: Some(Uuid::new_v4().to_string()),
+                ..opts.clone()
+            },
+            ObjectOptions {
+                preserve_etag: None,
+                ..opts.clone()
+            },
+            ObjectOptions {
+                preserve_etag: Some(String::new()),
+                ..opts.clone()
+            },
+            ObjectOptions {
+                preserve_etag: Some("replacement-etag".to_owned()),
+                ..opts.clone()
+            },
+        ] {
+            assert!(!preserves_staged_encrypted_etag_proof(&source, &rejected, None));
+        }
+        for suffix in [
+            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
+            rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
+        ] {
+            let mut missing = source.clone();
+            rustfs_utils::http::remove_str(&mut missing.metadata, suffix);
+            assert!(!preserves_staged_encrypted_etag_proof(&missing, &opts, None));
+            insert_str(&mut missing.metadata, suffix, String::new());
+            assert!(!preserves_staged_encrypted_etag_proof(&missing, &opts, None));
+            let mut conflicting = source.clone();
+            conflicting
+                .metadata
+                .insert(format!("x-minio-internal-{suffix}"), "conflict".to_owned());
+            assert!(!preserves_staged_encrypted_etag_proof(&conflicting, &opts, None));
+        }
+        for value in [None, Some("unknown")] {
+            let mut unproven = source.clone();
+            rustfs_utils::http::remove_str(&mut unproven.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
+            if let Some(value) = value {
+                insert_str(&mut unproven.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, value.to_owned());
+            }
+            let request_claim = ObjectOptions {
+                user_defined: source.metadata.clone(),
+                ..opts.clone()
+            };
+            assert!(!preserves_staged_encrypted_etag_proof(&unproven, &request_claim, None));
+        }
+    }
+
+    #[test]
+    fn restore_etag_proof_requires_matching_staged_operation() {
+        let operation_id = Uuid::new_v4();
+        for version_id in [None, Some(Uuid::new_v4())] {
+            let mut staged = FileInfo {
+                version_id,
+                metadata: HashMap::from([("etag".to_owned(), "source-etag".to_owned())]),
+                ..Default::default()
+            };
+            let opts = ObjectOptions {
+                version_id: version_id.map(|id| id.to_string()),
+                preserve_etag: Some("source-etag".to_owned()),
+                ..Default::default()
+            };
+            insert_str(&mut staged.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+            assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
+            insert_str(&mut staged.metadata, SUFFIX_RESTORE_OPERATION_ID, operation_id.to_string());
+            assert!(preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
+            assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(Uuid::new_v4())));
+            rustfs_utils::http::remove_str(&mut staged.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG);
+            assert!(!preserves_staged_encrypted_etag_proof(&staged, &opts, Some(operation_id)));
+        }
+    }
+
+    #[tokio::test]
+    async fn complete_data_movement_etag_proof_rejects_version_rewrites_and_late_claims() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "movement-etag-proof";
+        make_bucket_on_all(&disks, bucket).await;
+        for (object, staged_proof, publish_versioned, late_claim, expected_proof) in [
+            ("same-version", true, true, false, true),
+            ("rewritten-version", true, false, false, false),
+            ("late-claim", false, true, true, false),
+        ] {
+            let version = Uuid::new_v4();
+            let mut opts = ObjectOptions {
+                data_movement: true,
+                versioned: true,
+                version_id: Some(version.to_string()),
+                preserve_etag: Some("protected-source-etag".to_owned()),
+                user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_owned(), "AES256".to_owned())]),
+                ..Default::default()
+            };
+            insert_str(
+                &mut opts.user_defined,
+                rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
+                "source-generation".to_owned(),
+            );
+            if staged_proof {
+                insert_str(&mut opts.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+            }
+            let (upload_id, parts) = stage_upload_with_create_opts(&set, bucket, object, b"raw source bytes", &opts).await;
+            opts.versioned = publish_versioned;
+            if late_claim {
+                let mut metadata = HashMap::new();
+                insert_str(&mut metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG, "v1".to_owned());
+                opts.eval_metadata = Some(metadata);
+            }
+            let completed = set
+                .clone()
+                .complete_multipart_upload(bucket, object, &upload_id, parts, &opts)
+                .await
+                .expect("complete raw internal movement");
+            assert_eq!(completed.version_id, publish_versioned.then_some(version), "{object}");
+            assert_eq!(completed.etag, opts.preserve_etag, "{object}");
+            let persisted = set
+                .get_object_info(
+                    bucket,
+                    object,
+                    &ObjectOptions {
+                        version_id: Some(crate::data_movement::data_movement_exact_version_id(completed.version_id)),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("read completed proof from disk");
+            assert_eq!(
+                rustfs_utils::http::etag_can_be_listed(&persisted.user_defined),
+                expected_proof,
+                "{object}"
+            );
+            assert_eq!(
+                get_consistent_str(&persisted.user_defined, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+                expected_proof.then_some("v1"),
+                "{object}"
+            );
+        }
     }
 
     #[tokio::test]
@@ -16183,7 +16498,7 @@ mod tests {
             preserve_etag: Some(foreign_etag.clone()),
             ..Default::default()
         };
-        assert_eq!(resolve_complete_etag(&opts, &[]), foreign_etag);
+        assert_eq!(resolve_complete_etag(&opts, &[], &[]), (foreign_etag, false));
 
         // Empty preserve value degrades to the next source.
         let opts_empty = ObjectOptions {
@@ -16191,11 +16506,67 @@ mod tests {
             user_defined: std::collections::HashMap::from([("etag".to_string(), "override-etag".to_string())]),
             ..Default::default()
         };
-        assert_eq!(resolve_complete_etag(&opts_empty, &[]), "override-etag");
+        assert_eq!(resolve_complete_etag(&opts_empty, &[], &[]), ("override-etag".to_string(), false));
 
         // Without either source the ETag is computed from the parts.
-        let computed = resolve_complete_etag(&ObjectOptions::default(), &[]);
-        assert_eq!(computed, get_complete_multipart_md5(&[]));
+        let computed = resolve_complete_etag(&ObjectOptions::default(), &[], &[]);
+        assert_eq!(computed, (get_complete_multipart_md5(&[]), false));
+    }
+
+    #[test]
+    fn complete_etag_proof_requires_all_selected_parts_and_matching_overrides() {
+        for proofs in [&[][..], &[true], &[true, true], &[true, false], &[false]] {
+            let stored: Vec<_> = proofs
+                .iter()
+                .enumerate()
+                .map(|(index, &encrypted_etag)| ObjectPartInfo {
+                    number: index + 1,
+                    etag: "0123456789abcdef0123456789abcdef".to_owned(),
+                    encrypted_etag,
+                    ..Default::default()
+                })
+                .collect();
+            let uploaded: Vec<_> = stored
+                .iter()
+                .map(|part| CompletePart {
+                    part_num: part.number,
+                    etag: Some(part.etag.clone()),
+                    ..Default::default()
+                })
+                .collect();
+            let computed = get_complete_multipart_md5(&uploaded);
+            let protected = !proofs.is_empty() && proofs.iter().all(|&proof| proof);
+            assert_eq!(
+                resolve_complete_etag(&ObjectOptions::default(), &uploaded, &stored),
+                (computed.clone(), protected)
+            );
+
+            for preserved in ["foreign-etag", computed.as_str(), ""] {
+                for opts in [
+                    ObjectOptions {
+                        preserve_etag: Some(preserved.to_owned()),
+                        ..Default::default()
+                    },
+                    ObjectOptions {
+                        user_defined: HashMap::from([("etag".to_owned(), preserved.to_owned())]),
+                        ..Default::default()
+                    },
+                    ObjectOptions {
+                        preserve_etag: Some(preserved.to_owned()),
+                        user_defined: HashMap::from([("etag".to_owned(), "metadata-etag".to_owned())]),
+                        ..Default::default()
+                    },
+                ] {
+                    let expected_etag = if !preserved.is_empty() || opts.preserve_etag.is_none() {
+                        preserved
+                    } else {
+                        opts.user_defined.get("etag").map_or(computed.as_str(), String::as_str)
+                    };
+                    let result = resolve_complete_etag(&opts, &uploaded, &stored);
+                    assert_eq!(result, (expected_etag.to_owned(), protected && expected_etag == computed));
+                }
+            }
+        }
     }
 
     /// Accumulate the per-part digests and the running full-object checksum the

crates/ecstore/src/set_disk/ops/multipart_quorum_tests.rs

@@ -0,0 +1,303 @@
+//! Exercise participant preservation through real multipart staging and commit.
+
+use super::*;
+use crate::disk::FileWriter;
+use crate::erasure::coding::CustomWriter;
+use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
+use crate::storage_api_contracts::object::ObjectIO as _;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll, Waker};
+use tokio::io::AsyncWrite;
+
+#[derive(Clone, Copy)]
+enum Fault {
+    DelayedShutdown,
+    WriteFailure,
+}
+
+struct Round {
+    fault: Fault,
+    legacy_policy: bool,
+    failed_slot: usize,
+    completed: AtomicUsize,
+    delayed_observed: AtomicBool,
+    delayed_waker: Mutex<Option<Waker>>,
+    delayed_dropped: AtomicBool,
+}
+
+type RegisteredWriter = (Arc<Round>, usize);
+static WRITER_ROUNDS: OnceLock<Mutex<HashMap<String, RegisteredWriter>>> = OnceLock::new();
+
+struct WriterRound {
+    endpoints: Vec<String>,
+    round: Arc<Round>,
+}
+
+impl WriterRound {
+    fn install(disks: &[DiskStore], failed_slot: usize, fault: Fault, legacy_policy: bool) -> Self {
+        let round = Arc::new(Round {
+            fault,
+            legacy_policy,
+            failed_slot,
+            completed: AtomicUsize::new(0),
+            delayed_observed: AtomicBool::new(false),
+            delayed_waker: Mutex::new(None),
+            delayed_dropped: AtomicBool::new(false),
+        });
+        let endpoints = disks.iter().map(|disk| disk.endpoint().to_string()).collect::<Vec<_>>();
+        let mut registry = WRITER_ROUNDS
+            .get_or_init(Mutex::default)
+            .lock()
+            .expect("writer hook registry");
+        for (slot, endpoint) in endpoints.iter().enumerate() {
+            assert!(registry.insert(endpoint.clone(), (round.clone(), slot)).is_none());
+        }
+        Self { endpoints, round }
+    }
+}
+
+pub(super) fn write_policy(
+    disks: &[Option<DiskStore>],
+    quorum: usize,
+) -> Option<crate::erasure::coding::encode::ShardWritePolicy> {
+    let registry = WRITER_ROUNDS.get()?.lock().expect("writer hook registry");
+    disks.iter().flatten().find_map(|disk| {
+        registry.get(&disk.endpoint().to_string()).and_then(|(round, _)| {
+            round
+                .legacy_policy
+                .then(|| crate::erasure::coding::encode::ShardWritePolicy::object(quorum))
+        })
+    })
+}
+
+impl Drop for WriterRound {
+    fn drop(&mut self) {
+        let mut registry = WRITER_ROUNDS
+            .get()
+            .expect("installed writer registry")
+            .lock()
+            .expect("writer hook registry");
+        for endpoint in &self.endpoints {
+            registry.remove(endpoint);
+        }
+    }
+}
+
+pub(super) async fn create_writer(
+    disk: &DiskStore,
+    path: &str,
+    length: i64,
+    shard_size: usize,
+) -> Option<disk::error::Result<BitrotWriterWrapper>> {
+    let registered = WRITER_ROUNDS
+        .get()?
+        .lock()
+        .expect("writer hook registry")
+        .get(&disk.endpoint().to_string())
+        .cloned();
+    let (round, slot) = registered?;
+    let length = usize::try_from(length).expect("the fixture writes known-size parts");
+    let encoded_length = coding::bitrot_shard_file_size(length, shard_size, HashAlgorithm::HighwayHash256S);
+    let inner = match disk
+        .create_file(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, path, encoded_length as i64)
+        .await
+    {
+        Ok(inner) => inner,
+        Err(error) => return Some(Err(error)),
+    };
+    let writer = ControlledWriter {
+        inner,
+        round,
+        slot,
+        delay: None,
+        completed: false,
+    };
+    Some(Ok(BitrotWriterWrapper::new(
+        CustomWriter::new_tokio_writer(Box::new(writer)),
+        shard_size,
+        HashAlgorithm::HighwayHash256S,
+    )))
+}
+
+struct ControlledWriter {
+    inner: FileWriter,
+    round: Arc<Round>,
+    slot: usize,
+    delay: Option<Pin<Box<tokio::time::Sleep>>>,
+    completed: bool,
+}
+
+impl Drop for ControlledWriter {
+    fn drop(&mut self) {
+        if self.slot == self.round.failed_slot && !self.completed {
+            self.round.delayed_dropped.store(true, Ordering::Release);
+        }
+    }
+}
+
+impl AsyncWrite for ControlledWriter {
+    fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<io::Result<usize>> {
+        if self.slot == self.round.failed_slot && matches!(self.round.fault, Fault::WriteFailure) {
+            return Poll::Ready(Err(io::Error::new(io::ErrorKind::BrokenPipe, "injected shard write failure")));
+        }
+        Pin::new(&mut self.inner).poll_write(cx, bytes)
+    }
+
+    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        Pin::new(&mut self.inner).poll_flush(cx)
+    }
+
+    fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        if self.slot == self.round.failed_slot && matches!(self.round.fault, Fault::DelayedShutdown) {
+            self.round.delayed_observed.store(true, Ordering::Release);
+            // Delay only after three real writers finish. This ensures the old
+            // post-quorum grace expires regardless of filesystem scheduling.
+            {
+                let mut waker = self.round.delayed_waker.lock().expect("delayed writer waker");
+                *waker = Some(cx.waker().clone());
+                if self.round.completed.load(Ordering::Acquire) < 3 {
+                    return Poll::Pending;
+                }
+            }
+            let delay = self
+                .delay
+                .get_or_insert_with(|| Box::pin(tokio::time::sleep(Duration::from_secs(2))));
+            if delay.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
+            }
+        }
+        let result = Pin::new(&mut self.inner).poll_shutdown(cx);
+        if matches!(result, Poll::Ready(Ok(()))) && !self.completed {
+            self.completed = true;
+            self.round.completed.fetch_add(1, Ordering::AcqRel);
+            if let Some(waker) = self.round.delayed_waker.lock().expect("delayed writer waker").take() {
+                waker.wake();
+            }
+        }
+        result
+    }
+}
+
+async fn upload_two_parts(
+    set: &Arc<SetDisks>,
+    disks: &[DiskStore],
+    bucket: &str,
+    object: &str,
+    fault: Fault,
+    legacy_policy: bool,
+) -> (String, Vec<CompletePart>, Vec<u8>, FileInfo) {
+    for disk in disks {
+        disk.make_volume(bucket).await.expect("bucket volume");
+    }
+    let opts = ObjectOptions::default();
+    let upload = set
+        .new_multipart_upload(bucket, object, &opts)
+        .await
+        .expect("new multipart upload");
+    let mut parts = Vec::new();
+    let mut expected = Vec::new();
+    for (number, size, slot) in [(1, 5 * 1024 * 1024, 2), (2, 4096, 3)] {
+        let payload = vec![number as u8; size];
+        expected.extend_from_slice(&payload);
+        let hook = WriterRound::install(disks, slot, fault, legacy_policy);
+        let mut reader = PutObjReader::from_vec(payload);
+        let part = set
+            .put_object_part(bucket, object, &upload.upload_id, number, &mut reader, &opts)
+            .await
+            .expect("three completed shards must admit each part");
+        match fault {
+            Fault::DelayedShutdown if !legacy_policy => {
+                assert!(hook.round.delayed_observed.load(Ordering::Acquire));
+                assert!(!hook.round.delayed_dropped.load(Ordering::Acquire), "finite slow shard must finish");
+                assert_eq!(hook.round.completed.load(Ordering::Acquire), 4, "all real shard writers finished");
+            }
+            Fault::DelayedShutdown | Fault::WriteFailure => {
+                assert!(hook.round.delayed_dropped.load(Ordering::Acquire), "failed shard must be dropped");
+                assert_eq!(hook.round.completed.load(Ordering::Acquire), 3);
+            }
+        }
+        parts.push(CompletePart {
+            part_num: number,
+            etag: part.etag,
+            ..Default::default()
+        });
+    }
+    let (metadata, _) = set
+        .check_upload_id_exists(bucket, object, &upload.upload_id, true)
+        .await
+        .expect("upload metadata");
+    (upload.upload_id, parts, expected, metadata)
+}
+
+#[tokio::test]
+async fn multipart_budget_keeps_different_finite_shutdown_stragglers_and_completes() {
+    let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+    let bucket = "multipart-finite-shutdown";
+    let object = "object";
+    let (upload, parts, expected, metadata) = upload_two_parts(&set, &disks, bucket, object, Fault::DelayedShutdown, false).await;
+    let data_dir = metadata.data_dir.expect("parts-backed data directory");
+    let erasure =
+        coding::Erasure::try_new(metadata.erasure.data_blocks, metadata.erasure.parity_blocks, metadata.erasure.block_size)
+            .expect("upload erasure layout");
+    let stage = SetDisks::get_upload_id_dir(bucket, object, &upload);
+    for disk in &disks {
+        for (number, size) in [(1, 5 * 1024 * 1024), (2, 4096)] {
+            let path = format!("{stage}/{data_dir}/part.{number}");
+            let bytes = disk
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
+                .await
+                .expect("every disk has every part");
+            let shard_len = erasure.shard_file_size(size);
+            let encoded_len =
+                coding::bitrot_shard_file_size(shard_len as usize, erasure.shard_size(), HashAlgorithm::HighwayHash256S);
+            assert_eq!(bytes.len(), encoded_len, "complete physical shard: {path}");
+        }
+    }
+    set.clone()
+        .complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
+        .await
+        .expect("all parts have a common completed quorum");
+    let mut reader = set
+        .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
+        .await
+        .expect("read committed object");
+    let mut actual = Vec::new();
+    reader.stream.read_to_end(&mut actual).await.expect("whole object readback");
+    assert_eq!(actual, expected);
+}
+
+#[tokio::test]
+async fn multipart_budget_distinct_failed_shards_do_not_bypass_complete_quorum() {
+    let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+    let bucket = "multipart-distinct-failures";
+    let object = "object";
+    let (upload, parts, _, _) = upload_two_parts(&set, &disks, bucket, object, Fault::WriteFailure, false).await;
+    let error = set
+        .clone()
+        .complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
+        .await
+        .expect_err("two common complete disks cannot meet the write quorum of three");
+    assert!(error.is_quorum_error(), "expected complete quorum refusal, got {error:?}");
+    set.check_upload_id_exists(bucket, object, &upload, false)
+        .await
+        .expect("failed completion retains retryable upload state");
+    assert!(
+        set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
+        "failed completion must not publish an object"
+    );
+}
+
+#[tokio::test]
+async fn multipart_budget_old_grace_loses_common_part_quorum() {
+    let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+    let bucket = "multipart-old-grace";
+    let object = "object";
+    let (upload, parts, _, _) = upload_two_parts(&set, &disks, bucket, object, Fault::DelayedShutdown, true).await;
+    let error = set
+        .clone()
+        .complete_multipart_upload(bucket, object, &upload, parts, &ObjectOptions::default())
+        .await
+        .expect_err("the old per-part grace loses the common completion quorum");
+    assert!(error.is_quorum_error(), "expected complete quorum refusal, got {error:?}");
+}

crates/ecstore/src/set_disk/ops/object.rs

@@ -4562,7 +4562,14 @@ impl SetDisks {
                 SmallWritePath::Pipeline => IntegrityEncodeMode::Streaming,
             };
             let (reader, w_size, inline_shards, integrity) = Arc::clone(&erasure)
-                .encode_with_shard_integrity(stream, &mut writers, write_quorum, 1, mode, protect_write)
+                .encode_with_shard_integrity(
+                    stream,
+                    &mut writers,
+                    coding::encode::ShardWritePolicy::object(write_quorum),
+                    1,
+                    mode,
+                    protect_write,
+                )
                 .await?;
             let encode_elapsed = encode_stage_start.map(|stage_start| stage_start.elapsed());
             let encode_ms = encode_elapsed.map(|elapsed| elapsed.as_millis() as u64).unwrap_or_default();
@@ -9096,6 +9103,9 @@ impl SetDisks {
         let mut futures = Vec::with_capacity(disks.len());
         let mut errs = Vec::with_capacity(disks.len());
 
+        let requires_bounded_delete_recovery = commit_context
+            .options
+            .is_some_and(|options| options.delete_recovery_work_budget.is_some());
         for disk in disks.iter() {
             let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
             let transaction_guard = commit_context.transaction_guard.as_ref();
@@ -9107,6 +9117,7 @@ impl SetDisks {
                     let options = DeleteOptions {
                         old_data_dir: Some(rollback_dir),
                         delete_recovery_bucket_incarnation: source_bucket_incarnation_id,
+                        requires_bounded_delete_recovery,
                         ..Default::default()
                     };
                     let result = if is_meta_bucketname(bucket) {
@@ -10550,6 +10561,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             .collect();
         let mut futures = Vec::with_capacity(disks.len());
         let lock_lost_during_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let requires_bounded_delete_recovery = opts.delete_recovery_work_budget.is_some();
 
         // let mut errors = Vec::with_capacity(disks.len());
 
@@ -10567,6 +10579,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     let options = DeleteOptions {
                         old_data_dir: Some(rollback_dir),
                         delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id,
+                        requires_bounded_delete_recovery,
                         ..Default::default()
                     };
                     if current_generations
@@ -17166,6 +17179,9 @@ mod transition_commit_failure_tests {
     #[tokio::test]
     #[serial_test::serial]
     async fn multipart_restore_copies_the_stored_representation_back_verbatim() {
+        use crate::io_support::rio::{WriteEncryption, WritePlan};
+        use rustfs_utils::http::{SUFFIX_OPAQUE_ENCRYPTED_ETAG, etag_can_be_listed};
+
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
         let bucket = "restore-multipart-stored-coordinates-bucket";
         let object = "object.bin";
@@ -17173,9 +17189,8 @@ mod transition_commit_failure_tests {
             disk.make_volume(bucket).await.expect("bucket volume should be created");
         }
 
-        // The minimum-part-size gate reads the PLAINTEXT length, so part one
-        // clears 5 MiB there while its stored form carries encoding overhead.
-        let part_shapes = [(6 * 1024 * 1024_usize, 9_216_usize), (256 * 1024_usize, 512_usize)];
+        // The first plaintext part clears the non-final 5 MiB threshold.
+        let part_sizes = [6 * 1024 * 1024_usize, 256 * 1024_usize];
         let mut user_defined = HashMap::new();
         user_defined.insert(rustfs_utils::http::INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256".to_string());
         user_defined.insert(
@@ -17196,20 +17211,22 @@ mod transition_commit_failure_tests {
             .expect("multipart upload should be created");
 
         let mut uploaded_parts = Vec::new();
-        let mut expected_stored = Vec::new();
-        for (index, (plaintext_len, overhead)) in part_shapes.iter().enumerate() {
-            let stored = stored_representation_bytes(index as u32 * 7 + 1, plaintext_len + overhead);
-            expected_stored.extend_from_slice(&stored);
-            let stored_len = stored.len() as i64;
-            let hash_reader =
-                HashReader::from_stream(std::io::Cursor::new(stored), stored_len, *plaintext_len as i64, None, None, false)
-                    .expect("hash reader over the stored representation");
-            let mut reader = PutObjReader::new(hash_reader);
+        for (index, &part_size) in part_sizes.iter().enumerate() {
+            let plaintext = stored_representation_bytes(index as u32 * 7 + 1, part_size);
+            let actual_size = part_size as i64;
+            let plaintext_reader =
+                HashReader::from_stream(std::io::Cursor::new(plaintext), actual_size, actual_size, None, None, false)
+                    .expect("plaintext part reader");
+            let encrypted_reader = WritePlan::new()
+                .with_encryption(WriteEncryption::multipart([7; 32], [9; 12], index + 1))
+                .apply(plaintext_reader, actual_size)
+                .expect("native protected encrypted writer");
+            let mut reader = PutObjReader::new(encrypted_reader);
             let info = set_disks
                 .put_object_part(bucket, object, &upload.upload_id, index + 1, &mut reader, &ObjectOptions::default())
                 .await
                 .expect("stored part should be staged");
-            assert_eq!(info.size as i64, stored_len, "a part is stored in its encoded length");
+            assert!(info.size as i64 > actual_size, "part contains actual encryption overhead");
             uploaded_parts.push(CompletePart {
                 part_num: info.part_num,
                 etag: info.etag,
@@ -17233,8 +17250,12 @@ mod transition_commit_failure_tests {
                 "the fixture must keep the two coordinate systems apart: stored {size} vs plaintext {actual_size}"
             );
         }
-        let stored_before = read_stored_representation(&set_disks, bucket, object).await;
-        assert_stored_representation_eq(&stored_before, &expected_stored, "the fixture must store its encoded bytes verbatim");
+        assert_eq!(
+            rustfs_utils::http::get_consistent_str(&original.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+            Some("v1")
+        );
+        assert!(etag_can_be_listed(&original.user_defined), "the source has a real producer-attested ETag");
+        let expected_stored = read_stored_representation(&set_disks, bucket, object).await;
 
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
         register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
@@ -17258,6 +17279,16 @@ mod transition_commit_failure_tests {
             .await
             .expect("multipart source should transition before restore");
 
+        let transitioned = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read metadata after actual tier transition");
+        assert_eq!(
+            rustfs_utils::http::get_consistent_str(&transitioned.user_defined, SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+            Some("v1")
+        );
+        assert!(etag_can_be_listed(&transitioned.user_defined), "transition retains the source proof");
+
         let operation_id = Uuid::new_v4();
         set_disks
             .put_object_metadata(
@@ -17303,6 +17334,11 @@ mod transition_commit_failure_tests {
         // backlog#2369 P7.1: the copy-back digests stored bytes, so the object's
         // public ETag has to be carried over rather than recomputed.
         assert_eq!(restored.etag, original.etag, "restore must preserve the object ETag");
+        assert_eq!(restored.version_id, original.version_id, "restore keeps the same version");
+        assert!(
+            etag_can_be_listed(&restored.user_defined),
+            "the restored version retains its protected ETag proof"
+        );
     }
 
     #[tokio::test]
@@ -20425,7 +20461,7 @@ mod transition_upload_integrity_tests {
         assert_local_source_intact(&set_disks, bucket, object, &payload).await;
     }
 
-    #[tokio::test(flavor = "multi_thread")]
+    #[tokio::test(flavor = "current_thread")]
     #[serial_test::serial]
     async fn commit_lock_acquire_failure_cleans_remote_candidate_and_preserves_source() {
         let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
@@ -20434,7 +20470,16 @@ mod transition_upload_integrity_tests {
         let bucket = "transition-lock-acquire-failure-bucket";
         let object = "object.bin";
         let payload = b"transition commit lock failure must clean the remote candidate".repeat(1024);
-        let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        let original = temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await },
+        )
+        .await;
+        // Complete source I/O before controlling the quorum retry deadline.
+        tokio::time::pause();
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
         let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
@@ -20446,13 +20491,8 @@ mod transition_upload_integrity_tests {
             .await
             .expect_err("transition must fail when the commit namespace write lock cannot reach quorum");
         assert!(
-            matches!(
-                error,
-                StorageError::NamespaceLockQuorumUnavailable { .. }
-                    | StorageError::Lock(LockError::QuorumNotReached { .. })
-                    | StorageError::Lock(LockError::Internal { .. })
-            ),
-            "unexpected transition lock-acquire error: {error:?}"
+            matches!(error, StorageError::NamespaceLockQuorumUnavailable { required: 2, .. }),
+            "the injected permanent quorum failure must reach the transition commit boundary: {error:?}"
         );
         assert_eq!(
             backend.put_count().await,

crates/ecstore/src/store/init.rs

@@ -24035,6 +24035,7 @@ mod tests {
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-record", &[4])).await;
         let mutation_id = uuid::Uuid::new_v4();
         let intent = TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -24220,6 +24221,7 @@ mod tests {
         let (_ctx, store, _shutdown) =
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan", &[4])).await;
         let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -24281,6 +24283,7 @@ mod tests {
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-advance", &[4])).await;
         let mutation_id = uuid::Uuid::new_v4();
         let intent = TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -24355,6 +24358,7 @@ mod tests {
 
         let abort_id = uuid::Uuid::new_v4();
         let abort_intent = TierMutationIntent {
+            requires_native_azure: false,
             mutation_id: abort_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -24397,6 +24401,7 @@ mod tests {
         candidate_digest: [u8; 32],
     ) -> TierMutationIntent {
         TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,
@@ -25473,6 +25478,7 @@ mod tests {
         let (_ctx, store, _shutdown) =
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-mutation-intent-scan-page", &[4])).await;
         let build_intent = |mutation_id: uuid::Uuid, tier_name: &str| TierMutationIntent {
+            requires_native_azure: false,
             mutation_id,
             revision: 1,
             kind: TierMutationIntentKind::Edit,

crates/ecstore/src/store/object.rs

@@ -3218,7 +3218,12 @@ impl ECStore {
     pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
         check_valid_bucket_name(bucket)?;
         let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
+        let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+            &self.ctx,
+            bucket,
+            "object_lock_admission_metadata_read",
+        )
+        .await?;
         let (state, bucket_incarnation_id, config_revision) =
             get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
         if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
@@ -3238,7 +3243,12 @@ impl ECStore {
     pub(crate) async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
         check_valid_bucket_name(bucket)?;
         let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
+        let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+            &self.ctx,
+            bucket,
+            "object_lock_snapshot_metadata_read",
+        )
+        .await?;
         let (state, bucket_incarnation_id, config_revision, quota_res) =
             get_object_lock_and_quota_config_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
         if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
@@ -3267,7 +3277,12 @@ impl ECStore {
         if lifecycle_fence.is_lock_lost() {
             return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
         }
-        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
+        let metadata_guard = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+            &self.ctx,
+            bucket,
+            "object_lock_completion_metadata_read",
+        )
+        .await?;
         let (state, bucket_incarnation_id, config_revision) =
             get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&metadata_guard)).await?;
         if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {

crates/ecstore/src/store/rebalance.rs

@@ -445,6 +445,13 @@ impl ECStore {
             return Ok(());
         }
 
+        let scoped_cleanup = opts
+            .scanner_publication_commit_scope
+            .as_ref()
+            .zip(opts.http_preconditions.as_ref())
+            .filter(|_| bucket == RUSTFS_META_BUCKET && opts.delete_prefix_object);
+        let _cleanup_outcome =
+            scoped_cleanup.map(|(scope, _)| crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone()));
         let mut first_error = None;
         let mut first_volume_error = None;
         let mut has_success = false;
@@ -457,9 +464,30 @@ impl ECStore {
             }
             let mut opts = opts.clone();
             opts.delete_prefix = true;
+            if let Some((scope, preconditions)) = scoped_cleanup {
+                // The exact-object delete retains its namespace writer across
+                // this lookup and dispatch. Never remove an unobserved revision.
+                let lookup = ObjectOptions {
+                    no_lock: true,
+                    suppress_read_repair: true,
+                    preserve_delete_marker: true,
+                    ..Default::default()
+                };
+                let disks = pool.get_disks_by_key(object);
+                let current = disks.get_object_info(bucket, object, &lookup).await;
+                if let Some(error) =
+                    crate::set_disk::SetDisks::evaluate_write_precondition(current, preconditions, bucket, object, &lookup)
+                {
+                    if matches!(error, StorageError::PreconditionFailed) || crate::error::is_err_object_not_found(&error) {
+                        continue;
+                    }
+                    return Err(error);
+                }
+                opts.scanner_publication_commit_scope = Some(scope.pool_delete_participant());
+            }
             match pool.delete_object(bucket, object, opts).await {
                 Ok(_) => has_success = true,
-                Err(err) if is_err_strict_volume_not_found(&err) => {
+                Err(err) if scoped_cleanup.is_none() && is_err_strict_volume_not_found(&err) => {
                     if first_volume_error.is_none() {
                         first_volume_error = Some(err);
                     }
@@ -472,14 +500,21 @@ impl ECStore {
             }
         }
 
-        match first_error {
+        let result = match first_error {
             Some(err) => Err(err),
             None if has_success => Ok(()),
             None => match first_volume_error {
                 Some(err) => Err(err),
+                None if scoped_cleanup.is_some() => Err(StorageError::PreconditionFailed),
                 None => Ok(()),
             },
+        };
+        if let Some((scope, _)) = scoped_cleanup
+            && result.is_ok()
+        {
+            let _ = scope.mark_committed();
         }
+        result
     }
 
     pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option<usize> {
@@ -1379,6 +1414,129 @@ mod tests {
         (temp_dir, store, shutdown)
     }
 
+    async fn run_scanner_observed_cleanup(
+        if_match_override: Option<&str>,
+        second_payload: Option<&[u8]>,
+        lose_second_quorum: bool,
+    ) -> (
+        Result<crate::object_api::ObjectInfo>,
+        crate::object_api::ScannerPublicationCommitState,
+        Vec<bool>,
+    ) {
+        use crate::config::com::save_config_with_opts_and_metadata;
+        use crate::storage_api_contracts::object::HTTPPreconditions;
+        use std::time::Duration;
+
+        let (_directories, store, shutdown) = setup_multi_pool_test_store("scanner-observed-cleanup", &[4, 4]).await;
+        let object = "buckets/.usage.observed.json";
+        let mut tags = Vec::new();
+        let old_payload = b"old observed snapshot";
+        for (pool, payload) in store.pools.iter().zip([Some(old_payload.as_slice()), second_payload]) {
+            let Some(payload) = payload else { continue };
+            let info = save_config_with_opts_and_metadata(pool.clone(), object, payload.to_vec(), &ObjectOptions::default())
+                .await
+                .expect("seed an observed snapshot on each pool");
+            tags.push(info.etag.expect("seed has CAS tag"));
+        }
+        if let Some(second_payload) = second_payload {
+            assert_eq!(tags[0] == tags[1], old_payload == second_payload);
+        }
+        let epoch = store.scanner_data_usage_publication_epoch().await.expect("publication epoch");
+        let scope = store
+            .scanner_data_usage_publication_commit_scope(epoch, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
+            .await
+            .expect("admit observed cleanup");
+        let disconnected = if lose_second_quorum {
+            let pool = &store.pools[1];
+            tokio::time::timeout(Duration::from_secs(10), pool.stop_endpoint_monitor_for_test())
+                .await
+                .expect("fixture must own the disk slots");
+            let set = pool.get_disks_by_key(object);
+            let original = {
+                let mut disks = set.disks.write().await;
+                let offline = vec![None; disks.len()];
+                std::mem::replace(&mut *disks, offline)
+            };
+            Some((set, original))
+        } else {
+            None
+        };
+        let result = store
+            .delete_object(
+                crate::disk::RUSTFS_META_BUCKET,
+                object,
+                ObjectOptions {
+                    delete_prefix: true,
+                    delete_prefix_object: true,
+                    http_preconditions: Some(HTTPPreconditions {
+                        if_match: Some(if_match_override.map_or_else(|| tags.remove(0), str::to_owned)),
+                        ..Default::default()
+                    }),
+                    scanner_publication_commit_scope: Some(scope.clone()),
+                    ..Default::default()
+                },
+            )
+            .await;
+        if let Some((set, disks)) = disconnected {
+            *set.disks.write().await = disks;
+            assert!(!scope.release_movement_permit().await, "partial cleanup cannot release its permit");
+        }
+        let mut still_present = Vec::new();
+        for pool in &store.pools {
+            still_present.push(
+                read_config_no_lock_preserve_empty_with_metadata(pool.clone(), object)
+                    .await
+                    .is_ok(),
+            );
+        }
+        shutdown.cancel();
+        (result, scope.state(), still_present)
+    }
+
+    #[tokio::test]
+    async fn scanner_observed_cleanup_must_keep_scope_live_across_both_pools() {
+        let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"old observed snapshot"), false).await;
+        assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
+        assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
+        assert_eq!(still_present, vec![false, false]);
+    }
+
+    #[tokio::test]
+    async fn scanner_observed_cleanup_must_preserve_both_pools_if_cas_changed() {
+        let (result, state, still_present) =
+            run_scanner_observed_cleanup(Some("stale-cas-token"), Some(b"old observed snapshot"), false).await;
+        assert!(
+            matches!(result, Err(StorageError::PreconditionFailed))
+                && state == crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
+                && still_present == [true, true],
+            "result {result:?}, scope {state:?}, pool copies {still_present:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn scanner_observed_cleanup_must_preserve_a_nonmatching_pool_copy() {
+        let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"newer observed snapshot"), false).await;
+        assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
+        assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
+        assert_eq!(still_present, vec![false, true]);
+    }
+
+    #[tokio::test]
+    async fn scanner_observed_cleanup_must_succeed_when_another_pool_has_no_copy() {
+        let (result, state, still_present) = run_scanner_observed_cleanup(None, None, false).await;
+        assert!(result.is_ok(), "result {result:?}, scope {state:?}, pool copies {still_present:?}");
+        assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Committed);
+        assert_eq!(still_present, vec![false, false]);
+    }
+
+    #[tokio::test]
+    async fn scanner_observed_cleanup_must_retain_ownership_after_partial_failure() {
+        let (result, state, still_present) = run_scanner_observed_cleanup(None, Some(b"old observed snapshot"), true).await;
+        assert!(result.is_err(), "partial cleanup must fail");
+        assert_eq!(state, crate::object_api::ScannerPublicationCommitState::Indeterminate);
+        assert_eq!(still_present, vec![false, true]);
+    }
+
     struct LifecycleDeleteAllFailureGuard;
 
     impl Drop for LifecycleDeleteAllFailureGuard {

crates/filemeta/src/fileinfo.rs

@@ -70,13 +70,17 @@ pub struct ObjectPartInfo {
     pub error: Option<String>,
     #[serde(default)]
     pub integrity: Option<crate::shard_integrity::PartIntegrity>,
+    /// The stored ETag was generated by the protected encrypted writer.
+    #[serde(default)]
+    pub encrypted_etag: bool,
 }
 
 impl Serialize for ObjectPartInfo {
     fn serialize<S: serde::Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
         // Named fields let release readers ignore optional extensions. Appending
         // a ninth element to the old positional array makes those readers fail.
-        let mut map = serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some())))?;
+        let mut map =
+            serializer.serialize_map(Some(8 + usize::from(self.integrity.is_some()) + usize::from(self.encrypted_etag)))?;
         map.serialize_entry("etag", &self.etag)?;
         map.serialize_entry("number", &self.number)?;
         map.serialize_entry("size", &self.size)?;
@@ -88,6 +92,9 @@ impl Serialize for ObjectPartInfo {
         if let Some(integrity) = &self.integrity {
             map.serialize_entry("integrity", integrity)?;
         }
+        if self.encrypted_etag {
+            map.serialize_entry("encrypted_etag", &true)?;
+        }
         map.end()
     }
 }
@@ -1143,6 +1150,7 @@ impl FileInfo {
             checksums,
             error: None,
             integrity: None,
+            encrypted_etag: false,
         };
 
         for p in self.parts.iter_mut() {
@@ -2231,6 +2239,59 @@ mod tests {
             )
     }
 
+    #[test]
+    fn object_part_encrypted_etag_round_trips_with_legacy_compatibility() {
+        #[derive(Serialize, Deserialize)]
+        struct LegacyPart {
+            etag: String,
+            number: usize,
+            size: usize,
+            actual_size: i64,
+            mod_time: Option<OffsetDateTime>,
+            index: Option<Bytes>,
+            checksums: Option<HashMap<String, String>>,
+            error: Option<String>,
+        }
+
+        let legacy = LegacyPart {
+            etag: "0123456789abcdef0123456789abcdef".to_owned(),
+            number: 1,
+            size: 42,
+            actual_size: 42,
+            mod_time: None,
+            index: None,
+            checksums: None,
+            error: None,
+        };
+        for encoded in [
+            rmp_serde::to_vec(&legacy).expect("legacy positional part encodes"),
+            rmp_serde::to_vec_named(&legacy).expect("legacy named part encodes"),
+        ] {
+            let decoded = ObjectPartInfo::unmarshal(&encoded).expect("legacy part remains readable");
+            assert_eq!(decoded.etag, legacy.etag);
+            assert!(!decoded.encrypted_etag, "an older writer cannot attest a protected ETag");
+        }
+
+        for encrypted_etag in [false, true] {
+            let current = ObjectPartInfo {
+                etag: legacy.etag.clone(),
+                number: legacy.number,
+                size: legacy.size,
+                actual_size: legacy.actual_size,
+                encrypted_etag,
+                ..Default::default()
+            };
+            let encoded = current.marshal_msg().expect("current part encodes as a map");
+            let field_count = rmp::decode::read_map_len(&mut encoded.as_slice()).expect("part format is a map");
+            assert_eq!(field_count, if encrypted_etag { 9 } else { 8 });
+            assert_eq!(ObjectPartInfo::unmarshal(&encoded).expect("proof round trips"), current);
+            let old_reader: LegacyPart = rmp_serde::from_slice(&encoded).expect("old readers ignore optional proof");
+            assert_eq!(old_reader.etag, current.etag);
+            assert_eq!(old_reader.number, current.number);
+            assert_eq!(old_reader.size, current.size);
+        }
+    }
+
     fn object_part_info_strategy() -> impl Strategy<Value = ObjectPartInfo> {
         (
             small_string_strategy(),
@@ -2252,6 +2313,7 @@ mod tests {
                 checksums,
                 error,
                 integrity: None,
+                encrypted_etag: false,
             })
     }
 
@@ -2461,6 +2523,7 @@ mod tests {
                 ),
                 error: Some("part-error".to_string()),
                 integrity: None,
+                encrypted_etag: false,
             }],
             erasure: ErasureInfo {
                 algorithm: "erasure-algorithm".to_string(),

crates/filemeta/src/filemeta/version.rs

@@ -1346,6 +1346,7 @@ impl<'a> BorrowedMetaObject<'a> {
                             checksums: None,
                             error: None,
                             integrity: None,
+                            encrypted_etag: false,
                         });
                     }
                 }
@@ -2625,6 +2626,7 @@ impl From<MetaObjectV1Part> for ObjectPartInfo {
             checksums: value.checksums,
             error: value.error,
             integrity: None,
+            encrypted_etag: false,
         }
     }
 }

crates/heal-contracts/Cargo.toml

@@ -30,7 +30,7 @@ workspace = true
 [dependencies]
 serde = { workspace = true, features = ["derive"] }
 tokio = { workspace = true, features = ["sync"] }
-uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics", "serde"] }
 
 [dev-dependencies]
 serde_json = { workspace = true }

crates/io-metrics/src/internode_metrics.rs

@@ -429,6 +429,7 @@ const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network
 const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
 const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
 const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
+const INTERNODE_OPERATION_SERVER_DURATION_MS: &str = "rustfs_system_network_internode_operation_server_duration_ms";
 const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
 const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
 const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
@@ -562,6 +563,10 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
         name: INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL,
         labels: SERVER_OPERATION_BACKEND_LABELS,
     },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_SERVER_DURATION_MS,
+        labels: SERVER_OPERATION_BACKEND_LABELS,
+    },
 ];
 
 static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
@@ -687,6 +692,78 @@ impl GrpcReadVersionMetricHandles {
 static GRPC_READ_VERSION_METRIC_HANDLES: LazyLock<GrpcReadVersionMetricHandles> =
     LazyLock::new(|| GrpcReadVersionMetricHandles::new(current_server_label()));
 
+#[cfg(not(test))]
+static GRPC_READ_MULTIPLE_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
+    operation_duration_histogram(
+        INTERNODE_OPERATION_DURATION_MS,
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    )
+});
+
+#[cfg(not(test))]
+static GRPC_BATCH_READ_VERSION_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
+    operation_duration_histogram(
+        INTERNODE_OPERATION_DURATION_MS,
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    )
+});
+
+#[cfg(not(test))]
+static GRPC_READ_MULTIPLE_SERVER_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
+    operation_duration_histogram(
+        INTERNODE_OPERATION_SERVER_DURATION_MS,
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    )
+});
+
+#[cfg(not(test))]
+static GRPC_BATCH_READ_VERSION_SERVER_DURATION: LazyLock<metrics::Histogram> = LazyLock::new(|| {
+    operation_duration_histogram(
+        INTERNODE_OPERATION_SERVER_DURATION_MS,
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    )
+});
+
+#[cfg(not(test))]
+fn grpc_operation_duration_if_ready(operation: &'static str, backend: &'static str) -> Option<&'static metrics::Histogram> {
+    STABLE_SERVER_LABEL.get()?;
+    if backend != INTERNODE_TRANSPORT_BACKEND_GRPC {
+        return None;
+    }
+    match operation {
+        INTERNODE_OPERATION_GRPC_READ_VERSION => Some(&GRPC_READ_VERSION_METRIC_HANDLES.duration),
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE => Some(&GRPC_READ_MULTIPLE_DURATION),
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION => Some(&GRPC_BATCH_READ_VERSION_DURATION),
+        _ => None,
+    }
+}
+
+#[cfg(not(test))]
+fn grpc_server_duration_if_ready(operation: &'static str, backend: &'static str) -> Option<&'static metrics::Histogram> {
+    STABLE_SERVER_LABEL.get()?;
+    if backend != INTERNODE_TRANSPORT_BACKEND_GRPC {
+        return None;
+    }
+    match operation {
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE => Some(&GRPC_READ_MULTIPLE_SERVER_DURATION),
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION => Some(&GRPC_BATCH_READ_VERSION_SERVER_DURATION),
+        _ => None,
+    }
+}
+
+fn operation_duration_histogram(name: &'static str, operation: &'static str, backend: &'static str) -> metrics::Histogram {
+    metrics::histogram!(
+        name,
+        SERVER_LABEL => current_server_label(),
+        OPERATION_LABEL => operation,
+        BACKEND_LABEL => backend
+    )
+}
+
 #[cfg(not(test))]
 fn server_metric_handles_if_ready() -> Option<&'static InternodeServerMetricHandles> {
     STABLE_SERVER_LABEL.get()?;
@@ -944,17 +1021,28 @@ impl InternodeMetrics {
     pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
         let duration_ms = duration.as_secs_f64() * 1000.0;
         #[cfg(not(test))]
-        if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
-            handles.duration.record(duration_ms);
+        if let Some(histogram) = grpc_operation_duration_if_ready(operation, backend) {
+            histogram.record(duration_ms);
             return;
         }
-        metrics::histogram!(
-            INTERNODE_OPERATION_DURATION_MS,
-            SERVER_LABEL => current_server_label(),
-            OPERATION_LABEL => operation,
-            BACKEND_LABEL => backend
-        )
-        .record(duration_ms);
+        operation_duration_histogram(INTERNODE_OPERATION_DURATION_MS, operation, backend).record(duration_ms);
+    }
+
+    /// Records server-side handler latency in a separate histogram from client RPC duration.
+    /// The caller supplies the handler lifetime, including any error or cancellation path.
+    pub fn record_server_duration_for_operation_and_backend(
+        &self,
+        operation: &'static str,
+        backend: &'static str,
+        duration: Duration,
+    ) {
+        let duration_ms = duration.as_secs_f64() * 1000.0;
+        #[cfg(not(test))]
+        if let Some(histogram) = grpc_server_duration_if_ready(operation, backend) {
+            histogram.record(duration_ms);
+            return;
+        }
+        operation_duration_histogram(INTERNODE_OPERATION_SERVER_DURATION_MS, operation, backend).record(duration_ms);
     }
 
     pub fn record_stage_duration_for_operation_and_backend(
@@ -1699,7 +1787,7 @@ mod tests {
 
     #[test]
     fn operation_metric_descriptors_include_backend_and_operation_labels() {
-        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 22);
+        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 23);
         for metric in &INTERNODE_OPERATION_METRICS[..6] {
             assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
         }
@@ -1737,6 +1825,8 @@ mod tests {
         // Payload histogram + large-payload counter carry operation+backend labels.
         assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
         assert_eq!(INTERNODE_OPERATION_METRICS[21].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
+        assert_eq!(INTERNODE_OPERATION_METRICS[22].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
+        assert_eq!(INTERNODE_OPERATION_METRICS[22].name, INTERNODE_OPERATION_SERVER_DURATION_MS);
     }
 
     #[test]

crates/io-metrics/tests/internode_cached_durations.rs

@@ -0,0 +1,206 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit, with_local_recorder};
+use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_UNKNOWN,
+    InternodeMetrics, set_internode_server_label,
+};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Duration;
+
+const DURATION_METRIC: &str = "rustfs_system_network_internode_operation_duration_ms";
+const SERVER_DURATION_METRIC: &str = "rustfs_system_network_internode_operation_server_duration_ms";
+const SERVER_READ_OPERATIONS: [&str; 2] = [
+    INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+];
+const READ_OPERATIONS: [&str; 3] = [
+    INTERNODE_OPERATION_GRPC_READ_VERSION,
+    INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+];
+
+#[derive(Default)]
+struct RegistrationCountingRecorder {
+    inner: DebuggingRecorder,
+    duration_registrations: AtomicUsize,
+}
+
+impl Recorder for RegistrationCountingRecorder {
+    fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.inner.describe_counter(key, unit, description);
+    }
+
+    fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.inner.describe_gauge(key, unit, description);
+    }
+
+    fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.inner.describe_histogram(key, unit, description);
+    }
+
+    fn register_counter(&self, key: &Key, metadata: &Metadata<'_>) -> Counter {
+        self.inner.register_counter(key, metadata)
+    }
+
+    fn register_gauge(&self, key: &Key, metadata: &Metadata<'_>) -> Gauge {
+        self.inner.register_gauge(key, metadata)
+    }
+
+    fn register_histogram(&self, key: &Key, metadata: &Metadata<'_>) -> Histogram {
+        if key.name() == DURATION_METRIC || key.name() == SERVER_DURATION_METRIC {
+            self.duration_registrations.fetch_add(1, Ordering::Relaxed);
+        }
+        self.inner.register_histogram(key, metadata)
+    }
+}
+
+// Integration tests exercise the production cache, which is disabled in unit tests
+// so individual unit tests can install independent local recorders.
+#[test]
+fn unary_read_duration_handles_are_cached_only_after_server_label_is_ready() {
+    let recorder = RegistrationCountingRecorder::default();
+    let snapshotter = recorder.inner.snapshotter();
+    let metrics = InternodeMetrics::default();
+
+    with_local_recorder(&recorder, || {
+        for operation in READ_OPERATIONS {
+            metrics.record_duration_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_GRPC, Duration::ZERO);
+        }
+        for operation in SERVER_READ_OPERATIONS {
+            metrics.record_server_duration_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_GRPC, Duration::ZERO);
+        }
+        set_internode_server_label("cached-duration-test");
+
+        for operation in READ_OPERATIONS {
+            metrics.record_duration_for_operation_and_backend(
+                operation,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                Duration::from_micros(125),
+            );
+            let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
+            metrics.record_duration_for_operation_and_backend(
+                operation,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                Duration::from_micros(250),
+            );
+            assert_eq!(
+                recorder.duration_registrations.load(Ordering::Relaxed),
+                registrations,
+                "{operation} should reuse its duration handle after warm-up"
+            );
+        }
+
+        for operation in SERVER_READ_OPERATIONS {
+            metrics.record_server_duration_for_operation_and_backend(
+                operation,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                Duration::from_micros(1500),
+            );
+            let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
+            metrics.record_server_duration_for_operation_and_backend(
+                operation,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                Duration::from_micros(2500),
+            );
+            assert_eq!(
+                recorder.duration_registrations.load(Ordering::Relaxed),
+                registrations,
+                "{operation} should reuse its separate server duration handle after warm-up"
+            );
+        }
+
+        // Other backends and operations must retain their own labels and fallback.
+        for (operation, backend) in READ_OPERATIONS
+            .map(|operation| (operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN))
+            .into_iter()
+            .chain([(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC)])
+        {
+            let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
+            metrics.record_duration_for_operation_and_backend(operation, backend, Duration::from_millis(1));
+            assert_eq!(recorder.duration_registrations.load(Ordering::Relaxed), registrations + 1);
+        }
+        for (operation, backend) in SERVER_READ_OPERATIONS
+            .map(|operation| (operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN))
+            .into_iter()
+            .chain([(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC)])
+        {
+            let registrations = recorder.duration_registrations.load(Ordering::Relaxed);
+            metrics.record_server_duration_for_operation_and_backend(operation, backend, Duration::from_millis(3));
+            assert_eq!(recorder.duration_registrations.load(Ordering::Relaxed), registrations + 1);
+        }
+    });
+
+    let rows = snapshotter.snapshot().into_vec();
+    let durations: Vec<_> = rows
+        .iter()
+        .filter(|(key, _, _, _)| key.key().name() == DURATION_METRIC || key.key().name() == SERVER_DURATION_METRIC)
+        .collect();
+    assert_eq!(
+        durations
+            .iter()
+            .filter(|(key, _, _, _)| key.key().name() == DURATION_METRIC)
+            .count(),
+        10
+    );
+    assert_eq!(
+        durations
+            .iter()
+            .filter(|(key, _, _, _)| key.key().name() == SERVER_DURATION_METRIC)
+            .count(),
+        7
+    );
+    for (key, _, _, value) in durations {
+        let server_duration = key.key().name() == SERVER_DURATION_METRIC;
+        let operations = if server_duration {
+            SERVER_READ_OPERATIONS.as_slice()
+        } else {
+            READ_OPERATIONS.as_slice()
+        };
+        let labels: Vec<_> = key.key().labels().map(|label| (label.key(), label.value())).collect();
+        assert_eq!(labels.len(), 3);
+        let server = labels.iter().find(|(name, _)| *name == "server").expect("server label").1;
+        let operation = labels
+            .iter()
+            .find(|(name, _)| *name == "operation")
+            .expect("operation label")
+            .1;
+        let backend = labels.iter().find(|(name, _)| *name == "backend").expect("backend label").1;
+        let expected: &[f64] = if server == "unset" {
+            assert!(operations.contains(&operation));
+            assert_eq!(backend, INTERNODE_TRANSPORT_BACKEND_GRPC);
+            &[0.0]
+        } else {
+            assert_eq!(server, "cached-duration-test");
+            if backend == INTERNODE_TRANSPORT_BACKEND_GRPC && operations.contains(&operation) {
+                if server_duration { &[1.5, 2.5] } else { &[0.125, 0.25] }
+            } else {
+                assert!(
+                    (backend == INTERNODE_TRANSPORT_BACKEND_UNKNOWN && operations.contains(&operation))
+                        || (backend == INTERNODE_TRANSPORT_BACKEND_GRPC && operation == INTERNODE_OPERATION_GRPC_READ_ALL)
+                );
+                if server_duration { &[3.0] } else { &[1.0] }
+            }
+        };
+        match value {
+            DebugValue::Histogram(samples) => {
+                assert_eq!(samples.iter().map(|sample| sample.0).collect::<Vec<_>>(), expected);
+            }
+            other => panic!("duration metric must be a histogram, got {other:?}"),
+        }
+    }
+}

crates/lock/src/client/local.rs

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 use std::collections::HashMap;
+use std::ops::{Deref, DerefMut};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, SystemTime};
-use tokio::sync::RwLock;
-use tokio::time::{Instant, timeout};
+use tokio::sync::Notify;
+use tokio::time::{Instant, timeout, timeout_at};
 
 use crate::{
     FastLockGuard, GlobalLockManager, LockClient, LockId, LockInfo, LockManager, LockMetadata, LockPriority, LockRequest,
@@ -26,13 +28,98 @@ use crate::{
 
 /// Default shard count for guard storage (must be power of 2)
 const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
+const EVENT_NAMESPACE_LOCK_ACQUIRE: &str = "namespace_lock_acquire";
 const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
 const LOG_COMPONENT_LOCK: &str = "lock";
 const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
 
-type GuardShard = Arc<RwLock<HashMap<LockId, LocalGuardEntry>>>;
+type GuardMap = HashMap<LockId, LocalGuardEntry>;
+type GuardShard = Arc<GuardRegistry>;
 type GuardStorage = Arc<Vec<GuardShard>>;
 
+/// Registry access never reserves the table for a task that has not resumed.
+/// Only short synchronous map operations hold the native lock; contention
+/// waits on a notification without blocking an executor thread.
+#[derive(Debug, Default)]
+struct GuardRegistry {
+    entries: RwLock<GuardMap>,
+    changed: Notify,
+}
+
+struct GuardRegistryWake<'a>(&'a Notify);
+
+impl Drop for GuardRegistryWake<'_> {
+    fn drop(&mut self) {
+        self.0.notify_waiters();
+    }
+}
+
+struct GuardRegistryAccess<'a, G> {
+    // Field drop order unlocks the table before waking its waiters, including
+    // when a map operation unwinds. Physical guards are removed before Drop.
+    guard: G,
+    _wake: GuardRegistryWake<'a>,
+}
+
+impl<G: Deref> Deref for GuardRegistryAccess<'_, G> {
+    type Target = G::Target;
+
+    fn deref(&self) -> &Self::Target {
+        &self.guard
+    }
+}
+
+impl<G: DerefMut> DerefMut for GuardRegistryAccess<'_, G> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.guard
+    }
+}
+
+impl GuardRegistry {
+    fn access<G>(&self, guard: G) -> GuardRegistryAccess<'_, G> {
+        GuardRegistryAccess {
+            guard,
+            _wake: GuardRegistryWake(&self.changed),
+        }
+    }
+
+    async fn read(&self) -> GuardRegistryAccess<'_, RwLockReadGuard<'_, GuardMap>> {
+        tokio::task::consume_budget().await;
+        if let Some(guard) = self.entries.try_read() {
+            return self.access(guard);
+        }
+        loop {
+            tokio::task::consume_budget().await;
+            let notified = self.changed.notified();
+            tokio::pin!(notified);
+            notified.as_mut().enable();
+            if let Some(guard) = self.entries.try_read() {
+                return self.access(guard);
+            }
+            notified.await;
+        }
+    }
+
+    async fn write(&self) -> GuardRegistryAccess<'_, RwLockWriteGuard<'_, GuardMap>> {
+        tokio::task::consume_budget().await;
+        if let Some(guard) = self.entries.try_write() {
+            return self.access(guard);
+        }
+        loop {
+            tokio::task::consume_budget().await;
+            // Register before rechecking so an unlock between the first
+            // failed try and this wait cannot leave the task asleep.
+            let notified = self.changed.notified();
+            tokio::pin!(notified);
+            notified.as_mut().enable();
+            if let Some(guard) = self.entries.try_write() {
+                return self.access(guard);
+            }
+            notified.await;
+        }
+    }
+}
+
 /// Local lock client using FastLock with sharded guard storage for better concurrency
 #[derive(Debug)]
 pub struct LocalClient {
@@ -94,7 +181,7 @@ impl LocalClient {
     pub fn with_shard_count(shard_count: usize) -> Self {
         assert!(shard_count.is_power_of_two(), "Shard count must be power of 2");
 
-        let guard_storage: Vec<GuardShard> = (0..shard_count).map(|_| Arc::new(RwLock::new(HashMap::new()))).collect();
+        let guard_storage: Vec<GuardShard> = (0..shard_count).map(|_| Arc::new(GuardRegistry::default())).collect();
 
         Self::with_storage(Arc::new(guard_storage), None, crate::fast_lock::CLEANUP_INTERVAL)
     }
@@ -115,7 +202,7 @@ impl LocalClient {
     /// This allows simulating multi-node environments where each node has its own lock backend
     pub fn with_manager(manager: Arc<GlobalLockManager>) -> Self {
         let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT)
-            .map(|_| Arc::new(RwLock::new(HashMap::new())))
+            .map(|_| Arc::new(GuardRegistry::default()))
             .collect();
         Self::with_storage(Arc::new(guard_storage), Some(manager), crate::fast_lock::CLEANUP_INTERVAL)
     }
@@ -123,7 +210,7 @@ impl LocalClient {
     #[cfg(test)]
     pub(crate) fn with_manager_and_reaper_interval(manager: Arc<GlobalLockManager>, reaper_interval: Duration) -> Self {
         let guard_storage = (0..DEFAULT_GUARD_SHARD_COUNT)
-            .map(|_| Arc::new(RwLock::new(HashMap::new())))
+            .map(|_| Arc::new(GuardRegistry::default()))
             .collect();
         Self::with_storage(Arc::new(guard_storage), Some(manager), reaper_interval)
     }
@@ -140,7 +227,7 @@ impl LocalClient {
     }
 
     /// Get the shard for a given lock ID
-    fn get_shard(&self, lock_id: &LockId) -> &Arc<RwLock<HashMap<LockId, LocalGuardEntry>>> {
+    fn get_shard(&self, lock_id: &LockId) -> &GuardShard {
         let index = self.get_resource_shard_index(&lock_id.resource);
         &self.guard_storage[index]
     }
@@ -232,6 +319,7 @@ impl Default for LocalClient {
 #[async_trait::async_trait]
 impl LockClient for LocalClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
+        let started = Instant::now();
         self.ensure_reaper();
         if request.lock_id.resource != request.resource {
             return Ok(LockResponse::failure(
@@ -251,6 +339,7 @@ impl LockClient for LocalClient {
                 .await
                 .unwrap_or(0)
         };
+        let reclaim_elapsed = started.elapsed();
 
         let build_lock_request = |acquire_timeout| match request.lock_type {
             LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
@@ -265,17 +354,65 @@ impl LockClient for LocalClient {
             if remaining.is_zero() {
                 return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
             }
+            let manager_started = Instant::now();
             match lock_manager.acquire_lock(build_lock_request(remaining)).await {
                 Ok(guard) => {
+                    let manager_elapsed = manager_started.elapsed();
+                    let registration_started = Instant::now();
+                    let physical_guard_id = guard.guard_id();
                     let lock_id = request.lock_id.clone();
                     let entry = LocalGuardEntry::new(guard, request.ttl);
                     let acquired_at = entry.acquired_at;
                     let expires_at = entry.expires_at;
 
-                    {
+                    let (replaced_entry, registration_wait, registration_mutation, registry_entries, registry_capacity) = {
                         let shard = self.get_shard(&lock_id);
-                        let mut guards = shard.write().await;
-                        guards.insert(lock_id.clone(), entry);
+                        let Ok(mut guards) = timeout_at(acquire_deadline, shard.write()).await else {
+                            return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
+                        };
+                        // A ready table cannot publish a grant after the peer
+                        // budget expired while its acquisition task was idle.
+                        if Instant::now() >= acquire_deadline {
+                            drop(guards);
+                            return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
+                        }
+                        let registration_wait = registration_started.elapsed();
+                        let mutation_started = Instant::now();
+                        let replaced_entry = guards.insert(lock_id.clone(), entry);
+                        (
+                            replaced_entry,
+                            registration_wait,
+                            mutation_started.elapsed(),
+                            guards.len(),
+                            guards.capacity(),
+                        )
+                    };
+                    // Repeated shared requests can replace an entry. Releasing
+                    // its physical guard must not reenter locks under the table.
+                    drop(replaced_entry);
+
+                    let registration_elapsed = registration_started.elapsed();
+                    if started.elapsed() >= Duration::from_millis(100) {
+                        tracing::trace!(
+                            target: "rustfs_lock::namespace_owner",
+                            event = EVENT_NAMESPACE_LOCK_ACQUIRE,
+                            component = LOG_COMPONENT_LOCK,
+                            subsystem = LOG_SUBSYSTEM_NAMESPACE,
+                            state = "local_completed",
+                            resource = %request.resource,
+                            lock_id = %lock_id,
+                            mode = ?request.lock_type,
+                            physical_guard_id,
+                            reclaim_ms = reclaim_elapsed.as_millis(),
+                            manager_ms = manager_elapsed.as_millis(),
+                            registration_ms = registration_elapsed.as_millis(),
+                            registration_wait_ms = registration_wait.as_millis(),
+                            registration_mutation_ms = registration_mutation.as_millis(),
+                            registry_entries,
+                            registry_capacity,
+                            acquire_ms = started.elapsed().as_millis(),
+                            "Namespace local lock acquisition completed slowly"
+                        );
                     }
 
                     let lock_info = LockInfo {
@@ -359,6 +496,16 @@ impl LockClient for LocalClient {
         }
     }
 
+    async fn cleanup_grant(&self, lock_id: &LockId) -> Result<super::LockCleanupOutcome> {
+        self.release(lock_id).await.map(|released| {
+            if released {
+                super::LockCleanupOutcome::Released
+            } else {
+                super::LockCleanupOutcome::AlreadyAbsent
+            }
+        })
+    }
+
     async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
         let shard = self.get_shard(lock_id);
         let expired_entry = {
@@ -459,6 +606,8 @@ impl LockClient for LocalClient {
 mod tests {
     use super::*;
     use crate::{GlobalLockManager, LockClient, LockRequest, LockType};
+    use futures::FutureExt;
+    use std::task::Poll;
 
     fn request(resource: crate::ObjectKey, owner: &str, ttl: Duration) -> LockRequest {
         LockRequest::new(resource, LockType::Exclusive, owner)
@@ -466,6 +615,295 @@ mod tests {
             .with_acquire_timeout(Duration::from_millis(80))
     }
 
+    fn hold_guard_shard(shard: GuardShard) -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) {
+        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(0);
+        let (release_tx, release_rx) = std::sync::mpsc::channel();
+        let thread = std::thread::spawn(move || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("build the independent guard-table owner runtime");
+            let guard = runtime.block_on(shard.write());
+            entered_tx.send(()).expect("signal that the guard table is occupied");
+            // A dropped controller also releases the table on test failure.
+            let _ = release_rx.recv_timeout(Duration::from_secs(3));
+            drop(guard);
+        });
+        entered_rx
+            .recv_timeout(Duration::from_secs(3))
+            .expect("guard-table owner must enter before the tested operation");
+        (release_tx, thread)
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unpolled_registry_writer_does_not_block_compatible_acquisition() {
+        let manager = Arc::new(GlobalLockManager::new());
+        let client = LocalClient::with_manager(manager.clone());
+        client.reaper_started.store(true, Ordering::Release);
+        let first = LockRequest::new(crate::ObjectKey::new("bucket", ""), LockType::Shared, "first");
+        assert!(client.acquire_lock(&first).await.expect("first shared lock").success);
+        let (release_table, table_owner) = hold_guard_shard(client.get_shard(&first.lock_id).clone());
+        let mut pending_release = Box::pin(client.release(&first.lock_id));
+        assert!(matches!(futures::poll!(&mut pending_release), Poll::Pending));
+        release_table.send(()).expect("release the occupied table");
+        table_owner.join().expect("guard-table owner must finish");
+
+        // The earlier writer is deliberately not polled: an unoccupied
+        // bookkeeping table must remain usable by a currently running peer.
+        let second = LockRequest::new(first.resource.clone(), LockType::Shared, "second");
+        let response = client
+            .acquire_lock(&second)
+            .now_or_never()
+            .expect("a sleeping registry writer must not reserve the available table")
+            .expect("compatible shared acquisition should return a response");
+        assert!(response.success);
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled physical lock backend")
+                .get_shard(&first.resource)
+                .active_guard_count(),
+            2
+        );
+        assert!(
+            pending_release
+                .await
+                .expect("the earlier writer still releases its exact grant")
+        );
+        assert!(client.release(&second.lock_id).await.expect("release the second grant"));
+        assert_eq!(
+            manager
+                .as_fast_lock_manager()
+                .expect("enabled physical lock backend")
+                .get_shard(&first.resource)
+                .active_guard_count(),
+            0
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acquisition_deadline_rejects_registration_after_physical_grant() {
+        for unlock_before_poll in [false, true] {
+            let manager = Arc::new(GlobalLockManager::new());
+            let client = LocalClient::with_manager(manager.clone());
+            client.reaper_started.store(true, Ordering::Release);
+            let resource = crate::ObjectKey::new("bucket", "registration-deadline");
+            let blocker = manager
+                .acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "blocker"))
+                .await
+                .expect("hold the physical lock until guard-table preflight finishes");
+            let request =
+                request(resource.clone(), "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(20));
+            let mut acquire = Box::pin(tokio::task::unconstrained(client.acquire_lock(&request)));
+            assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+            let (release_table, table_owner) = hold_guard_shard(client.get_shard(&request.lock_id).clone());
+            drop(blocker);
+            assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+            assert_eq!(
+                manager
+                    .as_fast_lock_manager()
+                    .expect("enabled physical lock backend")
+                    .get_shard(&resource)
+                    .active_guard_count(),
+                1,
+                "the tested future owns a physical grant"
+            );
+            tokio::time::advance(Duration::from_millis(21)).await;
+            if unlock_before_poll {
+                release_table.send(()).expect("make registration ready after its deadline");
+                table_owner.join().expect("guard-table owner must finish");
+                assert!(!acquire.await.expect("expired registration response").success);
+            } else {
+                let response = timeout(Duration::from_millis(100), &mut acquire)
+                    .await
+                    .expect("registration must honor its deadline while the table remains occupied")
+                    .expect("expired registration response");
+                assert!(!response.success);
+                release_table.send(()).expect("release the occupied table");
+                table_owner.join().expect("guard-table owner must finish");
+            }
+            assert!(
+                client
+                    .check_status(&request.lock_id)
+                    .await
+                    .expect("check the expired identity")
+                    .is_none()
+            );
+            assert_eq!(
+                manager
+                    .as_fast_lock_manager()
+                    .expect("enabled physical lock backend")
+                    .get_shard(&resource)
+                    .active_guard_count(),
+                0,
+                "expired registration must release its grant"
+            );
+            assert!(
+                manager
+                    .acquire_lock(crate::ObjectLockRequest::new_write(resource, "next-owner"))
+                    .await
+                    .is_ok(),
+                "the next owner must not inherit an abandoned physical grant"
+            );
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn cancelling_registration_releases_the_unpublished_physical_grant() {
+        let manager = Arc::new(GlobalLockManager::new());
+        let client = LocalClient::with_manager(manager.clone());
+        client.reaper_started.store(true, Ordering::Release);
+        let resource = crate::ObjectKey::new("bucket", "cancel-registration");
+        let blocker = manager
+            .acquire_lock(crate::ObjectLockRequest::new_write(resource.clone(), "blocker"))
+            .await
+            .expect("hold the physical lock through registry preflight");
+        let request = request(resource.clone(), "cancelled", Duration::from_secs(30));
+        let mut acquire = Box::pin(tokio::task::unconstrained(client.acquire_lock(&request)));
+        assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+        let (release_table, table_owner) = hold_guard_shard(client.get_shard(&request.lock_id).clone());
+        drop(blocker);
+        assert!(matches!(futures::poll!(&mut acquire), Poll::Pending));
+        let backend = manager.as_fast_lock_manager().expect("enabled physical lock backend");
+        assert_eq!(backend.get_shard(&resource).active_guard_count(), 1);
+        drop(acquire);
+        assert_eq!(backend.get_shard(&resource).active_guard_count(), 0);
+        assert!(
+            manager
+                .acquire_lock(crate::ObjectLockRequest::new_write(resource, "next-owner"))
+                .await
+                .is_ok(),
+            "cancellation must release the physical lock without waiting for the table"
+        );
+        release_table.send(()).expect("release the occupied table");
+        table_owner.join().expect("guard-table owner must finish");
+        assert!(
+            client
+                .check_status(&request.lock_id)
+                .await
+                .expect("check cancelled identity")
+                .is_none()
+        );
+        assert!(client.list_lock_leases().await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn cancelling_a_registry_waiter_keeps_other_waiters_live() {
+        let shard = Arc::new(GuardRegistry::default());
+        let (release_table, table_owner) = hold_guard_shard(shard.clone());
+        let mut cancelled = Box::pin(shard.write());
+        let mut survivor = Box::pin(shard.read());
+        assert!(matches!(futures::poll!(&mut cancelled), Poll::Pending));
+        assert!(matches!(futures::poll!(&mut survivor), Poll::Pending));
+        drop(cancelled);
+        release_table.send(()).expect("wake registry waiters");
+        table_owner.join().expect("guard-table owner must finish");
+        let access = timeout(Duration::from_millis(100), survivor)
+            .await
+            .expect("a cancelled waiter must not consume another waiter's unlock notification");
+        assert!(access.is_empty());
+    }
+
+    #[tokio::test]
+    async fn uncontended_registry_access_retains_cooperative_scheduling() {
+        let shard = GuardRegistry::default();
+        let mut yielded = false;
+        for _ in 0..1024 {
+            match shard.read().now_or_never() {
+                Some(access) => drop(access),
+                None => {
+                    yielded = true;
+                    break;
+                }
+            }
+        }
+        assert!(yielded, "a loop of ready registry operations must still yield to other executor tasks");
+    }
+
+    #[tokio::test]
+    async fn repeated_shared_identity_replaces_only_its_previous_physical_guard() {
+        let manager = Arc::new(GlobalLockManager::new());
+        let client = LocalClient::with_manager(manager.clone());
+        let request = LockRequest::new(crate::ObjectKey::new("bucket", ""), LockType::Shared, "owner");
+        assert!(client.acquire_lock(&request).await.expect("first shared grant").success);
+        assert!(client.acquire_lock(&request).await.expect("repeated shared identity").success);
+        let backend = manager.as_fast_lock_manager().expect("enabled physical lock backend");
+        assert_eq!(backend.get_shard(&request.resource).active_guard_count(), 1);
+        assert_eq!(client.list_lock_leases().await.len(), 1);
+        assert!(client.release(&request.lock_id).await.expect("release the replacement grant"));
+        assert_eq!(backend.get_shard(&request.resource).active_guard_count(), 0);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn hot_shared_registry_keeps_refresh_reaper_and_snapshots_live() {
+        for width in [64, 128] {
+            let manager = Arc::new(GlobalLockManager::new());
+            let client = Arc::new(LocalClient::with_manager_and_reaper_interval(manager.clone(), Duration::from_millis(1)));
+            let resource = crate::ObjectKey::new("bucket", "");
+            let expired = LockRequest::new(resource.clone(), LockType::Shared, "expired").with_ttl(Duration::ZERO);
+            assert!(client.acquire_lock(&expired).await.expect("expired seed grant").success);
+            let acquired = Arc::new(tokio::sync::Barrier::new(width + 1));
+            let observed = Arc::new(Notify::new());
+            let mut tasks = tokio::task::JoinSet::new();
+            for index in 0..width {
+                let client = client.clone();
+                let resource = resource.clone();
+                let acquired = acquired.clone();
+                let observed = observed.clone();
+                tasks.spawn(async move {
+                    for iteration in 0..8 {
+                        let request = LockRequest::new(resource.clone(), LockType::Shared, format!("load-{index}"));
+                        assert!(client.acquire_lock(&request).await.expect("concurrent shared grant").success);
+                        if iteration == 0 {
+                            let notification = observed.notified();
+                            tokio::pin!(notification);
+                            notification.as_mut().enable();
+                            acquired.wait().await;
+                            notification.await;
+                        } else {
+                            tokio::task::yield_now().await;
+                        }
+                        assert!(client.refresh(&request.lock_id).await.expect("refresh concurrent grant"));
+                        assert!(
+                            client
+                                .check_status(&request.lock_id)
+                                .await
+                                .expect("read concurrent grant")
+                                .is_some()
+                        );
+                        assert!(client.release(&request.lock_id).await.expect("release concurrent grant"));
+                    }
+                });
+            }
+            timeout(Duration::from_secs(5), acquired.wait())
+                .await
+                .expect("all shared grants must reach the snapshot gate");
+            let snapshot = client.list_lock_leases().await;
+            assert_eq!(snapshot.iter().filter(|lease| lease.owner.starts_with("load-")).count(), width);
+            observed.notify_waiters();
+            let mut completed = 0;
+            timeout(Duration::from_secs(5), async {
+                while let Some(result) = tasks.join_next().await {
+                    result.expect("all shared grant workers must finish");
+                    completed += 1;
+                    let _ = client.list_lock_leases().await;
+                }
+            })
+            .await
+            .expect("refresh, cleanup, and concurrent snapshots must make progress");
+            assert_eq!(completed, width);
+            wait_until_reaped(&client, &expired.lock_id).await;
+            assert!(client.list_lock_leases().await.is_empty());
+            assert!(
+                manager
+                    .as_fast_lock_manager()
+                    .expect("enabled physical lock backend")
+                    .list_locks()
+                    .is_empty()
+            );
+        }
+    }
+
     async fn wait_until_reaped(client: &LocalClient, lock_id: &LockId) {
         for _ in 0..80 {
             if client.check_status(lock_id).await.unwrap().is_none() {
@@ -492,8 +930,9 @@ mod tests {
         }
 
         let lock_request = request(resource, "owner", Duration::from_secs(30));
-        let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
-            .await
+        let response = client
+            .acquire_lock(&lock_request)
+            .now_or_never()
             .expect("an acquire must not sweep unrelated guard shards")
             .expect("local acquire should return a response");
         assert!(response.success);
@@ -520,8 +959,9 @@ mod tests {
         }
 
         let replacement = request(resource, "replacement-owner", Duration::from_secs(30));
-        let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&replacement))
-            .await
+        let response = client
+            .acquire_lock(&replacement)
+            .now_or_never()
             .expect("expired guard reclaim must only write the resource shard")
             .expect("local acquire should return a response");
         assert!(response.success);
@@ -537,7 +977,7 @@ mod tests {
         client.reaper_started.store(true, Ordering::Release);
         let resource = crate::ObjectKey::new("bucket", "resource-shard-deadline");
         let target_shard = client.get_resource_shard_index(&resource);
-        let _target_write = client.guard_storage[target_shard].write().await;
+        let (release_table, table_owner) = hold_guard_shard(client.guard_storage[target_shard].clone());
         let lock_request = request(resource, "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
 
         let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
@@ -545,6 +985,8 @@ mod tests {
             .expect("acquire should return after its reclaim budget expires")
             .expect("local acquire should return a response");
         assert!(!response.success);
+        release_table.send(()).expect("release the occupied resource table");
+        table_owner.join().expect("guard-table owner must finish");
     }
 
     #[tokio::test]

crates/lock/src/client/mod.rs

@@ -20,6 +20,17 @@ use async_trait::async_trait;
 use futures::future::join_all;
 use std::sync::Arc;
 
+/// Outcome of cleaning up a lock whose acquisition has already completed.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum LockCleanupOutcome {
+    /// This request released the granted lock.
+    Released,
+    /// The authoritative backend no longer holds this exact lock ID.
+    AlreadyAbsent,
+    /// The backend did not confirm release or authoritative absence.
+    Unconfirmed,
+}
+
 /// Lock client trait
 #[async_trait]
 pub trait LockClient: Send + Sync + std::fmt::Debug {
@@ -37,6 +48,29 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
     /// Release lock
     async fn release(&self, lock_id: &LockId) -> Result<bool>;
 
+    /// Idempotently clean up a previously confirmed grant for this exact lock ID.
+    ///
+    /// Call only after acquisition has completed successfully. An absent lock
+    /// cannot settle an acquisition that may still grant it later; that
+    /// acquisition must retain its own completion cleanup. This operation must
+    /// never release another owner's lock on the same resource.
+    ///
+    /// The default preserves conservative behavior for clients without an
+    /// authoritative absence result. Cancellation or an error leaves cleanup
+    /// unconfirmed, so the caller must retain its retry ownership.
+    ///
+    /// # Errors
+    /// Returns backend or transport errors when cleanup cannot be confirmed.
+    async fn cleanup_grant(&self, lock_id: &LockId) -> Result<LockCleanupOutcome> {
+        self.release(lock_id).await.map(|released| {
+            if released {
+                LockCleanupOutcome::Released
+            } else {
+                LockCleanupOutcome::Unconfirmed
+            }
+        })
+    }
+
     /// Release multiple locks. Default implementation fans out to single-lock releases.
     async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
         Ok(join_all(lock_ids.iter().map(|lock_id| self.release(lock_id)))

crates/lock/src/lib.rs

@@ -38,7 +38,7 @@ pub mod types;
 // Re-export main types for easy access
 pub use crate::{
     // Client interfaces
-    client::{LockClient, local::LocalClient},
+    client::{LockCleanupOutcome, LockClient, local::LocalClient},
     distributed_lock::DistributedLockGuard,
     // Error types
     error::{LockError, Result},

crates/log-analyzer/src/rules/seed/lock.rs

@@ -59,13 +59,14 @@ pub(super) fn rules() -> Vec<Rule> {
             )
         },
         Rule {
-            anchors: strings(["distributed unlock failed on client"]),
+            anchors: strings(["Distributed unlock cleanup failed on client"]),
             ..base(
                 "dist-unlock-failed",
                 P2Degraded,
                 "lock",
                 "分布式加/解锁在部分节点失败",
                 any([
+                    contains("Distributed unlock cleanup failed on client"),
                     contains("distributed unlock failed on client"),
                     contains("Failed to acquire lock on client"),
                 ]),

crates/log-analyzer/src/rules/seed/tests.rs

@@ -165,7 +165,7 @@ fn every_rule_has_a_positive_sample() {
         ("lock-acquire-timeout", msg("Lock acquisition timeout for resource 'b/o' after 30s")),
         ("lock-quorum-nodes", msg("Quorum not reached: required 3, achieved 1")),
         ("lock-owner-mismatch", msg("Not the lock owner: lock_id abc, owner node1")),
-        ("dist-unlock-failed", msg("distributed unlock failed on client: node2")),
+        ("dist-unlock-failed", msg("Distributed unlock cleanup failed on client")),
         (
             "lock-state-inconsistent",
             msg("Atomic state inconsistency during exclusive lock release: owner=x, atomic_state=101"),
@@ -305,6 +305,32 @@ fn every_rule_has_a_positive_sample() {
     }
 }
 
+#[test]
+fn distributed_unlock_rule_matches_current_and_legacy_failures_only() {
+    let engine = engine();
+    for (message, expected) in [
+        ("Distributed unlock cleanup failed on client", true),
+        ("distributed unlock failed on client: node2", true),
+        ("Failed to acquire lock on client node2", true),
+        ("Distributed unlock cleanup completed", false),
+        ("Deferred distributed unlock cleanup converged", false),
+        ("Distributed unlock cleanup was not confirmed by client", false),
+        ("Distributed unlock cleanup remains unconfirmed after deferred retries", false),
+    ] {
+        let sample = Sample {
+            message,
+            level: Some(LogLevel::Warn),
+            target: "rustfs_lock::distributed_lock",
+            ..Default::default()
+        };
+        let matched = engine
+            .matches(&event(&sample))
+            .iter()
+            .any(|&index| engine.rules()[index].id == "dist-unlock-failed");
+        assert_eq!(matched, expected, "unexpected match for {message}");
+    }
+}
+
 #[test]
 fn heal_writer_rule_requires_all_targets_and_retains_legacy_messages() {
     let engine = engine();

crates/protocols/src/sftp/dir.rs

@@ -56,6 +56,17 @@ pub(super) fn dot_entries() -> Vec<File> {
     ]
 }
 
+/// Client-visible S3 key for an SFTP sub-directory.
+///
+/// `object_key` is the parsed prefix without a trailing slash. The
+/// returned key ends in `/` so the storage layer can encode it to the
+/// reserved directory marker. Callers must not encode the key first:
+/// `validate_object_key` rejects any client key that already contains
+/// the marker suffix.
+fn directory_client_key(object_key: &str) -> String {
+    format!("{object_key}/")
+}
+
 fn directory_listing_input(
     bucket: &str,
     prefix: &str,
@@ -300,19 +311,19 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         Ok(())
     }
 
-    /// MKDIR for a sub-directory path: write a zero-byte object at
-    /// encode_dir_object(prefix + "/"). The encoding maps "foo/" to
-    /// "foo__XLDIR__", which matches the RustFS marker convention used
-    /// by the S3, Swift, and WebDAV backends.
+    /// MKDIR for a sub-directory path: write a zero-byte object at the
+    /// client-visible directory key (`foo/`). Storage encodes that key to
+    /// the on-disk marker `foo__XLDIR__` after the S3 client-key boundary.
+    /// The encoded suffix is reserved, so PutObject must not be given it.
     pub(super) async fn mkdir_subdir_marker(&self, bucket: &str, object_key: &str) -> Result<(), SftpError> {
-        let marker_key = path::encode_dir_object(&format!("{object_key}/"));
-        self.authorize(&S3Action::PutObject, bucket, Some(&marker_key)).await?;
+        let dir_key = directory_client_key(object_key);
+        self.authorize(&S3Action::PutObject, bucket, Some(&dir_key)).await?;
 
         let body = stream::once(async { Ok::<Bytes, std::io::Error>(Bytes::new()) });
         let streaming = StreamingBlob::wrap(body);
         let input = PutObjectInput::builder()
             .bucket(bucket.to_string())
-            .key(marker_key.clone())
+            .key(dir_key)
             .content_length(Some(0))
             .body(Some(streaming))
             .build()
@@ -333,15 +344,14 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
     }
 
     /// RMDIR for a sub-directory path: validate no objects under the
-    /// prefix, then authorise and delete the __XLDIR__ marker that
-    /// represents the directory.
+    /// prefix, then authorise and delete the trailing-slash directory key.
+    /// Storage encodes that key to the on-disk marker.
     pub(super) async fn rmdir_subdir_marker(&self, bucket: &str, object_key: &str) -> Result<(), SftpError> {
-        let prefix = format!("{object_key}/");
-        self.validate_directory_empty(bucket, &prefix).await?;
+        let dir_key = directory_client_key(object_key);
+        self.validate_directory_empty(bucket, &dir_key).await?;
 
-        let marker_key = path::encode_dir_object(&prefix);
-        self.authorize(&S3Action::DeleteObject, bucket, Some(&marker_key)).await?;
-        self.run_backend("delete_object", self.storage.delete_object(bucket, &marker_key, self.credentials()))
+        self.authorize(&S3Action::DeleteObject, bucket, Some(&dir_key)).await?;
+        self.run_backend("delete_object", self.storage.delete_object(bucket, &dir_key, self.credentials()))
             .await?;
         Ok(())
     }
@@ -837,4 +847,95 @@ mod tests {
             "error-level event must carry the SFTP READDIR failed message, captured: {captured}"
         );
     }
+
+    #[tokio::test]
+    async fn mkdir_subdir_puts_trailing_slash_client_key() {
+        let backend = Arc::new(DummyBackend::new());
+        let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
+        let authorized = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let record = Arc::clone(&authorized);
+
+        let status = with_test_auth_override(
+            move |action, bucket, key| {
+                record
+                    .lock()
+                    .expect("auth log")
+                    .push((action.as_str(), bucket.to_string(), key.map(str::to_string)));
+                true
+            },
+            driver.mkdir(1, "/bucket/photos/2024".to_string(), russh_sftp::protocol::FileAttributes::default()),
+        )
+        .await
+        .expect("mkdir of an ordinary directory");
+
+        assert!(matches!(status.status_code, StatusCode::Ok));
+        let calls = backend.put_object_calls();
+        assert_eq!(calls.len(), 1, "mkdir must issue one PutObject");
+        assert_eq!(calls[0].bucket, "bucket");
+        assert_eq!(calls[0].key, "photos/2024/");
+        assert!(
+            !calls[0].key.contains(rustfs_utils::path::GLOBAL_DIR_SUFFIX),
+            "the client key must not carry the reserved directory-marker suffix"
+        );
+        let auth = authorized.lock().expect("auth log");
+        assert_eq!(
+            auth.as_slice(),
+            [("s3:PutObject", "bucket".to_string(), Some("photos/2024/".to_string()))]
+        );
+    }
+
+    #[tokio::test]
+    async fn rmdir_subdir_deletes_trailing_slash_client_key() {
+        let backend = Arc::new(DummyBackend::new());
+        backend.queue_delete_object_ok();
+        let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
+        let authorized = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let record = Arc::clone(&authorized);
+
+        let status = with_test_auth_override(
+            move |action, bucket, key| {
+                record
+                    .lock()
+                    .expect("auth log")
+                    .push((action.as_str(), bucket.to_string(), key.map(str::to_string)));
+                true
+            },
+            driver.rmdir(2, "/bucket/photos/2024".to_string()),
+        )
+        .await
+        .expect("rmdir of an empty directory");
+
+        assert!(matches!(status.status_code, StatusCode::Ok));
+        let calls = backend.delete_object_calls();
+        assert_eq!(calls.len(), 1, "rmdir must issue one DeleteObject");
+        assert_eq!(calls[0].bucket, "bucket");
+        assert_eq!(calls[0].key, "photos/2024/");
+        assert!(
+            !calls[0].key.contains(rustfs_utils::path::GLOBAL_DIR_SUFFIX),
+            "the client key must not carry the reserved directory-marker suffix"
+        );
+        let auth = authorized.lock().expect("auth log");
+        assert_eq!(
+            auth.as_slice(),
+            [
+                ("s3:ListBucket", "bucket".to_string(), Some("photos/2024/".to_string())),
+                ("s3:DeleteObject", "bucket".to_string(), Some("photos/2024/".to_string())),
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn mkdir_rejects_reserved_directory_marker_before_put() {
+        let backend = Arc::new(DummyBackend::new());
+        let mut driver = build_driver(Arc::clone(&backend), TEST_PART_SIZE);
+        let path = format!("/bucket/sub{}", rustfs_utils::path::GLOBAL_DIR_SUFFIX);
+
+        let err = driver
+            .mkdir(3, path, russh_sftp::protocol::FileAttributes::default())
+            .await
+            .expect_err("a client path containing the reserved marker must be rejected");
+
+        assert_eq!(StatusCode::from(err), StatusCode::BadMessage);
+        assert!(backend.put_object_calls().is_empty(), "rejection must happen before PutObject");
+    }
 }

crates/protocols/src/sftp/driver.rs

@@ -831,14 +831,10 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
 
     /// SSH_FXP_MKDIR, SFTP Internet Draft section 6.6. Bucket-level path
     /// (only the bucket component is set) issues CreateBucket. Sub-bucket
-    /// path issues PutObject of a zero-byte object at the encoded
-    /// directory marker key. MKDIR at the SFTP root returns Failure
-    /// because there is no parent into which a new top-level entity could
-    /// be added.
-    ///
-    /// The directory-marker key is built with rustfs_utils::path::
-    /// encode_dir_object so the key format matches the convention used
-    /// by the rest of RustFS (S3, Swift, WebDAV).
+    /// path issues PutObject of a zero-byte object at the logical prefix
+    /// plus a trailing slash. Storage encodes that client key into the
+    /// directory marker. MKDIR at the SFTP root returns Failure because
+    /// there is no parent into which a new top-level entity could be added.
     #[tracing::instrument(level = "info", skip(self, _attrs), fields(id, path = %sanitise_control_bytes(&path)), err(Debug))]
     async fn mkdir(&mut self, id: u32, path: String, _attrs: FileAttributes) -> Result<Status, Self::Error> {
         self.enforce_server_readonly()?;
@@ -858,8 +854,9 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
     /// SSH_FXP_RMDIR, SFTP Internet Draft section 6.6. Empty check then
     /// delete. Bucket-level path lists the bucket with max_keys=1 and,
     /// on an empty result, calls DeleteBucket. Sub-bucket path lists the
-    /// prefix and, on an empty result, calls DeleteObject on the encoded
-    /// directory marker.
+    /// prefix and, on an empty result, calls DeleteObject on the
+    /// trailing-slash directory key. Storage encodes that key to the
+    /// on-disk marker.
     ///
     /// validate_directory_empty propagates the list error rather than
     /// swallowing it. Without that, a transient backend error during

crates/protocols/src/swift/metadata_update.rs

@@ -43,14 +43,9 @@ const CONTAINER_META_HEADER_PREFIX: &str = "x-container-meta-";
 /// Request header prefix removing a container metadata item.
 const CONTAINER_META_REMOVE_HEADER_PREFIX: &str = "x-remove-container-meta-";
 
-/// Bucket-tag namespace holding account metadata items.
-pub(crate) const ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
-/// Bucket-tag namespace holding container metadata items.
-///
-/// Deliberately narrower than the `swift-` tags around it: the container ACL
-/// (`swift-acl-*`) and versioning (`swift-versions-location`) tags share this
-/// tag set and must survive a metadata POST.
-pub(crate) const CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
+pub(crate) use super::storage_api::tagging::{
+    SWIFT_ACCOUNT_META_TAG_PREFIX as ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX as CONTAINER_META_TAG_PREFIX,
+};
 
 /// The metadata changes carried by one account or container POST.
 ///

crates/protocols/src/swift/mod.rs

@@ -61,11 +61,9 @@ pub use errors::{SwiftError, SwiftResult};
 pub use metadata_update::MetadataUpdate;
 pub use router::{SwiftRoute, SwiftRouter};
 
-/// Maximum number of metadata headers allowed per resource (Swift standard)
-pub(crate) const MAX_METADATA_COUNT: usize = 90;
-
-/// Maximum size in bytes for a single metadata value (Swift standard)
-pub(crate) const MAX_METADATA_VALUE_SIZE: usize = 256;
+pub(crate) use storage_api::tagging::{
+    MAX_SWIFT_METADATA_COUNT as MAX_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE as MAX_METADATA_VALUE_SIZE,
+};
 
 /// Validate metadata against Swift limits
 ///

crates/protocols/src/swift/storage_api.rs

@@ -54,6 +54,12 @@ pub(crate) mod public_api {
     };
 }
 
+pub(crate) mod tagging {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::{
+        MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE, SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX,
+    };
+}
+
 pub(crate) mod versioning {
     pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
 }

crates/protocols/src/webdav/README.md

@@ -82,6 +82,22 @@ Configure WebDAV via environment variables:
 | `RUSTFS_WEBDAV_MAX_BODY_SIZE` | Max upload size (bytes) | 5GB |
 | `RUSTFS_WEBDAV_REQUEST_TIMEOUT` | Request timeout (seconds) | 300 |
 | `RUSTFS_WEBDAV_MAX_CONNECTIONS` | Max concurrently served connections | 1024 |
+| `RUSTFS_CORS_ALLOWED_ORIGINS` | Browser origins allowed by the S3 and WebDAV listeners | empty |
+
+## Browser Access and CORS
+
+WebDAV reads `RUSTFS_CORS_ALLOWED_ORIGINS` at server startup. Set a comma-separated list of exact frontend origins to allow browser clients, for example `https://dav-console.example,https://files.example`. Explicit origins allow credentialed requests. The default empty value disables cross-origin access. A standalone `*` permits any origin with credentials mode `omit` and emits no `Access-Control-Allow-Credentials` header.
+
+Valid `OPTIONS` preflight requests from an allowed origin receive `204 No Content` without Basic authentication. Actual WebDAV requests and ordinary `OPTIONS` requests still use Basic authentication and the existing IAM checks. Allowed origins receive CORS headers on success and error responses. Preflights allow WebDAV methods and explicitly echo valid requested header names, including `Authorization`, `Depth`, `Destination`, and `If-Match`. Responses expose `ETag`, `DAV`, `Allow`, `Lock-Token`, and range headers to browser code.
+
+```bash
+curl -i -X OPTIONS http://localhost:8080/ \
+  -H 'Origin: https://dav-console.example' \
+  -H 'Access-Control-Request-Method: PROPFIND' \
+  -H 'Access-Control-Request-Headers: authorization,depth,content-type'
+```
+
+Configure an explicit HTTPS frontend origin for credentialed browser access and use HTTPS on the WebDAV listener when sending access keys and secret keys. This listener-wide policy applies across WebDAV paths. S3 bucket CORS rules remain scoped to the S3 API.
 
 ## Quick Start
 

crates/protocols/src/webdav/config.rs

@@ -23,6 +23,10 @@ pub enum WebDavInitError {
     Bind(#[from] std::io::Error),
     #[error("server error: {0}")]
     Server(String),
+    #[error("WebDAV background task failed: {0}")]
+    TaskJoin(#[from] tokio::task::JoinError),
+    #[error("WebDAV connections did not drain before the shutdown deadline")]
+    ShutdownDeadline,
     #[error("invalid WebDAV configuration: {0}")]
     InvalidConfig(String),
     #[error("TLS error: {0}")]

crates/protocols/src/webdav/server.rs

@@ -19,8 +19,12 @@ use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext, is_tem
 use bytes::Bytes;
 use dav_server::DavHandler;
 use dav_server::fakels::FakeLs;
-use http::header::{AUTHORIZATION, REFERER, USER_AGENT};
-use http::{HeaderMap, HeaderValue};
+use http::header::{
+    ACCESS_CONTROL_ALLOW_CREDENTIALS, ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN,
+    ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, AUTHORIZATION, ORIGIN, REFERER,
+    USER_AGENT, VARY,
+};
+use http::{HeaderMap, HeaderValue, Method};
 use http_body_util::{BodyExt, Full, LengthLimitError, Limited};
 use hyper::body::Body as HttpBody;
 use hyper::server::conn::http1;
@@ -31,18 +35,23 @@ use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_
 use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
 use rustfs_utils::MaskedAccessKey;
 use std::convert::Infallible;
+use std::future::Future;
 use std::io;
 use std::net::IpAddr;
 use std::pin::Pin;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
 use subtle::ConstantTimeEq;
 use tokio::net::TcpListener;
 use tokio::sync::{Semaphore, broadcast, watch};
+use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_rustls::TlsAcceptor;
 use tracing::{Instrument, debug, error, info, info_span, warn};
 
+const CONNECTION_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
+
 const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
 const LOG_SUBSYSTEM_WEBDAV_SERVER: &str = "webdav_server";
 const LOG_SUBSYSTEM_WEBDAV_AUTH: &str = "webdav_auth";
@@ -60,6 +69,177 @@ const EVENT_WEBDAV_CONNECTION_CAP_STATE: &str = "webdav_connection_cap_state";
 /// materialise a whole object in memory for every GET.
 type WebDavBody = Pin<Box<dyn HttpBody<Data = Bytes, Error = io::Error> + Send>>;
 
+/// Listener-wide browser policy, parsed once and shared across connections.
+enum WebDavCors {
+    Disabled,
+    Wildcard,
+    Origins(Vec<HeaderValue>),
+}
+
+impl WebDavCors {
+    const METHODS: &str = "OPTIONS, GET, HEAD, PUT, DELETE, PATCH, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK";
+
+    fn new(origins: &str) -> Self {
+        match origins.trim() {
+            "" => Self::Disabled,
+            "*" => Self::Wildcard,
+            origins => Self::Origins(
+                origins
+                    .split(',')
+                    .map(str::trim)
+                    .filter(|origin| !origin.is_empty() && *origin != "*")
+                    .filter_map(|origin| HeaderValue::from_str(origin).ok())
+                    .collect(),
+            ),
+        }
+    }
+
+    fn allowed_origin(&self, headers: &HeaderMap) -> Option<HeaderValue> {
+        if matches!(self, Self::Disabled) {
+            return None;
+        }
+        let mut origins = headers.get_all(ORIGIN).iter();
+        let origin = origins.next()?;
+        let value = origin.to_str().ok()?;
+        if origins.next().is_some() || value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace() || byte == b',') {
+            return None;
+        }
+        match self {
+            Self::Origins(allowed) => allowed.iter().find(|allowed| *allowed == origin).cloned(),
+            Self::Wildcard => Some(HeaderValue::from_static("*")),
+            Self::Disabled => None,
+        }
+    }
+
+    /// Keep the CORS boundary outside authentication and DAV dispatch so every
+    /// application response, including early errors, follows the same policy.
+    async fn handle<B, F, Fut>(&self, req: Request<B>, next: F) -> Result<Response<WebDavBody>, Infallible>
+    where
+        F: FnOnce(Request<B>) -> Fut,
+        Fut: Future<Output = Result<Response<WebDavBody>, Infallible>>,
+    {
+        let allow_origin = self.allowed_origin(req.headers());
+        let is_options = req.method() == Method::OPTIONS;
+        let is_preflight =
+            is_options && req.headers().contains_key(ORIGIN) && req.headers().contains_key(ACCESS_CONTROL_REQUEST_METHOD);
+
+        let mut response = if is_preflight {
+            self.preflight(req.headers(), allow_origin.is_some())
+        } else {
+            next(req).await?
+        };
+
+        // Absent and rejected Origin values also select cache variants. Append
+        // rather than overwrite the DAV handler's existing Vary fields.
+        response.headers_mut().append(VARY, HeaderValue::from_static("Origin"));
+        if is_options {
+            response.headers_mut().append(
+                VARY,
+                HeaderValue::from_static("Access-Control-Request-Method, Access-Control-Request-Headers"),
+            );
+        }
+        if let Some(origin) = allow_origin {
+            response.headers_mut().insert(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
+            if matches!(self, Self::Origins(_)) {
+                response
+                    .headers_mut()
+                    .insert(ACCESS_CONTROL_ALLOW_CREDENTIALS, HeaderValue::from_static("true"));
+            }
+            if !is_preflight {
+                response.headers_mut().insert(
+                    ACCESS_CONTROL_EXPOSE_HEADERS,
+                    HeaderValue::from_static("ETag, DAV, Allow, Lock-Token, Content-Range, Accept-Ranges, WWW-Authenticate"),
+                );
+            }
+        }
+        Ok(response)
+    }
+
+    fn preflight(&self, headers: &HeaderMap, origin_allowed: bool) -> Response<WebDavBody> {
+        let mut methods = headers.get_all(ACCESS_CONTROL_REQUEST_METHOD).iter();
+        let method = methods.next().and_then(|value| value.to_str().ok());
+        let method_allowed = method.is_some_and(|method| Self::METHODS.split(", ").any(|allowed| allowed == method));
+        if !origin_allowed || !method_allowed || methods.next().is_some() {
+            return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
+        }
+        let Ok(allowed_headers) = cors_requested_headers(headers) else {
+            return error_response(StatusCode::FORBIDDEN, "CORS preflight denied");
+        };
+
+        let mut response = Response::new(fixed_body(Bytes::new()));
+        *response.status_mut() = StatusCode::NO_CONTENT;
+        response
+            .headers_mut()
+            .insert(ACCESS_CONTROL_ALLOW_METHODS, HeaderValue::from_static(Self::METHODS));
+        // A wildcard does not cover Authorization, and is not a wildcard for
+        // credentialed requests. Echo the validated names explicitly instead.
+        if let Some(value) = allowed_headers {
+            response.headers_mut().insert(ACCESS_CONTROL_ALLOW_HEADERS, value);
+        }
+        response
+    }
+}
+
+/// Validate borrowed field names, reusing an ordinary single header value.
+/// RFC 9110 section 5.6.1.2 requires recipients to ignore empty list elements;
+/// normalize those and repeated fields into one nonempty response list.
+fn cors_requested_headers(headers: &HeaderMap) -> Result<Option<HeaderValue>, ()> {
+    let values = headers.get_all(ACCESS_CONTROL_REQUEST_HEADERS);
+    let mut fields = values.iter();
+    let Some(first) = fields.next() else {
+        return Ok(None);
+    };
+    let mut normalize = fields.next().is_some();
+    let mut has_name = false;
+    for value in values.iter() {
+        for name in value.to_str().map_err(|_| ())?.split(',') {
+            let name = name.trim_matches([' ', '\t']);
+            if name.is_empty() {
+                normalize = true;
+                continue;
+            }
+            if !is_cors_header_name(name) {
+                return Err(());
+            }
+            has_name = true;
+        }
+    }
+    if !has_name {
+        return Err(());
+    }
+    if !normalize {
+        return Ok(Some(first.clone()));
+    }
+
+    let capacity = values.iter().map(|value| value.as_bytes().len() + 1).sum();
+    let mut normalized = String::with_capacity(capacity);
+    for value in values {
+        for name in value.to_str().map_err(|_| ())?.split(',') {
+            let name = name.trim_matches([' ', '\t']);
+            if name.is_empty() {
+                continue;
+            }
+            if !normalized.is_empty() {
+                normalized.push(',');
+            }
+            normalized.push_str(name);
+        }
+    }
+    HeaderValue::try_from(normalized).map(Some).map_err(|_| ())
+}
+
+/// RFC 9110 field-name is a nonempty sequence of ASCII tchar bytes.
+fn is_cors_header_name(name: &str) -> bool {
+    !name.is_empty()
+        && name.bytes().all(|byte| {
+            byte.is_ascii_alphanumeric()
+                || matches!(
+                    byte,
+                    b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~'
+                )
+        })
+}
+
 fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
     let mut policy_headers = HeaderMap::new();
     for name in [USER_AGENT, REFERER] {
@@ -74,6 +254,11 @@ fn policy_request_headers(headers: &HeaderMap) -> HeaderMap {
     policy_headers
 }
 
+struct ConnectionLimits {
+    max_body_size: u64,
+    request_timeout: Duration,
+}
+
 /// WebDAV server implementation
 pub struct WebDavServer<S>
 where
@@ -83,6 +268,7 @@ where
     config: WebDavConfig,
     /// S3 storage backend
     storage: S,
+    cors: Arc<WebDavCors>,
 }
 
 impl<S> WebDavServer<S>
@@ -100,7 +286,13 @@ where
     /// Create a new WebDAV server
     pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
         config.validate().await?;
-        Ok(Self { config, storage })
+        let origins =
+            rustfs_utils::get_env_str(rustfs_config::ENV_CORS_ALLOWED_ORIGINS, rustfs_config::DEFAULT_CORS_ALLOWED_ORIGINS);
+        Ok(Self {
+            config,
+            storage,
+            cors: Arc::new(WebDavCors::new(&origins)),
+        })
     }
 
     /// Start the WebDAV server
@@ -135,7 +327,8 @@ where
             bind_addr = %self.config.bind_addr,
             "WebDAV server listening"
         );
-        let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
+        let (shutdown_tx, shutdown) = watch::channel(false);
+        let mut reload_task = None;
 
         // Setup TLS if enabled
         let tls_acceptor = if self.config.tls_enabled {
@@ -151,15 +344,9 @@ where
 
                 let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
                     .map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?;
-                let _reload_task = spawn_server_cert_reload_loop(
-                    "webdav",
-                    resolver.clone(),
-                    Self::tls_reload_options(),
-                    reload_shutdown_rx.clone(),
-                );
-
-                let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
+                let server_config = crate::common::tls::server_config(resolver.clone(), self.config.ca_file.as_deref())
                     .map_err(|error| WebDavInitError::Tls(format!("Invalid client CA configuration: {error}")))?;
+                reload_task = spawn_server_cert_reload_loop("webdav", resolver, Self::tls_reload_options(), shutdown.clone());
 
                 Some(TlsAcceptor::from(Arc::new(server_config)))
             } else {
@@ -173,7 +360,15 @@ where
         let request_timeout = Duration::from_secs(self.config.request_timeout_secs);
         let connection_limiter = Arc::new(Semaphore::new(self.config.max_connections.min(Semaphore::MAX_PERMITS)));
 
-        loop {
+        let mut connections = JoinSet::new();
+        let mut failure = None;
+        'accept: loop {
+            while let Some(result) = connections.try_join_next() {
+                if let Err(error) = result {
+                    failure = Some(WebDavInitError::TaskJoin(error));
+                    break 'accept;
+                }
+            }
             // Admission control: hold a permit before accepting, so at
             // saturation the kernel backlog absorbs the burst instead of the
             // process spawning an unbounded number of connection tasks. The
@@ -204,7 +399,6 @@ where
                                 state = "shutdown_requested",
                                 "WebDAV shutdown requested"
                             );
-                            let _ = reload_shutdown_tx.send(true);
                             break;
                         }
                     }
@@ -216,25 +410,35 @@ where
                     match accept_result {
                         Ok((stream, addr)) => {
                             let storage = storage.clone();
+                            let cors = self.cors.clone();
                             let tls_acceptor = tls_acceptor.clone();
 
-                            let max_body_size = self.config.max_body_size;
+                            let limits = ConnectionLimits {
+                                max_body_size: self.config.max_body_size,
+                                request_timeout,
+                            };
                             let source_ip: IpAddr = addr.ip();
                             let span = info_span!(
                                 "webdav-connection",
                                 peer = %source_ip,
                                 transport = if tls_acceptor.is_some() { "tls" } else { "tcp" },
                             );
-                            tokio::spawn(
+                            let mut shutdown = shutdown.clone();
+                            connections.spawn(
                                 async move {
                                     let _permit = permit;
                                     if let Some(acceptor) = tls_acceptor {
                                         // A handshake that never completes would otherwise
                                         // hold its connection permit forever.
-                                        match timeout(request_timeout, acceptor.accept(stream)).await {
+                                        let handshake = tokio::select! {
+                                            biased;
+                                            _ = shutdown.changed() => return,
+                                            result = timeout(request_timeout, acceptor.accept(stream)) => result,
+                                        };
+                                        match handshake {
                                             Ok(Ok(tls_stream)) => {
                                                 let io = TokioIo::new(tls_stream);
-                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, max_body_size, request_timeout).await {
+                                                if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, true, limits, cors, shutdown).await {
                                                     debug!(
                                                         event = EVENT_WEBDAV_CONNECTION_STATE,
                                                         component = LOG_COMPONENT_PROTOCOLS,
@@ -272,7 +476,7 @@ where
                                         }
                                     } else {
                                         let io = TokioIo::new(stream);
-                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, max_body_size, request_timeout).await {
+                                        if let Err(e) = Self::handle_connection_impl(io, storage, source_ip, false, limits, cors, shutdown).await {
                                             debug!(
                                                 event = EVENT_WEBDAV_CONNECTION_STATE,
                                                 component = LOG_COMPONENT_PROTOCOLS,
@@ -309,13 +513,29 @@ where
                         state = "shutdown_requested",
                         "WebDAV shutdown requested"
                     );
-                    let _ = reload_shutdown_tx.send(true);
                     break;
                 }
             }
         }
 
-        let _ = reload_shutdown_tx.send(true);
+        drop(listener);
+        shutdown_tx.send_replace(true);
+        let drain = async {
+            while let Some(result) = connections.join_next().await {
+                if let Err(error) = result {
+                    failure.get_or_insert(WebDavInitError::TaskJoin(error));
+                }
+            }
+        };
+        if timeout(CONNECTION_DRAIN_TIMEOUT, drain).await.is_err() {
+            connections.shutdown().await;
+            failure.get_or_insert(WebDavInitError::ShutdownDeadline);
+        }
+        if let Some(task) = reload_task
+            && let Err(error) = task.await
+        {
+            failure.get_or_insert(WebDavInitError::TaskJoin(error));
+        }
         info!(
             event = EVENT_WEBDAV_SERVER_STATE,
             component = LOG_COMPONENT_PROTOCOLS,
@@ -323,7 +543,7 @@ where
             state = "stopped",
             "WebDAV server stopped"
         );
-        Ok(())
+        failure.map_or(Ok(()), Err)
     }
 
     /// Handle a single connection with hyper-util TokioIo wrapper
@@ -332,25 +552,49 @@ where
         storage: S,
         source_ip: IpAddr,
         secure_transport: bool,
-        max_body_size: u64,
-        request_timeout: Duration,
+        limits: ConnectionLimits,
+        cors: Arc<WebDavCors>,
+        mut shutdown: watch::Receiver<bool>,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
     where
         I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
     {
-        let service = service_fn(move |req: Request<hyper::body::Incoming>| {
-            let storage = storage.clone();
-            async move { Self::handle_request(req, storage, source_ip, secure_transport, max_body_size, request_timeout).await }
+        let ConnectionLimits {
+            max_body_size,
+            request_timeout,
+        } = limits;
+        // These borrows live for the connection. Preflights complete before
+        // invoking the closure that clones the backend for an actual request.
+        let cors = cors.as_ref();
+        let storage = &storage;
+        let received_request = AtomicBool::new(false);
+        let service = service_fn(|req: Request<hyper::body::Incoming>| {
+            received_request.store(true, Ordering::Relaxed);
+            cors.handle(req, move |req| {
+                Self::handle_request(req, storage.clone(), source_ip, secure_transport, max_body_size, request_timeout)
+            })
         });
 
         // A peer that opens a connection and dribbles (or never finishes)
         // request headers is disconnected once the configured request
         // timeout elapses. The timer is required for the deadline to apply.
-        http1::Builder::new()
+        let connection = http1::Builder::new()
             .timer(TokioTimer::new())
             .header_read_timeout(request_timeout)
-            .serve_connection(io, service)
-            .await?;
+            .serve_connection(io, service);
+        tokio::pin!(connection);
+        tokio::select! {
+            biased;
+            _ = shutdown.changed() => {
+                // Hyper starts in a busy state even before parsing the first
+                // request. An incomplete header has no application work to drain.
+                if received_request.load(Ordering::Relaxed) {
+                    connection.as_mut().graceful_shutdown();
+                    connection.await?;
+                }
+            }
+            result = &mut connection => result?,
+        }
 
         Ok(())
     }
@@ -693,6 +937,7 @@ mod tests {
     use s3s::dto::*;
     use std::fmt::{Debug, Formatter};
     use std::net::{Ipv4Addr, SocketAddr};
+    use std::sync::atomic::{AtomicUsize, Ordering};
     use std::task::{Context, Poll};
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use tokio::net::TcpStream;
@@ -701,8 +946,19 @@ mod tests {
 
     /// Storage double for the connection-level tests. Those requests are
     /// rejected before authentication succeeds, so storage is never reached.
-    #[derive(Clone)]
-    struct StubStorage;
+    #[derive(Default)]
+    struct StubStorage {
+        clone_count: Arc<AtomicUsize>,
+    }
+
+    impl Clone for StubStorage {
+        fn clone(&self) -> Self {
+            self.clone_count.fetch_add(1, Ordering::Relaxed);
+            Self {
+                clone_count: self.clone_count.clone(),
+            }
+        }
+    }
 
     impl Debug for StubStorage {
         fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
@@ -873,6 +1129,362 @@ mod tests {
             .expect("build get request")
     }
 
+    const BROWSER_ORIGIN: &str = "https://dav-console.example";
+
+    async fn cors_http_request(origins: &str, method: &str, headers: &str) -> String {
+        cors_http_exchange(origins, method, headers).await.0
+    }
+
+    async fn cors_http_exchange(origins: &str, method: &str, headers: &str) -> (String, usize) {
+        let storage = StubStorage::default();
+        let clone_count = storage.clone_count.clone();
+        let (mut client, server) = tokio::io::duplex(8192);
+        let (_shutdown_tx, shutdown) = tokio::sync::watch::channel(false);
+        let connection = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
+            TokioIo::new(server),
+            storage,
+            TEST_IP,
+            false,
+            ConnectionLimits {
+                max_body_size: 16,
+                request_timeout: Duration::from_secs(30),
+            },
+            Arc::new(WebDavCors::new(origins)),
+            shutdown,
+        ));
+        let request = format!("{method} / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n{headers}\r\n");
+        client.write_all(request.as_bytes()).await.expect("write browser request");
+        let mut response = Vec::new();
+        timeout(Duration::from_secs(5), client.read_to_end(&mut response))
+            .await
+            .expect("browser response timeout")
+            .expect("read browser response");
+        timeout(Duration::from_secs(5), connection)
+            .await
+            .expect("connection shutdown timeout")
+            .expect("connection task")
+            .expect("serve browser request");
+        (
+            String::from_utf8(response).expect("ASCII HTTP response"),
+            clone_count.load(Ordering::Relaxed),
+        )
+    }
+
+    fn browser_request(method: &str, path: &str) -> Request<Full<Bytes>> {
+        Request::builder()
+            .method(method)
+            .uri(path)
+            .header(ORIGIN, BROWSER_ORIGIN)
+            .body(Full::new(Bytes::new()))
+            .expect("build browser request")
+    }
+
+    fn assert_browser_headers(response: &Response<WebDavBody>) {
+        assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_ORIGIN], BROWSER_ORIGIN);
+        assert_eq!(response.headers()[ACCESS_CONTROL_ALLOW_CREDENTIALS], "true");
+        assert!(response.headers().get_all(VARY).iter().any(|value| value == "Origin"));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_preflight_bypasses_authentication() {
+        for method in [
+            "OPTIONS",
+            "GET",
+            "HEAD",
+            "PUT",
+            "DELETE",
+            "PATCH",
+            "PROPFIND",
+            "PROPPATCH",
+            "MKCOL",
+            "COPY",
+            "MOVE",
+            "LOCK",
+            "UNLOCK",
+        ] {
+            let headers = format!(
+                "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: {method}\r\nAccess-Control-Request-Headers: authorization,depth,content-type,destination,if-match\r\n"
+            );
+            let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
+            assert!(response.starts_with("HTTP/1.1 204"), "{method}: {response}");
+            assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
+            assert!(response.contains("access-control-allow-credentials: true\r\n"));
+            let allowed_methods = response
+                .lines()
+                .find_map(|line| line.strip_prefix("access-control-allow-methods: "))
+                .expect("preflight must advertise allowed methods");
+            assert!(allowed_methods.split(',').map(str::trim).any(|allowed| allowed == method));
+            assert!(response.contains("access-control-allow-headers: authorization,depth,content-type,destination,if-match\r\n"));
+            assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
+            assert!(!response.contains("www-authenticate:"));
+        }
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_actual_requests_and_plain_options_require_authentication() {
+        for method in ["GET", "PUT", "DELETE", "PROPFIND", "MOVE", "OPTIONS"] {
+            let response = cors_http_request(BROWSER_ORIGIN, method, &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
+            assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
+            assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
+            assert!(response.contains("www-authenticate: Basic"));
+        }
+        for method in ["GET", "PROPFIND"] {
+            for headers in ["", "Origin: https://other.example\r\n"] {
+                let response = cors_http_request(BROWSER_ORIGIN, method, headers).await;
+                assert!(response.starts_with("HTTP/1.1 401"), "{method}: {response}");
+                assert!(!response.contains("access-control-allow-origin:"));
+                assert!(response.contains("vary: Origin\r\n"));
+            }
+        }
+        for headers in [
+            format!("Origin: {BROWSER_ORIGIN}\r\nAuthorization: Basic !!!!\r\n"),
+            format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PUT\r\n"),
+        ] {
+            let response = cors_http_request(BROWSER_ORIGIN, "PUT", &headers).await;
+            assert!(response.starts_with("HTTP/1.1 401"));
+            assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
+        }
+        let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", "Access-Control-Request-Method: PROPFIND\r\n").await;
+        assert!(response.starts_with("HTTP/1.1 401"));
+        assert!(!response.contains("access-control-allow-origin:"));
+        assert!(response.contains("vary: Origin\r\n"));
+        assert!(response.contains("vary: Access-Control-Request-Method, Access-Control-Request-Headers\r\n"));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_keeps_early_body_limit_errors_visible() {
+        let response =
+            cors_http_request(BROWSER_ORIGIN, "PUT", &format!("Origin: {BROWSER_ORIGIN}\r\nContent-Length: 17\r\n")).await;
+        assert!(response.starts_with("HTTP/1.1 413"));
+        assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_rejects_unconfigured_and_unmatched_origins() {
+        for (configuration, origin) in [
+            ("", BROWSER_ORIGIN),
+            ("  ", BROWSER_ORIGIN),
+            (BROWSER_ORIGIN, "https://dav-console.example.attacker.test"),
+            (BROWSER_ORIGIN, "null"),
+            (BROWSER_ORIGIN, "https://other.example"),
+            ("*, https://other.example", BROWSER_ORIGIN),
+        ] {
+            let headers = format!("Origin: {origin}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
+            let response = cors_http_request(configuration, "OPTIONS", &headers).await;
+            assert!(response.starts_with("HTTP/1.1 403"), "{configuration:?}: {response}");
+            assert!(!response.contains("access-control-allow-origin:"));
+            assert!(!response.contains("access-control-allow-credentials:"));
+            assert!(response.contains("vary: Origin\r\n"));
+        }
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_rejects_malformed_preflights() {
+        for headers in [
+            format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: TRACE\r\n"),
+            format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: propfind\r\n"),
+            format!(
+                "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Method: GET\r\n"
+            ),
+            format!("Origin: {BROWSER_ORIGIN}\r\nOrigin: https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
+            format!("Origin: {BROWSER_ORIGIN},https://other.example\r\nAccess-Control-Request-Method: PROPFIND\r\n"),
+            format!(
+                "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: invalid header\r\n"
+            ),
+            format!(
+                "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: , \t,\r\n"
+            ),
+        ] {
+            let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
+            assert!(response.starts_with("HTTP/1.1 403"), "{headers}: {response}");
+            assert!(!response.contains("access-control-allow-methods:"));
+        }
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_normalizes_header_lists() {
+        for requested in [
+            "authorization, ,depth,",
+            " , authorization,\tdepth , ",
+            "authorization,\r\nAccess-Control-Request-Headers: , depth,",
+        ] {
+            let headers = format!(
+                "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: {requested}\r\n"
+            );
+            let response = cors_http_request(BROWSER_ORIGIN, "OPTIONS", &headers).await;
+            assert!(response.starts_with("HTTP/1.1 204"), "{response}");
+            assert!(response.contains("access-control-allow-headers: authorization,depth\r\n"));
+        }
+    }
+
+    #[test]
+    fn webdav_cors_header_name_validation_matches_http_tokens() {
+        for byte in u8::MIN..=u8::MAX {
+            let bytes = [b'x', byte, b'y'];
+            let actual = std::str::from_utf8(&bytes).is_ok_and(is_cors_header_name);
+            assert_eq!(actual, http::HeaderName::from_bytes(&bytes).is_ok(), "byte {byte}");
+        }
+        assert!(!is_cors_header_name(""));
+        assert!(is_cors_header_name("Authorization"));
+        assert!(is_cors_header_name("!#$%&'*+-.^_`|~"));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_only_clones_backend_for_actual_requests() {
+        for (method, requested_method, expected_clones, expected_status) in [
+            ("OPTIONS", Some("PROPFIND"), 0, "204"),
+            ("OPTIONS", Some("TRACE"), 0, "403"),
+            ("OPTIONS", None, 1, "401"),
+            ("PUT", Some("PUT"), 1, "401"),
+        ] {
+            let mut headers = format!("Origin: {BROWSER_ORIGIN}\r\n");
+            if let Some(requested_method) = requested_method {
+                headers.push_str(&format!("Access-Control-Request-Method: {requested_method}\r\n"));
+            }
+            let (response, clones) = cors_http_exchange(BROWSER_ORIGIN, method, &headers).await;
+            assert!(response.starts_with(&format!("HTTP/1.1 {expected_status}")), "{response}");
+            assert_eq!(clones, expected_clones, "{method} with {requested_method:?}");
+        }
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_wildcard_never_allows_credentials() {
+        let headers = format!(
+            "Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\nAccess-Control-Request-Headers: Authorization\r\n"
+        );
+        let preflight = cors_http_request(" * ", "OPTIONS", &headers).await;
+        assert!(preflight.starts_with("HTTP/1.1 204"));
+        assert!(preflight.contains("access-control-allow-origin: *\r\n"));
+        assert!(preflight.contains("access-control-allow-headers: Authorization\r\n"));
+        assert!(!preflight.contains("access-control-allow-credentials:"));
+        let actual = cors_http_request("*", "PROPFIND", &format!("Origin: {BROWSER_ORIGIN}\r\n")).await;
+        assert!(actual.starts_with("HTTP/1.1 401"));
+        assert!(actual.contains("access-control-allow-origin: *\r\n"));
+        assert!(!actual.contains("access-control-allow-credentials:"));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_allowlist_trims_entries_and_matches_exactly() {
+        let configuration = format!(" https://other.example , {BROWSER_ORIGIN} ");
+        let headers = format!("Origin: {BROWSER_ORIGIN}\r\nAccess-Control-Request-Method: PROPFIND\r\n");
+        let response = cors_http_request(&configuration, "OPTIONS", &headers).await;
+        assert!(response.starts_with("HTTP/1.1 204"));
+        assert!(response.contains(&format!("access-control-allow-origin: {BROWSER_ORIGIN}\r\n")));
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_wraps_dav_multistatus_etag_and_streamed_body() {
+        let cors = WebDavCors::new(BROWSER_ORIGIN);
+        let handler = memfs_handler();
+        let created = dispatch_dav(chunked_request(2, 4), handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await;
+        assert_eq!(created.status(), StatusCode::CREATED);
+
+        let propfind = cors
+            .handle(browser_request("PROPFIND", "/"), |req| async {
+                Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
+            })
+            .await
+            .expect("PROPFIND response");
+        assert_eq!(propfind.status(), StatusCode::MULTI_STATUS);
+        assert_browser_headers(&propfind);
+
+        let get = cors
+            .handle(browser_request("GET", "/upload.bin"), |req| async {
+                Ok(dispatch_dav(req, handler.clone(), TEST_IP, 16, Duration::from_secs(30)).await)
+            })
+            .await
+            .expect("GET response");
+        assert_eq!(get.status(), StatusCode::OK);
+        assert_browser_headers(&get);
+        assert!(get.headers().contains_key("etag"));
+        assert!(
+            get.headers()[ACCESS_CONTROL_EXPOSE_HEADERS]
+                .to_str()
+                .expect("exposed headers")
+                .contains("ETag")
+        );
+        assert!(get.body().size_hint().upper().is_none(), "CORS must preserve streaming");
+        let data = get.into_body().collect().await.expect("stream GET body").to_bytes();
+        assert_eq!(data.as_ref(), b"aaaaaaaa");
+
+        let mut put = browser_request("PUT", "/upload.bin");
+        put.headers_mut()
+            .insert("if-match", HeaderValue::from_static("\"stale-etag\""));
+        let conditional = cors
+            .handle(put, |req| async {
+                Ok(dispatch_dav(req, handler, TEST_IP, 16, Duration::from_secs(30)).await)
+            })
+            .await
+            .expect("conditional PUT response");
+        assert_eq!(conditional.status(), StatusCode::PRECONDITION_FAILED);
+        assert_browser_headers(&conditional);
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_wraps_dispatch_body_limit_errors() {
+        let cors = WebDavCors::new(BROWSER_ORIGIN);
+        let mut request = chunked_request(8, 8);
+        request.headers_mut().insert(ORIGIN, HeaderValue::from_static(BROWSER_ORIGIN));
+        let oversized = cors
+            .handle(request, |req| async {
+                Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
+            })
+            .await
+            .expect("oversized response");
+        assert_eq!(oversized.status(), StatusCode::PAYLOAD_TOO_LARGE);
+        assert_browser_headers(&oversized);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn webdav_cors_wraps_dispatch_timeout_errors() {
+        let cors = WebDavCors::new(BROWSER_ORIGIN);
+        let request = Request::builder()
+            .method("PUT")
+            .uri("/stalled.bin")
+            .header(ORIGIN, BROWSER_ORIGIN)
+            .body(StalledBody)
+            .expect("stalled browser request");
+        let response = cors
+            .handle(request, |req| async {
+                Ok(dispatch_dav(req, memfs_handler(), TEST_IP, 16, Duration::from_secs(30)).await)
+            })
+            .await
+            .expect("timeout response");
+        assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT);
+        assert_browser_headers(&response);
+    }
+
+    #[tokio::test]
+    async fn webdav_cors_preserves_existing_vary_on_success_and_errors() {
+        let cors = WebDavCors::new(BROWSER_ORIGIN);
+        for status in [
+            StatusCode::OK,
+            StatusCode::FORBIDDEN,
+            StatusCode::NOT_FOUND,
+            StatusCode::INTERNAL_SERVER_ERROR,
+        ] {
+            let response = cors
+                .handle(browser_request("GET", "/"), |_| async {
+                    let mut response = error_response(status, "test response");
+                    response
+                        .headers_mut()
+                        .append(VARY, HeaderValue::from_static("Accept-Encoding"));
+                    Ok(response)
+                })
+                .await
+                .expect("wrapped response");
+            assert_eq!(response.status(), status);
+            assert_browser_headers(&response);
+            assert!(
+                response
+                    .headers()
+                    .get_all(VARY)
+                    .iter()
+                    .any(|value| value == "Accept-Encoding")
+            );
+        }
+    }
+
     #[test]
     fn policy_headers_drop_credentials_and_s3_auth_spoofing() {
         let mut headers = HeaderMap::new();
@@ -969,13 +1581,18 @@ mod tests {
     async fn slow_request_headers_hit_request_timeout() {
         let (mut client, server) = tokio::io::duplex(1024);
 
+        let (_shutdown_tx, shutdown) = tokio::sync::watch::channel(false);
         let conn = tokio::spawn(WebDavServer::<StubStorage>::handle_connection_impl(
             TokioIo::new(server),
-            StubStorage,
+            StubStorage::default(),
             TEST_IP,
             false,
-            1024,
-            Duration::from_secs(30),
+            ConnectionLimits {
+                max_body_size: 1024,
+                request_timeout: Duration::from_secs(30),
+            },
+            Arc::new(WebDavCors::new("")),
+            shutdown,
         ));
 
         client
@@ -987,6 +1604,40 @@ mod tests {
         assert!(outcome.is_ok(), "connection outlived the configured request timeout");
     }
 
+    #[tokio::test]
+    async fn shutdown_closes_connection_with_incomplete_first_headers() {
+        use std::future::{Future, poll_fn};
+        use std::task::Poll;
+
+        let (mut client, server) = tokio::io::duplex(1024);
+        let (shutdown_tx, shutdown) = watch::channel(false);
+        let connection = WebDavServer::<StubStorage>::handle_connection_impl(
+            TokioIo::new(server),
+            StubStorage::default(),
+            TEST_IP,
+            false,
+            ConnectionLimits {
+                max_body_size: 1024,
+                request_timeout: Duration::from_secs(30),
+            },
+            Arc::new(WebDavCors::new("")),
+            shutdown,
+        );
+        tokio::pin!(connection);
+        client
+            .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\n")
+            .await
+            .expect("write partial headers");
+        let pending = poll_fn(|cx| Poll::Ready(connection.as_mut().poll(cx).is_pending())).await;
+        assert!(pending, "connection should be waiting for the rest of the first header");
+        shutdown_tx.send_replace(true);
+
+        timeout(Duration::from_secs(1), connection.as_mut())
+            .await
+            .expect("shutdown should not wait for an incomplete header")
+            .expect("no request reached the service");
+    }
+
     /// R05-CAN-097: the accept loop must not serve more connections at once
     /// than the configured cap.
     #[tokio::test]
@@ -1000,7 +1651,7 @@ mod tests {
             request_timeout_secs: 30,
             max_connections: 1,
         };
-        let server = WebDavServer::new(config, StubStorage).await.expect("build server");
+        let server = WebDavServer::new(config, StubStorage::default()).await.expect("build server");
 
         let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.expect("bind listener");
         let addr = listener.local_addr().expect("listener addr");
@@ -1035,8 +1686,16 @@ mod tests {
             .expect("read second");
         assert!(served > 0, "second connection returned no data");
 
-        let _ = shutdown_tx.send(());
-        let _ = timeout(Duration::from_secs(10), serving).await;
+        shutdown_tx.send(()).expect("signal server");
+        timeout(Duration::from_secs(5), serving)
+            .await
+            .expect("server did not retire the keep-alive connection")
+            .expect("server task panicked")
+            .expect("server did not shut down cleanly");
+        let remaining = timeout(Duration::from_secs(5), second.read_to_end(&mut Vec::new()))
+            .await
+            .expect("connection remained open after server exit");
+        remaining.expect("read remaining response and EOF");
     }
 
     #[tokio::test]

crates/protocols/tests/swift_metadata_persistence.rs

@@ -234,13 +234,18 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
     env.make_bucket(&ContainerMapper::default().swift_to_s3_bucket(archive, project_id), false)
         .await;
 
-    update_container_metadata(&swift_account, container, &credentials, MetadataUpdate::default().set("color", "blue"))
-        .await
-        .expect("container metadata POST should succeed");
+    update_container_metadata(
+        &swift_account,
+        container,
+        &credentials,
+        MetadataUpdate::default().set("color", "blue,green"),
+    )
+    .await
+    .expect("container metadata POST should succeed");
     container::enable_versioning(&swift_account, container, archive, &credentials)
         .await
         .expect("enable versioning should succeed");
-    container::set_container_acl(&swift_account, container, Some(".r:*"), Some("AUTH_other"), &credentials)
+    container::set_container_acl(&swift_account, container, Some(".r:*,AUTH_other"), Some("AUTH_other"), &credentials)
         .await
         .expect("set container ACL should succeed");
 
@@ -248,7 +253,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
 
     // All three writers' state coexists after a disk-truth reload.
     let meta = persisted_container_metadata(&bucket).await;
-    assert_eq!(meta.get("color").map(String::as_str), Some("blue"));
+    assert_eq!(meta.get("color").map(String::as_str), Some("blue,green"));
     assert_eq!(
         container::get_versions_location(&swift_account, container, &credentials)
             .await
@@ -259,7 +264,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
     let acl = container::get_container_acl(&swift_account, container, &credentials)
         .await
         .expect("container ACL should load");
-    assert!(!acl.read.is_empty(), "read ACL must survive the reload");
+    assert!(acl.check_read_access(None, None, None), "public read ACL must survive the reload");
     assert!(!acl.write.is_empty(), "write ACL must survive the reload");
 
     // Disabling versioning drops only the versioning tag.
@@ -279,7 +284,7 @@ async fn tag_writers_preserve_each_others_state(env: &TestECStoreEnv) {
     let meta = persisted_container_metadata(&bucket).await;
     assert_eq!(
         meta.get("color").map(String::as_str),
-        Some("blue"),
+        Some("blue,green"),
         "disable_versioning must not disturb container metadata"
     );
     let acl = container::get_container_acl(&swift_account, container, &credentials)

crates/protos/src/generated/proto_gen/node_service.rs

@@ -924,6 +924,9 @@ pub struct GenerallyLockResponse {
     /// JSON serialized LockInfo
     #[prost(string, optional, tag = "3")]
     pub lock_info: ::core::option::Option<::prost::alloc::string::String>,
+    /// Only populated by unlock RPCs.
+    #[prost(enumeration = "LockReleaseStatus", tag = "4")]
+    pub release_status: i32,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct BatchGenerallyLockRequest {
@@ -939,6 +942,9 @@ pub struct GenerallyLockResult {
     /// JSON serialized LockInfo
     #[prost(string, optional, tag = "3")]
     pub lock_info: ::core::option::Option<::prost::alloc::string::String>,
+    /// Per-ID result for unlock batches.
+    #[prost(enumeration = "LockReleaseStatus", tag = "4")]
+    pub release_status: i32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct BatchGenerallyLockResponse {
@@ -1875,6 +1881,37 @@ impl CompareAndUpdateFileOutcome {
         }
     }
 }
+/// Authoritative unlock result for cleanup of a previously confirmed grant.
+/// Older peers omit this field. Unspecified does not confirm absence.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum LockReleaseStatus {
+    Unspecified = 0,
+    Released = 1,
+    AlreadyAbsent = 2,
+}
+impl LockReleaseStatus {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unspecified => "LOCK_RELEASE_STATUS_UNSPECIFIED",
+            Self::Released => "LOCK_RELEASE_STATUS_RELEASED",
+            Self::AlreadyAbsent => "LOCK_RELEASE_STATUS_ALREADY_ABSENT",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "LOCK_RELEASE_STATUS_UNSPECIFIED" => Some(Self::Unspecified),
+            "LOCK_RELEASE_STATUS_RELEASED" => Some(Self::Released),
+            "LOCK_RELEASE_STATUS_ALREADY_ABSENT" => Some(Self::AlreadyAbsent),
+            _ => None,
+        }
+    }
+}
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
 pub enum TierMutationPeerState {

crates/protos/src/lib.rs

@@ -4090,3 +4090,63 @@ mod tests {
         evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
     }
 }
+
+#[cfg(test)]
+mod lock_release_tests {
+    use super::proto_gen::node_service::{GenerallyLockResponse, GenerallyLockResult, LockReleaseStatus};
+    use prost::Message;
+
+    /// The release response schema before structured outcomes were added.
+    #[derive(Clone, PartialEq, Message)]
+    struct LegacyReleaseResponse {
+        #[prost(bool, tag = "1")]
+        success: bool,
+        #[prost(string, optional, tag = "2")]
+        error_info: Option<String>,
+        #[prost(string, optional, tag = "3")]
+        lock_info: Option<String>,
+    }
+
+    #[test]
+    fn legacy_release_response_does_not_confirm_absence() {
+        let legacy = LegacyReleaseResponse {
+            success: false,
+            error_info: Some("lock not found for release".to_string()),
+            lock_info: None,
+        };
+        let encoded = legacy.encode_to_vec();
+        let response = GenerallyLockResponse::decode(encoded.as_slice()).expect("decode legacy release response");
+        let result = GenerallyLockResult::decode(encoded.as_slice()).expect("decode legacy batch entry");
+        assert!(!response.success);
+        assert_eq!(response.error_info, legacy.error_info);
+        assert_eq!(response.release_status(), LockReleaseStatus::Unspecified);
+        assert_eq!(result.release_status(), LockReleaseStatus::Unspecified);
+    }
+
+    #[test]
+    fn structured_release_results_preserve_legacy_success_and_failure() {
+        for (status, success, error) in [
+            (LockReleaseStatus::Released, true, None),
+            (LockReleaseStatus::AlreadyAbsent, false, Some("lock not found for release".to_string())),
+        ] {
+            let response = GenerallyLockResponse {
+                success,
+                error_info: error.clone(),
+                lock_info: None,
+                release_status: status.into(),
+            };
+            let result = GenerallyLockResult {
+                success,
+                error_info: error.clone(),
+                lock_info: None,
+                release_status: status.into(),
+            };
+            for encoded in [response.encode_to_vec(), result.encode_to_vec()] {
+                let legacy = LegacyReleaseResponse::decode(encoded.as_slice()).expect("old peer ignores additive outcome");
+                assert_eq!(legacy.success, success);
+                assert_eq!(legacy.error_info, error);
+                assert!(legacy.lock_info.is_none());
+            }
+        }
+    }
+}

crates/protos/src/node.proto

@@ -643,10 +643,19 @@ message GenerallyLockRequest {
     string args = 1;
 }
 
+// Authoritative unlock result for cleanup of a previously confirmed grant.
+// Older peers omit this field. Unspecified does not confirm absence.
+enum LockReleaseStatus {
+  LOCK_RELEASE_STATUS_UNSPECIFIED = 0;
+  LOCK_RELEASE_STATUS_RELEASED = 1;
+  LOCK_RELEASE_STATUS_ALREADY_ABSENT = 2;
+}
+
 message GenerallyLockResponse {
   bool success = 1;
   optional string error_info = 2;
   optional string lock_info = 3; // JSON serialized LockInfo
+  LockReleaseStatus release_status = 4; // Only populated by unlock RPCs.
 }
 
 message BatchGenerallyLockRequest {
@@ -657,6 +666,7 @@ message GenerallyLockResult {
   bool success = 1;
   optional string error_info = 2;
   optional string lock_info = 3; // JSON serialized LockInfo
+  LockReleaseStatus release_status = 4; // Per-ID result for unlock batches.
 }
 
 message BatchGenerallyLockResponse {

crates/scanner/src/data_usage_define.rs

@@ -1823,7 +1823,12 @@ impl DataUsageCache {
             return;
         };
 
-        if top_e.children.len() > 250_000 {
+        // Wide bucket roots previously only flattened above a hardcoded 250k
+        // fanout, so caches with 10k–250k direct children (common with many
+        // top-level prefixes) never shrank and retained one entry per prefix
+        // after #8971's non-root compaction. Honor `limit` here so completed
+        // sweeps and emergency compaction actually bound root fanout (#8961).
+        if top_e.children.len() > limit {
             self.reduce_children_of(&hash_path(&self.info.name), limit, true);
         }
         if self.cache.len() <= limit {

crates/scanner/src/data_usage_define/tests.rs

@@ -3223,6 +3223,200 @@ fn test_reduce_children_of_usize_underflow_saturates() {
     assert!(cache.find("bucket/large/b").is_some());
 }
 
+#[test]
+fn test_force_compact_flattens_wide_root_above_limit() {
+    // Regression for #8961: a bucket root with fanout between the ordinary
+    // compact limit and the old hardcoded 250_000 ceiling must shrink.
+    // Before the fix, force_compact kept one entry per prefix forever.
+    const PREFIXES: usize = 64;
+    const LIMIT: usize = 16;
+    const {
+        assert!(PREFIXES > LIMIT);
+    }
+
+    let root = hash_path("bucket");
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:04}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                size: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+    assert_eq!(cache.find("bucket").map(|entry| entry.children.len()), Some(PREFIXES));
+    assert_eq!(cache.cache.len(), PREFIXES + 1);
+
+    cache.force_compact(LIMIT);
+
+    let root_entry = cache.find("bucket").expect("root retained");
+    assert!(root_entry.compacted, "wide root must flatten under force_compact");
+    assert!(root_entry.children.is_empty(), "flattened root must drop per-prefix children");
+    assert_eq!((root_entry.objects, root_entry.size), (PREFIXES, PREFIXES));
+    assert_eq!(cache.cache.len(), 1, "prefix leaves must be released after root flatten");
+}
+
+#[test]
+fn test_force_compact_keeps_narrow_root_children() {
+    const PREFIXES: usize = 4;
+    const LIMIT: usize = 16;
+    const {
+        assert!(PREFIXES < LIMIT);
+    }
+
+    let root = hash_path("bucket");
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    cache.replace_hashed(
+        &root,
+        &None,
+        &DataUsageEntry {
+            objects: PREFIXES,
+            ..Default::default()
+        },
+    );
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:04}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+
+    cache.force_compact(LIMIT);
+
+    let root_entry = cache.find("bucket").expect("root retained");
+    assert!(!root_entry.compacted);
+    assert_eq!(root_entry.children.len(), PREFIXES);
+    assert_eq!(cache.cache.len(), PREFIXES + 1);
+}
+
+#[test]
+fn test_force_compact_interrupted_ceiling_retains_wide_root_children() {
+    // Interrupted sweeps pass the emergency FORCE ceiling into force_compact so
+    // a resume frontier under a 10k–250k-wide root stays addressable.
+    const PREFIXES: usize = 64;
+    const ORDINARY_LIMIT: usize = 16;
+    const INTERRUPTED_CEILING: usize = 250_000;
+    const {
+        assert!(PREFIXES > ORDINARY_LIMIT);
+        assert!(PREFIXES < INTERRUPTED_CEILING);
+    }
+
+    let root = hash_path("bucket");
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:04}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                size: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+
+    cache.force_compact(INTERRUPTED_CEILING);
+
+    let root_entry = cache.find("bucket").expect("root retained");
+    assert!(!root_entry.compacted, "interrupted ceiling must not flatten a 64-wide root");
+    assert_eq!(root_entry.children.len(), PREFIXES);
+    assert_eq!(cache.cache.len(), PREFIXES + 1);
+    // Sanity: the ordinary completion limit would have flattened this layout.
+    let mut completed = cache.clone();
+    completed.force_compact(ORDINARY_LIMIT);
+    assert!(
+        completed.find("bucket").is_some_and(|entry| entry.compacted),
+        "ordinary limit still flattens the same wide root after completion"
+    );
+}
+
+#[test]
+fn test_force_compact_interrupted_ceiling_seals_reloaded_frontier() {
+    // The interrupted call runs before seal_scan_frontier. Flattening at the
+    // ordinary limit deletes the child that frontier names, so a saved cache
+    // reloads with no resume position and the next cycle replays finished prefixes.
+    const PREFIXES: usize = 64;
+    const ORDINARY_LIMIT: usize = 16;
+    const INTERRUPTED_CEILING: usize = 250_000;
+    const {
+        assert!(PREFIXES > ORDINARY_LIMIT);
+        assert!(PREFIXES < INTERRUPTED_CEILING);
+    }
+    let frontier = format!("bucket/prefix-{:04}", PREFIXES / 2);
+
+    let mut cache = DataUsageCache::default();
+    cache.prepare_bucket_checkpoint("bucket", 8, 1, DataUsageCacheSource::new(0, 0), TEST_PLAN_DIGEST, valid_scan_identity());
+    let root = hash_path("bucket");
+    cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:04}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                size: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+
+    let mut interrupted = cache.clone();
+    interrupted.force_compact(INTERRUPTED_CEILING);
+    assert!(interrupted.find(&frontier).is_some(), "interrupted ceiling must keep the frontier child");
+    interrupted
+        .seal_scan_frontier(Some(&frontier))
+        .expect("seal frontier after interrupted compaction");
+    let encoded = rmp_serde::to_vec(&interrupted).expect("partial cache should encode");
+    let reloaded: DataUsageCache = rmp_serde::from_slice(&encoded).expect("partial cache should reload");
+    assert_eq!(reloaded.validated_scan_frontier(), Some(frontier.as_str()));
+    assert!(reloaded.find(&frontier).is_some(), "reloaded frontier child must stay addressable");
+
+    cache.force_compact(ORDINARY_LIMIT);
+    assert!(cache.find(&frontier).is_none(), "ordinary limit removes the frontier child");
+    cache
+        .seal_scan_frontier(Some(&frontier))
+        .expect("seal after a completed flatten");
+    assert!(
+        cache.validated_scan_frontier().is_none(),
+        "a flattened root cannot seal the removed frontier child"
+    );
+}
+
 #[tokio::test]
 async fn test_retry_save_op_times_out_and_returns_error_after_retries() {
     let attempts = Arc::new(AtomicUsize::new(0));

crates/scanner/src/lib.rs

@@ -109,10 +109,10 @@ pub use scanner_io::{
     ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
     ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
     acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
-    record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
-    record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
-    record_status_class_mutation, replay_durable_dirty_usage_producer_record, scanner_activity_epoch,
-    scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
+    probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
+    record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
+    record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
+    scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
     scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
 };
 pub use segment_invalidation::SegmentInvalidationProducerIdentity;

crates/scanner/src/scanner_folder.rs

@@ -523,6 +523,18 @@ fn data_usage_entry_has_progress(entry: &DataUsageEntry) -> bool {
     data_usage_root_has_progress(entry)
 }
 
+/// Root-fanout limit for `DataUsageCache::force_compact` at the end of a sweep.
+///
+/// An interrupted sweep still has to seal its resume frontier, so it keeps the
+/// emergency ceiling. A finished walk applies the ordinary child limit.
+fn force_compact_limit_for_sweep(interrupted: bool) -> usize {
+    if interrupted {
+        DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS
+    } else {
+        DATA_SCANNER_COMPACT_AT_CHILDREN
+    }
+}
+
 fn set_scan_checkpoint(cache: &mut DataUsageCache, reason: DataUsageScanCheckpointReason) {
     let resume_after = clone_cache_value(&cache.info.scan_resume_after).or_else(|| {
         cache
@@ -1415,7 +1427,7 @@ impl FolderScanner {
         mode: FolderScanMode,
     ) -> Result<FolderScanOutcome, ScannerError> {
         if self.checkpoint_tx.is_none() || self.old_cache.info.scan_progress.is_none() {
-            return Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode)).await;
+            return Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode, false)).await;
         }
         let depth = self.checkpoint_depth;
         if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
@@ -1429,7 +1441,7 @@ impl FolderScanner {
         std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash);
         std::mem::swap(parent_entry, &mut self.checkpoint_ancestors[depth].entry);
         self.checkpoint_depth += 1;
-        let result = Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode)).await;
+        let result = Box::pin(self.scan_folder_inner(ctx, folder, child_entry, mode, false)).await;
         self.checkpoint_depth -= 1;
         // Restore the parent before cancellation/error handling stitches partial work.
         std::mem::swap(parent_hash, &mut self.checkpoint_ancestors[depth].hash);
@@ -1850,7 +1862,7 @@ impl FolderScanner {
         folder: CachedFolder,
         into: &mut DataUsageEntry,
     ) -> Result<(), ScannerError> {
-        self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace)
+        self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false)
             .await
             .map(|_| ())
     }
@@ -1863,6 +1875,7 @@ impl FolderScanner {
         folder: CachedFolder,
         into: &mut DataUsageEntry,
         mode: FolderScanMode,
+        isolate_compacted_namespace_errors: bool,
     ) -> Result<FolderScanOutcome, ScannerError> {
         let done_folder = Metrics::time(Metric::ScanFolder);
         let trace_started_at = trace_start_instant();
@@ -2533,13 +2546,23 @@ impl FolderScanner {
             }
 
             // Checkpointed sweeps need child entries to validate their resume
-            // frontier. Keep the emergency compaction limit for large trees.
-            let should_compact = (self.new_cache.info.name != folder.name
-                && self.old_cache.info.scan_progress.is_none()
-                && queued_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
-                || queued_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
-
+            // frontier (#8292). Outside that, also compact a wide bucket root:
+            // excluding it left #8961's ~170k top-level prefixes unbounded until
+            // the emergency 250k ceiling. Keep FORCE as the absolute backstop.
+            let is_scan_root = folder.name == self.old_cache.info.name;
+            let forward_sweep = self.old_cache.info.scan_progress.is_some();
             let total_folders = queued_folders.len();
+            let force_compact = total_folders >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
+            // Reused root children still have links in the new cache.
+            let should_compact = force_compact
+                || (!forward_sweep
+                    && total_folders >= DATA_SCANNER_COMPACT_AT_FOLDERS
+                    && (!is_scan_root || into.children.is_empty()));
+            // Preserve the ordinary namespace walk's child isolation when the
+            // root starts sharing one accumulator across its descendants.
+            let isolate_compacted_children =
+                isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !force_compact);
+
             self.alert_excessive_folders(&folder.name, total_folders);
 
             if !into.compacted && should_compact {
@@ -2562,7 +2585,6 @@ impl FolderScanner {
                 }
             }
 
-            let is_scan_root = folder.name == self.old_cache.info.name;
             let scan_checkpoint = self.old_cache.info.scan_checkpoint.as_ref();
             let checkpoint_resume_after = scan_checkpoint.and_then(|checkpoint| {
                 if is_scan_root {
@@ -2586,7 +2608,6 @@ impl FolderScanner {
                 .is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
             let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
             let has_queued_folders = !queued_folders.is_empty();
-            let forward_sweep = self.old_cache.info.scan_progress.is_some();
             let resume_order = if forward_sweep {
                 order_queued_folders_for_resume(&mut queued_folders, None)
             } else {
@@ -2689,19 +2710,52 @@ impl FolderScanner {
 
                 (self.update_current_path)(&folder_item.name).await;
 
-                let outcome = if into.compacted {
-                    Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode)).await?
+                // Keep a compacted namespace child's accounting local until it
+                // succeeds. Ordinary child errors can then discard it without
+                // rolling back totals already committed by healthy siblings.
+                let isolate_compacted_child =
+                    into.compacted && isolate_compacted_children && child_mode == FolderScanMode::Namespace;
+                let outcome = if into.compacted && !isolate_compacted_child {
+                    // Keep the shared-accumulator path free of child summaries.
+                    Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode, false)).await?
                 } else {
-                    let mut dst = DataUsageEntry::default();
-                    match self
-                        .scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
-                        .await
-                    {
-                        Ok(outcome) => outcome,
+                    let mut dst = DataUsageEntry {
+                        compacted: isolate_compacted_child,
+                        ..Default::default()
+                    };
+                    // Provisional child usage must not escape through progress
+                    // updates before this accumulator is committed to its parent.
+                    let suspended_updates = if isolate_compacted_child {
+                        self.usage_updates.take()
+                    } else {
+                        None
+                    };
+                    let result = if isolate_compacted_child {
+                        Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), &mut dst, child_mode, true)).await
+                    } else {
+                        self.scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
+                            .await
+                    };
+                    if isolate_compacted_child {
+                        self.usage_updates = suspended_updates;
+                    }
+                    match result {
+                        Ok(outcome) => {
+                            if isolate_compacted_child {
+                                into.merge(&dst);
+                            }
+                            outcome
+                        }
                         Err(e) => {
                             if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
-                                self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
-                                    .await;
+                                if isolate_compacted_child {
+                                    // Cancellation and resource deferral keep completed
+                                    // work even though the child has not finished.
+                                    into.merge(&dst);
+                                } else {
+                                    self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
+                                        .await;
+                                }
                                 return Err(e);
                             }
                             if child_mode != FolderScanMode::Namespace {
@@ -2712,13 +2766,15 @@ impl FolderScanner {
                                 event = EVENT_SCANNER_FOLDER_STATE,
                                 component = LOG_COMPONENT_SCANNER,
                                 subsystem = LOG_SUBSYSTEM_FOLDER,
+                                state = "child_scan_failed",
                                 folder = %folder.name,
                                 child_folder = %folder_item.name,
-                                state = "child_scan_failed",
                                 error = %e,
                                 "Scanner child folder scan failed"
                             );
-                            self.coverage_gap |= forward_sweep;
+                            // A skipped namespace is a coverage gap even when no
+                            // checkpointed forward sweep is active.
+                            self.coverage_gap = true;
                             self.finish_raw_enumeration_parent(&folder_item.name);
                             continue;
                         }
@@ -3204,13 +3260,24 @@ impl FolderScanner {
         self.compact_completed_folder(&folder, &this_hash, into);
 
         // Compact if too many children...
+        // Allow compact_self on the bucket root when this is not a checkpointed
+        // forward sweep so wide roots can flatten instead of retaining one
+        // entry per top-level prefix (#8961). Checkpointed sweeps keep the
+        // previous root exclusion so resume frontiers stay addressable (#8292).
         if !into.compacted {
             let done_compact = Metrics::time(Metric::CompactFolder);
-            self.new_cache.reduce_children_of(
-                &this_hash,
-                DATA_SCANNER_COMPACT_AT_CHILDREN,
-                self.new_cache.info.name != folder.name,
-            );
+            let compact_self = self.new_cache.info.name != folder.name || self.old_cache.info.scan_progress.is_none();
+            self.new_cache
+                .reduce_children_of(&this_hash, DATA_SCANNER_COMPACT_AT_CHILDREN, compact_self);
+            // The root accumulator is also used by interruption handling.
+            // Reduction may have removed all its old child links, so publish
+            // the aggregate there before any cancellation-prone flush wait.
+            if folder.parent.is_none()
+                && let Some(root) = self.new_cache.cache.get(&this_hash.0)
+                && root.compacted
+            {
+                into.clone_from(root);
+            }
             done_compact();
         }
 
@@ -3224,6 +3291,8 @@ impl FolderScanner {
         }
 
         if folder.parent.is_none() {
+            #[cfg(test)]
+            tests::checkpoint_fixture::deep_compacted::cancel_at_root_flush(&self.root, &self.new_cache, &ctx);
             wait_for_scanner_io(&ctx, self.sleeper.flush_pacing(&mut self.pacing_debt)).await?;
         }
         self.finish_size_reconciliation_batch();
@@ -3415,7 +3484,7 @@ pub(crate) async fn scan_data_folder_scoped(
             let coverage_gap = scanner.coverage_gap;
             scanner.reconcile_failed_objects_after_full_scan();
             let new_cache = scanner.as_mut_new_cache();
-            new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
+            new_cache.force_compact(force_compact_limit_for_sweep(false));
             new_cache.info.last_update = Some(SystemTime::now());
             new_cache.info.next_cycle = next_cycle;
             let unresolved_objects = coverage_gap
@@ -3437,6 +3506,9 @@ pub(crate) async fn scan_data_folder_scoped(
                     progress.started_plan = progress.requested_plan;
                     new_cache.info.scan_plan_digest = None;
                 }
+            } else if coverage_gap {
+                // Legacy sampling must not certify an incomplete namespace.
+                new_cache.info.scan_plan_digest = None;
             }
             let had_scan_checkpoint = had_scan_checkpoint || new_cache.info.scan_checkpoint.is_some();
             new_cache.info.scan_resume_after = None;
@@ -3482,6 +3554,7 @@ pub(crate) async fn scan_data_folder_scoped(
                 let root_hash = hash_path(&bucket_name);
                 let root_has_progress = data_usage_root_has_progress(&root);
                 let pending_heals_changed = scanner.pending_heals_changed;
+                let legacy_coverage_gap = scanner.coverage_gap && scanner.old_cache.info.scan_progress.is_none();
                 let raw_enumeration_path_page_indexes = scanner.raw_enumeration_path_page_indexes();
                 let (raw_enumeration_cursor, raw_enumeration_page_index) = scanner.take_raw_enumeration_resume_state();
                 let carry_forward_cache = ((raw_enumeration_cursor.is_some() || raw_enumeration_page_index.is_some())
@@ -3516,11 +3589,20 @@ pub(crate) async fn scan_data_folder_scoped(
                     || !new_cache.info.size_reconciliation.is_empty()
                 {
                     if new_cache.root().is_some() {
-                        new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
+                        // Interrupted sweeps seal a resume frontier after this.
+                        // Flattening the root at the ordinary 10k limit would
+                        // drop frontier-named children and force the next cycle
+                        // to replay completed prefixes (#8292 / review on #9193).
+                        // Keep only the emergency FORCE ceiling here; successful
+                        // completions above apply DATA_SCANNER_COMPACT_AT_CHILDREN.
+                        new_cache.force_compact(force_compact_limit_for_sweep(true));
                     }
                     new_cache.info.last_update = Some(SystemTime::now());
                     new_cache.info.next_cycle = next_cycle;
                     new_cache.info.snapshot_complete = false;
+                    if legacy_coverage_gap {
+                        new_cache.info.scan_plan_digest = None;
+                    }
                     if root_has_progress {
                         set_scan_checkpoint(new_cache, checkpoint_reason_from_budget(budget.reason()));
                     }

crates/scanner/src/scanner_folder/tests.rs

@@ -4475,3 +4475,110 @@ fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappea
         "same source identity should keep the resume index even when read_dir order changes"
     );
 }
+
+#[test]
+fn reduce_children_of_wide_root_keeps_into_aligned_before_cancel_wait() {
+    // Non-checkpointed scans can reduce a reused wide root (>10k clean children)
+    // into a compacted aggregate. The live `into` accumulator must be refreshed
+    // from that cache entry before any await (flush_pacing): otherwise a cancel
+    // handler that replace_hashed's `into` reintroduces deleted child links.
+    const PREFIXES: usize = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
+    const {
+        assert!(PREFIXES > DATA_SCANNER_COMPACT_AT_CHILDREN);
+    }
+
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    let root = hash_path("bucket");
+    cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:05}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                size: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+
+    let mut into = cache.find("bucket").expect("root").clone();
+    assert!(!into.compacted);
+    assert_eq!(into.children.len(), PREFIXES);
+
+    cache.reduce_children_of(&root, DATA_SCANNER_COMPACT_AT_CHILDREN, true);
+    let cached = cache.find("bucket").expect("compacted root").clone();
+    assert!(cached.compacted);
+    assert!(cached.children.is_empty());
+    assert_eq!(cached.objects, PREFIXES);
+
+    // Pre-fix shape: into still holds the deleted child links.
+    assert_eq!(into.children.len(), PREFIXES);
+    assert!(!into.compacted);
+
+    // Production fix: copy the compacted aggregate back into the accumulator
+    // before any cancel-sensitive wait.
+    if cached.compacted {
+        into = cached;
+    }
+
+    // Simulate the interruption handler overwriting the cache from `into`.
+    cache.replace_hashed(&root, &None, &into);
+    let after_cancel = cache.find("bucket").expect("root after cancel overwrite");
+    assert!(after_cancel.compacted, "cancel must not resurrect a non-compacted root");
+    assert!(after_cancel.children.is_empty(), "cancel must not reintroduce dangling child links");
+    assert_eq!(after_cancel.objects, PREFIXES);
+    assert_eq!(cache.cache.len(), 1, "deleted children must stay deleted after cancel overwrite");
+}
+
+#[test]
+fn force_compact_limit_for_interrupted_sweep_keeps_frontier_above_ordinary_limit() {
+    const PREFIXES: usize = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
+    const {
+        assert!(PREFIXES > DATA_SCANNER_COMPACT_AT_CHILDREN);
+        assert!(PREFIXES < DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS);
+    }
+
+    let frontier = format!("bucket/prefix-{:05}", PREFIXES / 2);
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    let root = hash_path("bucket");
+    cache.replace_hashed(&root, &None, &DataUsageEntry::default());
+    for index in 0..PREFIXES {
+        let child = hash_path(&format!("bucket/prefix-{index:05}"));
+        cache.replace_hashed(
+            &child,
+            &Some(root.clone()),
+            &DataUsageEntry {
+                objects: 1,
+                size: 1,
+                compacted: true,
+                ..Default::default()
+            },
+        );
+    }
+
+    let mut completed = cache.clone();
+    completed.force_compact(force_compact_limit_for_sweep(false));
+    assert!(completed.find("bucket").is_some_and(|entry| entry.compacted));
+    assert!(completed.find(&frontier).is_none());
+
+    cache.force_compact(force_compact_limit_for_sweep(true));
+    assert!(cache.find(&frontier).is_some(), "interrupted ceiling must keep the frontier child");
+    let root_entry = cache.find("bucket").expect("root retained");
+    assert!(!root_entry.compacted);
+    assert_eq!(root_entry.children.len(), PREFIXES);
+}

crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs

@@ -21,7 +21,7 @@ use std::io::Cursor;
 use tokio::io::AsyncReadExt;
 
 mod completed_compaction;
-mod deep_compacted;
+pub(in crate::scanner_folder) mod deep_compacted;
 mod memory_budget;
 mod periodic_snapshot;
 mod segment_observation;

crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs

@@ -293,3 +293,507 @@ async fn check_large_prefix_resumption(mode: HealScanMode) {
     }
     panic!("large prefix must finish within repeated bounded scans");
 }
+
+#[cfg(unix)]
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_permission_failure_isolated_to_child() {
+    use std::os::unix::fs::PermissionsExt;
+
+    assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    write_checkpoint_object(&root, "healthy/object", &[(None, 7)]).await;
+    let bucket = root.join("bucket");
+    let denied = bucket.join("denied");
+    std::fs::create_dir(&denied).expect("create denied root child");
+    for idx in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
+        std::fs::create_dir(bucket.join(format!("empty-{idx:05}"))).expect("create wide directory");
+    }
+    assert_eq!(std::fs::read_dir(&bucket).expect("list root").count(), DATA_SCANNER_COMPACT_AT_FOLDERS);
+    std::fs::set_permissions(&denied, std::fs::Permissions::from_mode(0o000)).expect("remove child read access");
+
+    match std::fs::read_dir(&denied) {
+        Err(error) => assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied),
+        Ok(_) => {
+            std::fs::set_permissions(denied, std::fs::Permissions::from_mode(0o700)).expect("restore child access");
+            eprintln!("skipping permission-denied fixture because this process can read mode-000 directories");
+            return;
+        }
+    }
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    assert!(cache.info.scan_progress.is_none());
+    cache.info.skip_healing = true;
+    let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
+    let wide = scanner
+        .local_disk
+        .clone()
+        .nsscanner_disk(
+            budget.token(),
+            budget,
+            vec![scanner.local_disk.clone()],
+            cache,
+            None,
+            super::scan_options(HealScanMode::Normal),
+        )
+        .await;
+    std::fs::set_permissions(denied, std::fs::Permissions::from_mode(0o700)).expect("restore child access");
+    let ScannerDiskScanOutcome::Partial(wide) = wide.expect("wide legacy scanner must keep the same child-error isolation")
+    else {
+        panic!("an unreadable prefix must prevent complete snapshot publication");
+    };
+    assert!(!wide.info.snapshot_complete);
+    let summary = wide.checked_flatten("bucket").expect("aggregate healthy child");
+    assert!(summary.compacted);
+    assert_eq!((summary.objects, summary.size), (1, 7));
+    assert_eq!(wide.cache.len(), 1);
+}
+
+// Select one fixture disk so unrelated concurrent scans cannot trigger this
+// cancellation. The hook runs at the root flush boundary, after compaction.
+static CANCEL_ROOT_FLUSH: Mutex<Option<std::path::PathBuf>> = Mutex::new(None);
+
+pub(in crate::scanner_folder) fn cancel_at_root_flush(root: &str, cache: &DataUsageCache, ctx: &CancellationToken) {
+    let mut selected = CANCEL_ROOT_FLUSH.lock().expect("root flush cancellation lock");
+    if selected.as_deref() == Some(std::path::Path::new(root)) {
+        let cached = cache.root().expect("root cached before flush");
+        assert!(cached.compacted, "fixture must reach post-reduction compaction");
+        assert!(cached.children.is_empty());
+        *selected = None;
+        ctx.cancel();
+    }
+}
+
+struct RootFlushCancellationGuard;
+
+impl Drop for RootFlushCancellationGuard {
+    fn drop(&mut self) {
+        *CANCEL_ROOT_FLUSH.lock().expect("clear root flush cancellation") = None;
+    }
+}
+
+async fn wide_legacy_root() -> (FolderScanner, std::path::PathBuf, TestGuard, DataUsageCache) {
+    let (scanner, root) = build_test_scanner().await;
+    let cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    write_checkpoint_object(&root, "a-healthy/object", &[(Some(Uuid::from_u128(101)), 7)]).await;
+    write_checkpoint_object(&root, "z-healthy/object", &[(Some(Uuid::from_u128(102)), 11)]).await;
+    for index in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
+        std::fs::create_dir(root.join("bucket").join(format!("empty-{index:05}"))).expect("create wide root child");
+    }
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    assert!(cache.info.scan_progress.is_none());
+    (scanner, root, cleanup, cache)
+}
+
+#[cfg(unix)]
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_namespace_error_isolated_to_child() {
+    assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
+    let (scanner, root, _cleanup, cache) = wide_legacy_root().await;
+    // The error occurs while resolving a symlink inside the child, after its
+    // directory opened successfully. An open-only permission check misses it.
+    let regular = root.join("regular-file");
+    std::fs::write(&regular, b"payload").expect("write non-directory symlink target");
+    for broken in ["m-broken", "nested/broken"] {
+        let directory = root.join("bucket").join(broken);
+        std::fs::create_dir_all(&directory).expect("create failing namespace child");
+        let link = directory.join("bad-link");
+        symlink(regular.join("child"), &link).expect("create NotADirectory symlink");
+        assert_eq!(
+            std::fs::metadata(link).expect_err("invalid symlink target").kind(),
+            ErrorKind::NotADirectory
+        );
+    }
+    write_checkpoint_object(&root, "nested/healthy/object", &[(Some(Uuid::from_u128(103)), 13)]).await;
+    write_checkpoint_object(&root, "m-partial", &[(Some(Uuid::from_u128(104)), 101)]).await;
+    // Fail enumeration after this child's metadata contributed usage. Simply
+    // catching the error with a shared accumulator would retain its 101 bytes.
+    let _partial =
+        erasure_probe::OrderedDirectory::failing(&scanner.local_disk.path().join("bucket/m-partial"), 1, ErrorKind::Other);
+    let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 5).await;
+    let ScannerDiskScanOutcome::Partial(cache) = outcome else {
+        panic!("namespace errors must prevent complete snapshot publication without aborting healthy siblings")
+    };
+    assert!(!cache.info.snapshot_complete);
+    assert_eq!(budget.progress().0, 4, "healthy siblings and the failing child must read metadata");
+    let total = cache.checked_flatten("bucket").expect("valid compacted root");
+    assert!(total.compacted);
+    assert_eq!((total.objects, total.versions, total.size), (3, 3, 31));
+    assert_eq!(cache.cache.len(), 1, "child isolation must retain aggregate-only storage");
+    let mut reloaded = save_reload(&FixtureStore::new(), &cache).await;
+    assert_eq!(reloaded.checked_flatten("bucket").expect("persisted aggregate").objects, 3);
+    drop(_partial);
+    for broken in ["m-broken", "nested/broken"] {
+        std::fs::remove_dir_all(root.join("bucket").join(broken)).expect("remove failed namespace fixture");
+    }
+    assert_eq!(
+        reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false),
+        crate::DataUsageCachePrepareOutcome::Reset
+    );
+    reloaded.info.skip_healing = true;
+    let (outcome, budget) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 5).await;
+    let ScannerDiskScanOutcome::Complete(verified) = outcome else {
+        panic!("a clean retry must rebuild complete coverage")
+    };
+    assert!(verified.info.snapshot_complete);
+    assert_eq!(
+        budget.progress().0,
+        4,
+        "retry must revisit every object including the previously failed child"
+    );
+    let total = verified.checked_flatten("bucket").expect("verified complete aggregate");
+    assert_eq!((total.objects, total.versions, total.size), (4, 4, 132));
+}
+
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_child_cancellation_preserves_partial_accounting() {
+    let (scanner, _root, _cleanup, cache) = wide_legacy_root().await;
+    let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 1).await;
+    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
+    assert_eq!(budget.progress().0, 1);
+    let ScannerDiskScanOutcome::Partial(cache) = outcome else {
+        panic!("cancellation inside a compacted namespace child must propagate")
+    };
+    let total = cache.checked_flatten("bucket").expect("partial compacted accounting");
+    assert!(total.compacted);
+    assert_eq!((total.objects, total.versions), (1, 1));
+    assert!([7, 11].contains(&total.size), "completed child-local bytes must survive cancellation");
+    let reloaded = save_reload(&FixtureStore::new(), &cache).await;
+    assert_eq!(reloaded.checked_flatten("bucket").expect("persisted partial accounting").size, total.size);
+}
+
+#[tokio::test]
+#[serial]
+async fn wide_root_reused_children_cancellation_preserves_compacted_accounting() {
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    cache.replace("bucket", "", DataUsageEntry::default());
+    let children = DATA_SCANNER_COMPACT_AT_CHILDREN + 1;
+    for index in 0..children {
+        let name = format!("bucket/clean-{index:05}");
+        std::fs::create_dir_all(root.join(&name)).expect("create reusable root child");
+        cache.replace(
+            &name,
+            "bucket",
+            DataUsageEntry {
+                compacted: true,
+                objects: 1,
+                versions: 2,
+                size: 7,
+                ..Default::default()
+            },
+        );
+    }
+    assert_eq!(cache.root().expect("wide baseline root").children.len(), children);
+    *CANCEL_ROOT_FLUSH.lock().expect("install root flush cancellation") = Some(scanner.local_disk.path());
+    let _cancel_guard = RootFlushCancellationGuard;
+    let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
+    let mut options = super::scan_options(HealScanMode::Normal);
+    options.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".into()]));
+    let outcome = scanner
+        .local_disk
+        .clone()
+        .nsscanner_disk(budget.token(), budget.clone(), vec![scanner.local_disk.clone()], cache, None, options)
+        .await
+        .expect("cancelled root flush must return a partial cache");
+    assert!(CANCEL_ROOT_FLUSH.lock().expect("flush hook observed").is_none());
+    assert_eq!(budget.progress().0, 0, "clean children must be reused without metadata reads");
+    let ScannerDiskScanOutcome::Partial(cache) = outcome else {
+        panic!("flush cancellation must propagate")
+    };
+    assert!(!cache.info.snapshot_complete);
+    let total = cache
+        .checked_flatten("bucket")
+        .expect("cancellation must not leave dangling root links");
+    assert!(total.compacted);
+    assert!(total.children.is_empty());
+    assert_eq!((total.objects, total.versions, total.size), (children, children * 2, children * 7));
+    assert_eq!(cache.cache.len(), 1);
+    let reloaded = save_reload(&FixtureStore::new(), &cache).await;
+    assert_eq!(reloaded.checked_flatten("bucket").expect("persisted root accounting").objects, children);
+}
+
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_child_resource_deferral_preserves_partial_accounting() {
+    use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
+    use std::num::NonZeroU16;
+
+    let (mut scanner, _root, _cleanup, cache) = wide_legacy_root().await;
+    scanner.old_cache = cache.clone();
+    scanner.new_cache.info = cache.info;
+    scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
+    let admission = NodeAdmission::new(NonZeroU16::MIN);
+    admission.update(
+        &mut FeedbackController::new(),
+        Some(FeedbackObservation {
+            generation: 1,
+            max_concurrency: 1,
+            pressure: ResourcePressure::Unknown,
+            completed_units: None,
+        }),
+    );
+    scanner.node_admission = Some(admission.clone());
+    let (sender, mut receiver) = mpsc::channel(16);
+    scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".into()));
+    let deferred = Arc::new(AtomicBool::new(false));
+    let observed = deferred.clone();
+    scanner.update_current_path = Arc::new(move |path| {
+        if path == "bucket/a-healthy/object" {
+            // Defer the next directory after this object's metadata completed.
+            observed.store(true, Ordering::SeqCst);
+        } else if observed.load(Ordering::SeqCst) {
+            admission.update(&mut FeedbackController::new(), None);
+        }
+        Box::pin(async {})
+    });
+    let mut total = DataUsageEntry::default();
+    let result = scanner
+        .scan_folder(
+            CancellationToken::new(),
+            CachedFolder {
+                name: "bucket".into(),
+                parent: None,
+                object_heal_prob_div: 1,
+            },
+            &mut total,
+        )
+        .await;
+    assert!(deferred.load(Ordering::SeqCst), "first metadata object must be reached");
+    assert!(
+        matches!(result, Err(ScannerError::ResourceDeferred)),
+        "resource pressure must propagate: {result:?}"
+    );
+    assert!(total.compacted);
+    assert_eq!((total.objects, total.versions, total.size), (1, 1, 7));
+    assert!(scanner.usage_updates.is_some(), "resource deferral must restore the progress owner");
+    let committed = receiver.try_recv().expect("completed sibling progress");
+    assert_eq!((committed.objects, committed.versions, committed.size), (1, 1, 7));
+}
+
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_usage_updates_exclude_uncommitted_child_accounting() {
+    temp_env::async_with_vars([(rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, Some("1"))], async {
+        let (mut scanner, root, _cleanup, cache) = wide_legacy_root().await;
+        write_checkpoint_object(&root, "000-partial", &[(Some(Uuid::from_u128(105)), 101)]).await;
+        let _partial = erasure_probe::OrderedDirectory::failing(&root.join("bucket/000-partial"), 1, ErrorKind::Other);
+        scanner.old_cache = cache.clone();
+        scanner.new_cache.info = cache.info;
+        scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
+        let (sender, mut receiver) = mpsc::channel(16);
+        scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".into()));
+        scanner
+            .scan_folder(
+                CancellationToken::new(),
+                CachedFolder {
+                    name: "bucket".into(),
+                    parent: None,
+                    object_heal_prob_div: 1,
+                },
+                &mut DataUsageEntry::default(),
+            )
+            .await
+            .expect("failed namespace must not abort healthy siblings");
+        let committed = scanner
+            .new_cache
+            .checked_flatten("bucket")
+            .expect("canonical committed aggregate");
+        assert_eq!((committed.objects, committed.versions, committed.size), (2, 2, 18));
+        let first = receiver.try_recv().expect("committed usage progress");
+        assert_eq!(
+            (first.objects, first.versions, first.size),
+            (1, 1, 7),
+            "provisional 101-byte child must never be published"
+        );
+        assert!(receiver.try_recv().is_err(), "minute interval still bounds updates");
+        let updates = scanner
+            .usage_updates
+            .as_mut()
+            .expect("requested usage owner must be restored");
+        let total = updates.cache.checked_flatten("bucket").expect("connected committed progress");
+        assert_eq!((total.objects, total.versions, total.size), (2, 2, 18));
+        assert_eq!(
+            updates.cache.cache.len(),
+            1,
+            "compacted progress must not retain provisional child entries"
+        );
+        updates.last_update = SystemTime::UNIX_EPOCH;
+        scanner.send_update().await;
+        let last = receiver.try_recv().expect("final committed aggregate");
+        assert_eq!((last.objects, last.versions, last.size), (2, 2, 18));
+    })
+    .await;
+}
+
+#[tokio::test]
+#[serial]
+async fn legacy_namespace_gap_requires_fresh_verification_before_sampling() {
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    for (object, version, size) in [("prefix/000-partial", 111, 101), ("prefix/healthy", 112, 7)] {
+        write_checkpoint_object(&root, object, &[(Some(Uuid::from_u128(version)), size)]).await;
+    }
+    let partial = erasure_probe::OrderedDirectory::failing(
+        &scanner.local_disk.path().join("bucket/prefix/000-partial"),
+        1,
+        ErrorKind::Other,
+    );
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 3).await;
+    let ScannerDiskScanOutcome::Partial(cache) = outcome else {
+        panic!("a narrow root must also refuse to certify skipped namespace coverage")
+    };
+    assert_eq!(budget.progress().0, 2, "failed child must reach provisional metadata accounting");
+    assert!(!cache.info.snapshot_complete);
+    assert!(cache.find(PREFIX).expect("incomplete aggregate").compacted);
+    assert_eq!(
+        cache.info.scan_plan_digest, None,
+        "incomplete coverage must invalidate sampled baseline reuse"
+    );
+    let mut reloaded = save_reload(&FixtureStore::new(), &cache).await;
+    drop(partial);
+    assert_eq!(
+        reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false),
+        crate::DataUsageCachePrepareOutcome::Reset,
+        "a legacy retry must not sample an incomplete compacted subtree"
+    );
+    reloaded.info.skip_healing = true;
+    let (outcome, budget) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 3).await;
+    let ScannerDiskScanOutcome::Complete(cache) = outcome else {
+        panic!("clean retry must restore complete coverage")
+    };
+    assert!(cache.info.snapshot_complete);
+    assert_eq!(budget.progress().0, 2, "both metadata objects must be rechecked");
+    let total = cache.checked_flatten("bucket").expect("verified retry accounting");
+    assert_eq!((total.objects, total.versions, total.size), (2, 2, 108));
+}
+
+#[cfg(unix)]
+#[tokio::test]
+#[serial]
+async fn wide_root_legacy_symlink_lookup_failure_isolated_to_child() {
+    use std::os::unix::fs::symlink;
+
+    assert!(!crate::scanner_is_erasure().await, "single-drive Namespace scan");
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    write_checkpoint_object(&root, "healthy/object", &[(None, 7)]).await;
+    let bucket = root.join("bucket");
+    let bad = bucket.join("bad");
+    std::fs::create_dir(&bad).expect("create bad root child");
+    let regular_file = root.join("regular-file");
+    std::fs::write(&regular_file, b"not a directory").expect("create regular symlink ancestor");
+    let broken_link = bad.join("broken-link");
+    symlink(regular_file.join("child"), &broken_link).expect("create symlink through non-directory");
+    assert_eq!(
+        std::fs::metadata(broken_link).expect_err("target ancestor is a file").kind(),
+        std::io::ErrorKind::NotADirectory
+    );
+    for idx in 0..DATA_SCANNER_COMPACT_AT_FOLDERS - 2 {
+        std::fs::create_dir(bucket.join(format!("empty-{idx:05}"))).expect("create wide directory");
+    }
+
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    let (outcome, _) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 100).await;
+    let ScannerDiskScanOutcome::Partial(wide) = outcome else {
+        panic!("a failed child must preserve sibling progress without certifying complete coverage");
+    };
+    assert!(!wide.info.snapshot_complete);
+    let summary = wide.checked_flatten("bucket").expect("aggregate healthy child");
+    assert!(summary.compacted);
+    assert_eq!((summary.objects, summary.size), (1, 7));
+    assert_eq!(wide.cache.len(), 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn legacy_failed_object_keeps_retry_history_after_partial_scan() {
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    write_test_object_metadata_bytes(&root, "bucket", "broken/object", &[]).await;
+
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    let (outcome, _) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 100).await;
+    let ScannerDiskScanOutcome::Partial(partial) = outcome else {
+        panic!("a failed metadata read must keep the sweep partial");
+    };
+    assert_eq!(partial.info.failed_objects.len(), 1, "failed metadata should enter the retry cache");
+    let mut reloaded = save_reload(&FixtureStore::new(), &partial).await;
+    reloaded.prepare_for_scan("bucket", 12, 7, SOURCE, PLAN, false);
+    assert_eq!(
+        reloaded.info.failed_objects, partial.info.failed_objects,
+        "same-plan retry must retain the timestamp that suppresses repeated repair attempts"
+    );
+}
+
+#[cfg(unix)]
+#[tokio::test]
+#[serial]
+async fn interrupted_legacy_gap_requires_fresh_verification_before_sampling() {
+    use std::os::unix::fs::symlink;
+
+    let (scanner, root) = build_test_scanner().await;
+    let _cleanup = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    for (object, size) in [("prefix/a-bad/object", 101), ("prefix/z-good", 7), ("z-tail/object", 11)] {
+        write_checkpoint_object(&root, object, &[(None, size)]).await;
+    }
+    let regular_file = root.join("regular-file");
+    std::fs::write(&regular_file, b"not a directory").expect("create regular symlink ancestor");
+    let broken_link = root.join("bucket/prefix/a-bad/bad-link");
+    symlink(regular_file.join("child"), &broken_link).expect("create symlink through non-directory");
+
+    let mut cache = DataUsageCache::default();
+    cache.prepare_for_scan("bucket", 11, 7, SOURCE, PLAN, false);
+    cache.info.skip_healing = true;
+    let (outcome, budget) = scan(&scanner.local_disk, cache, HealScanMode::Normal, 2).await;
+    assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
+    let ScannerDiskScanOutcome::Partial(partial) = outcome else {
+        panic!("the tail must interrupt the scan after the prefix was compacted");
+    };
+    let prefix = partial.find(PREFIX).expect("incomplete compacted prefix");
+    assert!(prefix.compacted);
+    assert_eq!((prefix.objects, prefix.size), (1, 7));
+
+    let mut reloaded = save_reload(&FixtureStore::new(), &partial).await;
+    std::fs::remove_file(broken_link).expect("restore namespace traversal");
+    let next_cycle = (12_u32..28)
+        .find(|cycle| !hash_path(PREFIX).mod_(*cycle, data_usage_update_dir_cycles()))
+        .expect("choose a cycle that would otherwise sample the incomplete prefix");
+    reloaded.prepare_for_scan("bucket", u64::from(next_cycle), 7, SOURCE, PLAN, false);
+    reloaded.info.skip_healing = true;
+    let (outcome, _) = scan(&scanner.local_disk, reloaded, HealScanMode::Normal, 4).await;
+    let ScannerDiskScanOutcome::Complete(verified) = outcome else {
+        panic!("retry must complete after the namespace is repaired");
+    };
+    let total = verified.checked_flatten("bucket").expect("verified retry accounting");
+    assert_eq!((total.objects, total.size), (3, 119));
+}

crates/scanner/src/scanner_folder/tests/erasure_probe.rs

@@ -15,7 +15,7 @@ enum DirectoryOrder {
 pub(in crate::scanner_folder) type OrderedEntries = VecDeque<std::io::Result<native_io::DirEntry>>;
 static ORDERED_DIRECTORIES: LazyLock<Mutex<HashMap<String, DirectoryOrder>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
 
-struct OrderedDirectory(String);
+pub(in crate::scanner_folder) struct OrderedDirectory(String);
 
 impl OrderedDirectory {
     fn new(path: &std::path::Path) -> Self {
@@ -26,7 +26,7 @@ impl OrderedDirectory {
             .insert(path.clone(), DirectoryOrder::PartLast);
         Self(path)
     }
-    fn failing(path: &std::path::Path, entries: usize, kind: std::io::ErrorKind) -> Self {
+    pub(in crate::scanner_folder) fn failing(path: &std::path::Path, entries: usize, kind: std::io::ErrorKind) -> Self {
         let guard = Self::new(path);
         ORDERED_DIRECTORIES
             .lock()
@@ -205,6 +205,7 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
                 folder,
                 &mut DataUsageEntry::default(),
                 FolderScanMode::ErasureCandidate { reuse_namespace: false },
+                false,
             )
             .await
             .expect("classify full UUID namespace");
@@ -271,6 +272,7 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
             folder,
             &mut DataUsageEntry::default(),
             FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            false,
         )
         .await;
     task.await.expect("open-wait cancellation");
@@ -310,6 +312,7 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
             folder,
             &mut DataUsageEntry::default(),
             FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            false,
         )
         .await;
     assert!(ctx.is_cancelled());
@@ -379,7 +382,7 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
     scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
     let mut into = DataUsageEntry::default();
     let outcome = scanner
-        .scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false })
+        .scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
         .await
         .expect("classify candidate");
     assert_eq!(outcome, FolderScanOutcome::ErasureData);
@@ -592,6 +595,7 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
             folder,
             &mut DataUsageEntry::default(),
             FolderScanMode::ErasureCandidate { reuse_namespace: false },
+            false,
         )
         .await
         .expect("classify symlink namespace");

crates/scanner/src/scanner_io.rs

@@ -2024,10 +2024,10 @@ pub use dirty_usage::{
     ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError,
     ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation,
     acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record,
-    record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers,
-    record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change,
-    record_status_class_mutation, replay_durable_dirty_usage_producer_record, scanner_activity_epoch,
-    scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
+    probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
+    record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
+    record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
+    scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
     scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
 };
 #[cfg(test)]

crates/scanner/src/scanner_io/dirty_usage.rs

@@ -446,6 +446,38 @@ fn validate_durable_dirty_usage_replay_scope(
     }
 }
 
+/// Validate scoped ACK capability without clearing dirty work or taking a
+/// metadata mutation fence. Storage owns the read-only incarnation guards.
+/// A successful probe returns zero cleared records and retains no guards.
+///
+/// # Errors
+/// Rejects lost incarnation fences, a changed process epoch, or zero,
+/// saturated, or future generations.
+pub fn probe_scoped_dirty_usage(
+    instance_id: &str,
+    entries: &[(&crate::storage_api::EcstoreScannerBucketIncarnationReadGuard, u64)],
+) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
+    // Keep the generation observation ordered with dirty mutations. The probe
+    // neither changes this map nor acquires its scope/producer companion maps.
+    let _dirty = dirty_usage_buckets();
+    let checked = entries
+        .iter()
+        .map(|(guard, generation)| {
+            guard
+                .checked_bucket_incarnation()
+                .map(|(bucket, _)| (bucket, *generation))
+                .map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
+        })
+        .collect::<std::result::Result<Vec<_>, _>>()?;
+    validate_scoped_dirty_usage_generation(
+        instance_id,
+        scanner_activity_epoch(),
+        DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
+        &checked,
+    )?;
+    Ok(0)
+}
+
 /// A scoped ACK requires storage-owned lifecycle and incarnation fences.
 /// Callers must only send ACKs backed by durable per-bucket publication.
 pub fn acknowledge_scoped_dirty_usage(
@@ -512,16 +544,7 @@ fn apply_scoped_dirty_usage_ack(
     entries: &[(&str, u64)],
     probe_only: bool,
 ) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
-    if instance_id != current_instance {
-        return Err(ScannerDirtyUsageAckError::ProcessChanged);
-    }
-    if current_generation == u64::MAX
-        || entries
-            .iter()
-            .any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
-    {
-        return Err(ScannerDirtyUsageAckError::InvalidGeneration);
-    }
+    validate_scoped_dirty_usage_generation(instance_id, current_instance, current_generation, entries)?;
     let mut cleared = 0;
     if !probe_only {
         for (bucket, generation) in entries {
@@ -535,6 +558,25 @@ fn apply_scoped_dirty_usage_ack(
     Ok(cleared)
 }
 
+fn validate_scoped_dirty_usage_generation(
+    instance_id: &str,
+    current_instance: &str,
+    current_generation: u64,
+    entries: &[(&str, u64)],
+) -> std::result::Result<(), ScannerDirtyUsageAckError> {
+    if instance_id != current_instance {
+        return Err(ScannerDirtyUsageAckError::ProcessChanged);
+    }
+    if current_generation == u64::MAX
+        || entries
+            .iter()
+            .any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
+    {
+        return Err(ScannerDirtyUsageAckError::InvalidGeneration);
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod status_class_mutation_tests {
     use super::*;
@@ -623,6 +665,25 @@ mod scoped_dirty_usage_tests {
     use crate::segment_invalidation::SegmentInvalidationProducerIdentity;
     use serial_test::serial;
 
+    #[test]
+    fn scoped_dirty_usage_probe_rejects_invalid_epoch_and_generation() {
+        assert_eq!(validate_scoped_dirty_usage_generation("p", "p", 8, &[("cold", 8)]), Ok(()));
+        assert_eq!(
+            validate_scoped_dirty_usage_generation("old", "new", 8, &[("cold", 8)]),
+            Err(ScannerDirtyUsageAckError::ProcessChanged)
+        );
+        for generation in [0, 9, u64::MAX] {
+            assert_eq!(
+                validate_scoped_dirty_usage_generation("p", "p", 8, &[("cold", 8), ("hot", generation)]),
+                Err(ScannerDirtyUsageAckError::InvalidGeneration)
+            );
+        }
+        assert_eq!(
+            validate_scoped_dirty_usage_generation("p", "p", u64::MAX, &[("cold", 8)]),
+            Err(ScannerDirtyUsageAckError::InvalidGeneration)
+        );
+    }
+
     #[test]
     fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
         let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);

crates/scanner/src/storage_api.rs

@@ -53,8 +53,10 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
-    BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
-    get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
+    BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard,
+    ScannerBucketIncarnationReadGuard as EcstoreScannerBucketIncarnationReadGuard,
+    get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
+    get_replication_config as ecstore_get_replication_config,
 };
 pub(crate) use rustfs_ecstore::api::bucket::replication::{
     ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,

crates/signer/Cargo.toml

@@ -43,6 +43,7 @@ rustfs-utils = { workspace = true, features = ["full"] }
 s3s = { workspace = true, features = ["minio"] }
 base64-simd.workspace = true
 thiserror.workspace = true
+url.workspace = true
 
 [dev-dependencies]
 tracing-subscriber = { workspace = true, features = ["fmt"] }

crates/signer/src/lib.rs

@@ -13,12 +13,14 @@
 // limitations under the License.
 
 pub mod constants;
+pub mod request_signature_azure;
 pub mod request_signature_streaming;
 pub mod request_signature_streaming_unsigned_trailer;
 pub mod request_signature_v2;
 pub mod request_signature_v4;
 pub mod utils;
 
+pub use request_signature_azure::{AzureSignError, azure_shared_key_signature};
 pub use request_signature_streaming::streaming_sign_v4;
 pub use request_signature_streaming::try_streaming_sign_v4;
 pub use request_signature_v2::SignV2Error;

crates/signer/src/request_signature_azure.rs

@@ -0,0 +1,258 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared Key signing for Azure Blob REST requests.
+
+use http::HeaderMap;
+use rustfs_utils::crypto::hmac_sha256;
+use std::collections::BTreeMap;
+use url::Url;
+
+#[derive(Debug, thiserror::Error)]
+pub enum AzureSignError {
+    #[error("Azure Shared Key signing requires a non-empty account key")]
+    EmptyKey,
+    #[error("Azure Shared Key header {0} is not valid ASCII")]
+    InvalidHeader(String),
+    #[error("Azure Shared Key header {0} has multiple values")]
+    DuplicateHeader(&'static str),
+}
+
+/// Sign an Azure Blob request using a decoded storage-account key.
+///
+/// The caller supplies the final URL and headers, including x-ms-date and
+/// x-ms-version. Content headers and conditions are signed for writes as well
+/// as reads. This implements the Shared Key format for service versions
+/// 2015-02-21 and newer, where a zero Content-Length is signed as an empty
+/// field. The returned base64 signature does not include the account name.
+/// Service-header whitespace is folded only outside quoted strings; quoted
+/// whitespace and escaped characters retain their original bytes.
+///
+/// # Errors
+///
+/// Rejects an empty key, non-ASCII signed headers and repeated standard
+/// headers instead of signing a different request from the one on the wire.
+pub fn azure_shared_key_signature(
+    key: &[u8],
+    account: &str,
+    method: &str,
+    url: &Url,
+    headers: &HeaderMap,
+) -> Result<String, AzureSignError> {
+    if key.is_empty() {
+        return Err(AzureSignError::EmptyKey);
+    }
+    let mut canonical = String::with_capacity(512);
+    canonical.push_str(method);
+    canonical.push('\n');
+    for name in [
+        "content-encoding",
+        "content-language",
+        "content-length",
+        "content-md5",
+        "content-type",
+        "date",
+        "if-modified-since",
+        "if-match",
+        "if-none-match",
+        "if-unmodified-since",
+        "range",
+    ] {
+        let mut values = headers.get_all(name).iter();
+        let value = values.next();
+        if values.next().is_some() {
+            return Err(AzureSignError::DuplicateHeader(name));
+        }
+        let value = value
+            .map(|value| {
+                value
+                    .to_str()
+                    .map(str::trim)
+                    .map_err(|_| AzureSignError::InvalidHeader(name.to_string()))
+            })
+            .transpose()?
+            .unwrap_or_default();
+        if !(name == "content-length" && value == "0" || name == "date" && headers.contains_key("x-ms-date")) {
+            canonical.push_str(value);
+        }
+        canonical.push('\n');
+    }
+
+    let mut service_headers: BTreeMap<&str, Vec<String>> = BTreeMap::new();
+    for (name, value) in headers {
+        if name.as_str().starts_with("x-ms-") {
+            let value = value.to_str().map_err(|_| AzureSignError::InvalidHeader(name.to_string()))?;
+            service_headers
+                .entry(name.as_str())
+                .or_default()
+                .push(normalize_service_header(value));
+        }
+    }
+    for (name, values) in service_headers {
+        canonical.push_str(name);
+        canonical.push(':');
+        canonical.push_str(&values.join(","));
+        canonical.push('\n');
+    }
+    canonical.push('/');
+    canonical.push_str(account);
+    canonical.push_str(url.path());
+    let mut query: BTreeMap<String, Vec<String>> = BTreeMap::new();
+    for (name, value) in url.query_pairs() {
+        query.entry(name.to_ascii_lowercase()).or_default().push(value.into_owned());
+    }
+    for (name, mut values) in query {
+        values.sort();
+        canonical.push('\n');
+        canonical.push_str(&name);
+        canonical.push(':');
+        canonical.push_str(&values.join(","));
+    }
+    Ok(base64_simd::STANDARD.encode_to_string(hmac_sha256(key, canonical.as_bytes())))
+}
+
+fn normalize_service_header(value: &str) -> String {
+    // HeaderValue::to_str already excludes non-ASCII bytes and CR/LF.
+    let mut normalized = String::with_capacity(value.len());
+    let mut quoted = false;
+    let mut escaped = false;
+    let mut pending_space = false;
+    for byte in value.bytes() {
+        if !quoted && matches!(byte, b' ' | b'\t') {
+            pending_space = !normalized.is_empty();
+            continue;
+        }
+        if pending_space {
+            normalized.push(' ');
+            pending_space = false;
+        }
+        normalized.push(char::from(byte));
+        if escaped {
+            escaped = false;
+        } else if quoted && byte == b'\\' {
+            escaped = true;
+        } else if byte == b'"' {
+            quoted = !quoted;
+        }
+    }
+    normalized
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use http::HeaderValue;
+
+    #[test]
+    fn signs_write_content_and_conditions_with_independent_hmac_vectors() {
+        let url = Url::parse("https://account.blob.core.windows.net/container/blob?comp=block&blockid=AAAA")
+            .expect("fixture URL should parse");
+        let mut headers = HeaderMap::new();
+        for (name, value) in [
+            ("content-length", "3"),
+            ("content-md5", "YWJj"),
+            ("content-type", "application/octet-stream"),
+            ("if-none-match", "*"),
+            ("x-ms-date", "Fri, 06 Nov 2026 01:00:00 GMT"),
+            ("x-ms-version", "2020-10-02"),
+        ] {
+            headers.insert(name, HeaderValue::from_static(value));
+        }
+        // Fixed vectors generated with Python's hmac/hashlib from the Azure
+        // documented canonical fields, independently of this implementation.
+        for (length, expected) in [
+            ("3", "TMvdha8PzCSf0SB3v9EQSv0Se1gpdu1HusHBamwkyws="),
+            ("0", "kxBwUHkI/MK9beMd85Mij0MvaAlBtQQVYo9Qvl9rams="),
+        ] {
+            headers.insert("content-length", HeaderValue::from_static(length));
+            let signature = azure_shared_key_signature(b"0123456789abcdef0123456789abcdef", "account", "PUT", &url, &headers)
+                .expect("valid write should sign");
+            assert_eq!(signature, expected);
+        }
+    }
+
+    #[test]
+    fn signs_quoted_content_disposition_without_collapsing_filename_whitespace() {
+        let mut headers = HeaderMap::new();
+        for (name, value) in [
+            ("content-length", "3"),
+            ("content-md5", "YWJj"),
+            ("content-type", "application/octet-stream"),
+            ("if-none-match", "*"),
+            ("x-ms-date", "Fri, 06 Nov 2026 01:00:00 GMT"),
+            ("x-ms-version", "2020-10-02"),
+        ] {
+            headers.insert(name, HeaderValue::from_static(value));
+        }
+        // Independent Python hmac/hashlib vectors use the documented canonical
+        // strings: only whitespace outside quoted strings is folded.
+        for (value, signatures) in [
+            (
+                "attachment; filename=\"a  b.txt\"",
+                [
+                    "QO88BLMyaCEREeo/zP7esZKpkx9N/bwRQDwAkrjotB8=",
+                    "jZKYLpdUVYulDaVaNmpjy46IBxuuHyDmZ26ecnEs8wo=",
+                ],
+            ),
+            (
+                "\tattachment; \tfilename=\"a\\\"  b.txt\" \t",
+                [
+                    "dJI4cH77ju0YzM37HT66vck9p9obz9jdp5k04nhLkA0=",
+                    "13VDC1M4IIl11zPnxjBQhOd/CnzrxRj59DJ+wgDNQrQ=",
+                ],
+            ),
+            (
+                "attachment; filename=\"a\t\tb.txt\"",
+                [
+                    "ONDPQmh7ulrnziZyCiHOm241GhEJ4iTZa/ZWmgdE4sY=",
+                    "pHkeDR6QaUwuQTJsP/RFTlfPqzOA86DG6HSpdm4Fh2U=",
+                ],
+            ),
+        ] {
+            headers.insert("x-ms-blob-content-disposition", HeaderValue::from_static(value));
+            for (address, expected) in [
+                "https://account.blob.core.windows.net/container/blob",
+                "https://account.blob.core.windows.net/container/blob?comp=blocklist",
+            ]
+            .into_iter()
+            .zip(signatures)
+            {
+                let url = Url::parse(address).expect("fixture URL should parse");
+                let signature = azure_shared_key_signature(b"0123456789abcdef0123456789abcdef", "account", "PUT", &url, &headers)
+                    .expect("quoted content disposition should sign");
+                assert_eq!(signature, expected);
+            }
+        }
+    }
+
+    #[test]
+    fn rejects_ambiguous_headers_without_exposing_their_values() {
+        let url = Url::parse("https://account.blob.core.windows.net/container/blob").expect("fixture URL should parse");
+        let mut headers = HeaderMap::new();
+        headers.append("content-length", HeaderValue::from_static("1"));
+        headers.append("content-length", HeaderValue::from_static("2"));
+        assert!(matches!(
+            azure_shared_key_signature(b"key", "account", "PUT", &url, &headers),
+            Err(AzureSignError::DuplicateHeader("content-length"))
+        ));
+        headers.clear();
+        headers.insert(
+            "x-ms-meta-value",
+            HeaderValue::from_bytes(b"\xff-private").expect("opaque header is legal HTTP"),
+        );
+        let err =
+            azure_shared_key_signature(b"key", "account", "PUT", &url, &headers).expect_err("opaque signed header must fail");
+        assert_eq!(err.to_string(), "Azure Shared Key header x-ms-meta-value is not valid ASCII");
+    }
+}

docs/architecture/ilm-tiering-persistence-contracts.md

@@ -187,6 +187,8 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
 
 `TierMutationIntent` models Add/Edit/Remove/Clear with `Prepared`, `Committed`, and `Aborted`. The identity includes the old tier-config ETag, candidate digest, canonical sorted affected targets and their old/new backend identities, and expiry. Revision and committed config ETag are the only mutable generation data.
 
+A candidate configuration containing a native Azure Blob tier adds the immutable `requires_native_azure: true` intent extension. The flag participates in intent identity and checksum validation. Older strict readers reject it before installing a peer Prepare block, so all-node Prepare gates publication on native-driver support. The flag is omitted for ordinary mutations, preserving their prior wire bytes and checksums. Native Azure's persisted type and upgrade restrictions are documented in [Azure Blob remote tiers](../operations/azure-blob-tiering.md#upgrade-and-rollback).
+
 New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence.
 
 The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.

docs/architecture/s3-tables-support-matrix.md

@@ -67,7 +67,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
 | Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. Object-backed recovery reports a staged request as SUPERSEDED only when a reachable finalized same-base winner proves it lost CAS and its idempotency binding is intact. The original staged receipt is retained; its old request still conflicts. The winning history must be retained with the losing receipt; missing or contradictory history requires manual review. |
 | Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
 | Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
-| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. |
+| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; vended storage sessions cannot authenticate Catalog requests; responses carry `Cache-Control: no-store, private`. |
 | Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and use zero-file retention until strong maintenance configuration is persisted. |
 | Durable catalog backup and restore | Preview / controlled | Durable-strong only. `POST /iceberg/v1/{warehouse}/catalog/backup` seals a table-bucket snapshot behind the publication and migration fences, records catalog/object watermarks, and stores an immutable backup artifact. `POST /iceberg/v1/{warehouse}/catalog/restore` verifies those watermarks, requires an explicit replacement decision for an existing target, applies the catalog snapshot with ETag-CAS, and leaves a recoverable restore intent if maintenance-state finalization fails. It does not copy data files or claim automatic cross-region failover. |
 | Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. |

docs/operations/azure-blob-tiering.md

@@ -0,0 +1,70 @@
+# Azure Blob Remote Tiers
+
+New Azure tiers use the native Azure Blob REST API with storage-account Shared Key authentication. Azure public cloud and Azure Stack Hub use the same driver; Stack Hub requires its own account Blob endpoint. The driver uses service version 2020-10-02, which Microsoft lists for Stack Hub 2301 and later. See [Microsoft's storage compatibility matrix](https://learn.microsoft.com/en-us/azure-stack/user/azure-stack-acs-differences).
+
+The implementation is in [warm_backend_azure_blob.rs](../../crates/ecstore/src/services/tier/warm_backend_azure_blob.rs). Account-key signing is shared with the on-demand migration reader through [request_signature_azure.rs](../../crates/signer/src/request_signature_azure.rs); the two features have different storage-operation and service-version contracts.
+
+## Configuration
+
+Create the destination container first. Reserve its selected prefix for RustFS tier data, and do not modify the generated blobs with another writer.
+
+| Console/API field | Azure meaning |
+|---|---|
+| Azure Blob | Select the Azure tier provider |
+| Access Key / accessKey | Storage account name |
+| Secret Key / secretKey | Base64 storage-account key |
+| Bucket / bucket | Existing container name |
+| Endpoint / endpoint | Account Blob service origin; an empty value derives the public-cloud endpoint |
+| Prefix / prefix | Optional prefix reserved for this tier |
+| Region / region | Not used by native Blob signing or routing |
+| Storage Class / storageClass | Leave empty; the Console placeholder STANDARD is normalized to the default |
+
+For Stack Hub, use the complete account endpoint supplied by its portal, such as https://account.blob.region.example. Do not supply a container path or a SAS query string as part of the endpoint. HTTPS uses the server's trusted CA roots, including the deployment's installed private CA bundle.
+
+The existing Add Tier API accepts this shape:
+
+~~~json
+{
+  "type": "azure",
+  "azure": {
+    "name": "AZURE_COLD",
+    "endpoint": "https://account.blob.region.example",
+    "accessKey": "account",
+    "secretKey": "<base64-account-key>",
+    "bucket": "archive",
+    "prefix": "rustfs"
+  }
+}
+~~~
+
+Only storage-account keys are supported by this tier driver. Service-principal credentials, SAS tokens and explicit Hot/Cool/Archive access-tier selection are not enabled. Stack Hub's own support matrix also differs from public Azure for authentication and access tiers; selecting Azure Blob does not promise every Azure feature.
+
+## Upload, Read, and Cleanup
+
+The driver writes Block Blobs. Small objects use Put Blob; larger objects use bounded Put Block requests followed by one conditional Put Block List. Each block has Content-MD5 and a unique upload-specific ID. The complete declared input must be consumed before commit, and the driver does not automatically replay failed requests. The Stack Hub limit of 50,000 blocks with at most 100 MiB per block bounds the accepted object size.
+
+For a container with Blob versioning enabled, Azure returns timestamp version IDs rather than UUIDs. Before transitioning objects into such a container, upgrade every participating reader, writer and recovery worker, enable both `RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE=true` and `RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED=true` on the serving fleet, and ensure its live remote-version-state capability proof is current. The flags are operator attestation and a write request; they do not replace the live proof. With either flag disabled or the proof absent/stale, RustFS refuses to commit a new transition carrying an opaque Azure version and retains opaque-version recovery cleanup. See [opaque remote-version metadata requirements](tier-ilm-debugging.md#metadata-key-conventions).
+
+The native Azure tier's all-peer Prepare check does not enable this separate gate. A successful Add/Verify probe checks the Blob transport and cleanup, so it does not by itself establish that versioned lifecycle writes can commit.
+
+RustFS retains an opaque Azure version ID when the service returns one. Reads and ordinary deletes of that object explicitly address that version. If Azure rejects deleting the current version with OperationNotAllowedOnRootBlob, cleanup verifies the base blob still has the target version, conditionally deletes that base with its strong ETag, then retries the exact-version delete. A missing base also permits only an exact-version retry. A replacement version or a failed condition stops this fallback. A partial completion or lost acknowledgement remains an error until a later exact cleanup succeeds. An empty remote version sends no versionid parameter; unversioned cleanup also obtains a strong ETag and uses If-Match. Snapshots and immutability policies may prevent deletion and leave the normal tier cleanup record pending.
+
+Metadata is encoded into legal Azure metadata names and values. The transaction and destination identities retain both RustFS and MinIO aliases. Recovery checks those identities before recognizing an uncertain upload as its candidate. Encoded metadata is limited to 8 KiB. The complete destination blob name, including the configured prefix, must fit the Stack Hub limit of 880 characters and 1,760 UTF-8 bytes; standalone dot path segments are rejected before I/O. Configuration validation normalizes one trailing prefix slash and reserves the entire canonical lifecycle object-name suffix before running a remote probe, so a prefix that fits a short probe but cannot fit actual transitions is rejected when constructing the tier.
+
+Interrupted block staging can leave uncommitted blocks that Azure expires according to its own policy. Lifecycle uploads retain the existing transaction-journal recovery contract for a lost commit acknowledgement. Add/Edit/Verify probes retain the existing bounded in-process compensation: killing the process during a validation probe is not covered by a durable probe recovery worker. See [validation probe recovery status](tier-ilm-debugging.md#validation-probe-crash-recovery-status).
+
+## Upgrade and Rollback
+
+Existing Azure configurations retain their legacy S3-compatible transport. New Add operations select native Blob and persist a distinct protocol version and an encoded endpoint marker inside the existing binary configuration shape. A legacy destination and a native destination have different backend identities even if their account, endpoint, container and prefix are identical. Credential edits do not change the stored protocol.
+
+Upgrade every topology member before adding a native Azure tier. A mutation whose candidate configuration contains a native Azure tier carries the immutable requires_native_azure intent extension. Earlier peers reject that extension during Prepare, before the new configuration can be committed. Ordinary tier mutations without native Azure keep their previous wire shape and checksums.
+
+A rejected attempt during a mixed-version upgrade may retain a coordinator recovery record until every peer can read the intent and complete its normal abort/recovery exchange. Finish upgrading the remaining nodes before retrying; do not manually remove mutation records.
+
+Do not downgrade a cluster while a native Azure tier or its recovery records remain in use. An older server can preserve and rewrite the existing Azure payload, but its S3 driver rejects the encoded native endpoint before making a request. Restore or migrate the referenced data and complete normal tier removal and recovery before rollback. Do not manually remove the native endpoint marker or protocol version, or delete tier metadata to bypass reference checks.
+
+## Diagnostics and Verification
+
+Native service failures retain a bounded HTTP status, Azure service error code and x-ms-request-id. Credential failures and deterministic rejected writes are reported directly; they do not cause a cleanup attempt against an object the write did not create. A transport failure or lost acknowledgement still requires candidate reconciliation, and an unconfirmed cleanup remains an error. Credentials, signed authorization values and raw response bodies are excluded from these diagnostics.
+
+Local regression tests cover the signed Blob wire requests, block commit boundaries, empty and truncated input, ranged and versioned reads, conditional cleanup, identity-bound recovery, service errors, bounded XML parsing, persisted protocol selection and rejection by the previous peer reader. Before certifying a Stack Hub release, run Add/Verify, lifecycle transition, read-back/restore, expiry and restart/failure tests against that actual Stack Hub version. Mock protocol tests and Azure public-cloud results alone do not establish Stack Hub interoperability.

docs/operations/drive-timeout-tuning.md

@@ -24,6 +24,12 @@ may therefore remain undetected until real I/O reaches it. Missing slots,
 invalidated identities, unhealthy drives, and remote endpoints still get probed
 for recovery. Scanner, heal, and other background services have separate controls.
 
+## Multipart shard completion
+
+UploadPart waits for every still-pending shard operation to finish or reach its existing `RUSTFS_OBJECT_DISK_WRITE_STALL_TIMEOUT` budget (default 30 seconds), bounded further by the remaining `RUSTFS_OBJECT_DISK_WRITE_ABSOLUTE_CAP` when configured. Reaching write quorum does not start the one-second straggler cutoff used by ordinary PUT. This keeps finite slow shards from being discarded on different disks for different parts, which could leave too few disks carrying the entire object at CompleteMultipartUpload.
+
+This does not require every disk to succeed, count an unfinished writer as committed, or reduce the final object's write quorum. Failed and expired writers remain excluded. The tradeoff is that a black-hole peer can delay each part up to the existing operation budget rather than one second after quorum. The stall budget resets per shard-write operation; the optional absolute cap belongs to one part's encoding instance, not the entire multipart upload. Setting both budgets to zero retains the existing unbounded-wait behavior. Actual failures can still leave different parts missing on different disks, in which case completion must fail if a common complete write quorum is unavailable.
+
 ## Background: what these timeouts bound
 
 Every foreground drive operation carries a liveness budget so a hung disk fails

docs/operations/encrypted-etags.md

@@ -11,18 +11,11 @@ to the original plaintext. Encryption and decryption formats do not change.
 
 ## Listings and older objects
 
-ListObjects, ListObjectsV2, and ListObjectVersions omit the optional ETag field
-for encrypted objects whose metadata does not attest to the protected format.
-This includes encrypted objects written before the upgrade, preserved source
-ETags without that proof, and completed encrypted multipart uploads. New
-single-part encrypted writes carry the internal format marker and expose their
-protected ETag in listings. Unencrypted listings retain their existing ETags.
-Existing persistent listing snapshots are invalidated; the provider falls
-back to verified storage until a new snapshot is built.
+ListObjects, ListObjectsV2, and ListObjectVersions omit the optional ETag field for encrypted objects whose metadata does not attest to the protected format. This includes older encrypted objects and preserved source ETags without that proof. New single-part encrypted writes carry the internal format marker and expose their protected ETag in listings. Completed encrypted multipart uploads also carry that marker when every selected part has quorum-backed proof that its stored ETag came from the encryption reader and the final ETag is the aggregate of those parts. Unencrypted listings retain their existing ETags.
 
-Clients that require an ETag for these older or multipart objects must issue an
-authorized HeadObject request, including the SSE-C key when required. GET,
-HEAD, and conditional requests continue to use the object's stored ETag.
+Each new encrypted part records its producer proof in optional part metadata. Older part metadata defaults to unproven. Mixing proven and unproven selected parts, or preserving a different source ETag, keeps the completed object unproven. The proof is checked during completion using the existing part metadata reads, so listings need no additional object reads or KMS requests. Internal same-version data movement, including rebalance and decommission, retains an existing source proof when quorum-validated staging metadata and the completed object agree on the version and preserved ETag. Tier restore also retains the proof for the same version and ETag when it commits under the matching restore operation. Sources without proof continue to require authorized HEAD access or a credentialed rewrite.
+
+Clients that require an ETag for unproven objects must issue an authorized HeadObject request, including the SSE-C key when required. GET, HEAD, and conditional requests use the object's stored ETag. Already-completed encrypted multipart versions keep their original metadata and can still have no listing ETag after upgrading. Clients that enumerate historical versions, including backup software, may therefore require migration or an application change even after new uploads work. See the existing-data guidance below.
 
 ## Replication
 

docs/operations/internode-grpc-benchmark-runbook.md

@@ -58,6 +58,7 @@ All names are defined in `crates/io-metrics/src/internode_metrics.rs`.
 | Metric | Stage signal |
 | --- | --- |
 | `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency (P1), first-op latency (P3) |
+| `rustfs_system_network_internode_operation_server_duration_ms{operation,backend}` | server-side `ReadMultiple` / `BatchReadVersion` handler lifetime, including decode, disk work, encode, errors, and cancellation; separate from client RTT |
 | `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) |
 | `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) |
 | `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) |

docs/operations/lock-rpc-storm-protection.md

@@ -17,7 +17,19 @@ A timed-out request is no longer cancelled. Cancelling sends `RST_STREAM`, and e
 
 Three consecutive lock-acquisition timeouts open a request-level per-peer breaker. While open, new `lock` and `lock_batch` calls fail fast as retryable contention; after the backoff window one half-open probe is admitted. The breaker does not block release or refresh traffic, and it is separate from channel eviction. A per-peer in-flight admission limit also bounds requests that were already admitted before the breaker opened.
 
-Unlocks that fail three quick retries no longer stop there. The background task continues with a deferred schedule (1s, 2s, 4s, 8s, 16s) before it gives up and leaves the entry to the server-side lease.
+Single-resource distributed acquisition bounds each peer's contention wait to one second, but keeps pending responses and partial grants in the same quorum round until a response proves that round cannot reach quorum or the caller's remaining acquisition budget expires. A delayed response can still complete a healthy quorum; it does not restart acquisition with a fresh lock ID merely because delivery took longer than the peer's contention wait. Explicit contention failures retain rollback and retry behavior. Grants whose conservative lease deadline has expired before quorum are rolled back. Cancelling acquisition transfers both confirmed grants and pending replies to background cleanup. The caller's total deadline, read/write quorums, and fast quorum return remain unchanged.
+
+Unlock cleanup that remains unconfirmed after three quick retries continues in the background with a deferred schedule (1s, 2s, 4s, 8s, 16s). Exhausting that schedule means cleanup was not confirmed; it does not prove that the peer still holds the lock. A server-side lease bounds any grant that remains.
+
+Cleanup of a previously confirmed grant is idempotent for its exact `LockId`: `Released` and authoritative `AlreadyAbsent` both complete cleanup. The unlock RPCs append a structured `release_status` to each response or batch entry while preserving the existing `success` and `error_info` fields. Ordinary release and force-release still report a missing lock as failure. A new client keeps retrying unknown transport/backend results and absence responses from older peers that omit the structured status; it never infers absence from diagnostic text. Older clients ignore the added field and retain their existing behavior during upgrades or rollback.
+
+An absence response cannot settle an acquisition that may still grant the same ID later. Pending acquisition completion retains its own cleanup owner, and uncertain remote acquisition failures keep their conservative late-release retries. Only a completed, confirmed grant can use authoritative absence to stop retrying.
+
+## Local grant registration
+
+The local guard registry uses nonblocking native-lock attempts for short map operations and async notifications when the table is occupied. An unpolled metadata waiter does not reserve the table, so currently running peers can register compatible grants without inheriting that waiter's scheduling delay. Notification registration precedes the retry, and unlocking precedes waking waiters. Every registry access and contention retry retains cooperative scheduling points; no executor thread waits on a blocking table-lock acquisition.
+
+Physical acquisition and registry contention share the peer's existing acquisition deadline. An expired registration releases its unpublished physical guard and returns retryable contention, including when the table becomes available after the deadline. Replaced, removed, and reclaimed physical guards are dropped outside the registry lock. This does not change object-lock fairness, quorum, lease duration, or caller cancellation handling.
 
 ## Configuration
 

docs/operations/multipart-memory.md

@@ -31,6 +31,10 @@ A previously published gauge can remain visible after a statistic becomes unavai
 
 When live requested bytes are unavailable, capture a supported heap profile or allocation/free trace. Preserve attribution to allocation stacks and lifetime; committed bytes and RSS alone cannot establish a leak. Other raw allocator statistics, including page accounting, depend on the allocator build and are not interchangeable with live application bytes.
 
+### Linux transparent huge pages
+
+Linux RustFS binaries disable transparent huge pages for mimalloc by default. This reduces resident memory when reclaimable allocations occupy only part of a huge page. Check `THP_enabled` in `/proc/<pid>/status` and `AnonHugePages` in `/proc/<pid>/smaps_rollup` to verify the running process. Both values should be zero with the default. The Linux binary also defaults to immediately purging free arena ranges. Set `MIMALLOC_PURGE_DELAY` before startup to override the delay in milliseconds.
+
 ### Idle reclaim
 
 The reclaim loop waits for request, delete-tail, scanner, heal, EC and GET-buffer activity to become idle. Continuous traffic can skip reclaim. Check the activity gauges, skipped reasons and idle streak rather than assuming that waiting a fixed period guarantees collection.

docs/operations/outbound-connection-policy.md

@@ -22,7 +22,7 @@ RustFS validates every operator-configured outbound destination to close a serve
 | On-demand migration sources | Literal check, relaxed | Same guard and same escape hatch as replication targets (see below) |
 | Site replication peers | Literal check | `rustfs/src/site_replication/mod.rs` |
 | Self-hosted tiering warm backends (S3, MinIO, RustFS) | Literal check, relaxed | `validate_outbound_url_allow_private` allows private storage addresses except known IPv6 metadata addresses; RustFS retains its own debug-only loopback exception |
-| Cloud tiering warm backends (Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; Wasabi additionally requires its canonical provider endpoint before using the S3 constructor |
+| Cloud tiering warm backends (Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`, or the direct literal check in `warm_backend_azure_blob.rs` for native Azure; Wasabi additionally requires its canonical provider endpoint before using the S3 constructor |
 | Keystone `auth_url` | Literal check | `crates/keystone/src/config.rs` |
 
 The allowlist affects only the "Full policy" rows. A literal-check subsystem rejects a hostname that is itself a restricted IP literal, does not re-check what a hostname resolves to, and cannot be widened by `RUSTFS_OUTBOUND_ALLOW_ORIGINS`.