chore(release): bump version to 1.0.0-rc.1 (#5834)

* chore(release): prepare 1.0.0-rc.1

* chore(release): align release assets for 1.0.0-rc.1

.agents/skills/rustfs-logging-governance/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: rustfs-logging-governance
-description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use when editing or reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
+description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use whenever a change adds or edits any `tracing` macro call (`error!`/`warn!`/`info!`/`debug!`/`trace!`/`#[instrument]`) — including a single log line added in passing while fixing unrelated logic, which is how most new log sites enter the repo — and when reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
 ---
 
 # RustFS Logging Governance

.config/make/lint-fmt.mak

@@ -60,6 +60,11 @@ body-cache-whitelist-check: ## Check the body-cache eligibility gate stays a fai
 	@echo "🧱 Checking body-cache whitelist guard..."
 	./scripts/check_body_cache_whitelist.sh
 
+.PHONY: s3s-footprint-check
+s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
+	@echo "📦 Checking s3s footprint ratchet..."
+	./scripts/check_s3s_footprint.sh
+
 .PHONY: fips-wording-check
 fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
 	@echo "📣 Checking FIPS wording guard..."

.config/make/pre-commit.mak

@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 
 .PHONY: dev-check
-dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
+dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
 	@echo "✅ Fast development checks passed!"

.config/nextest.toml

@@ -29,6 +29,8 @@
 
 [test-groups]
 ecstore-serial-flaky = { max-threads = 1 }
+embedded-test-ports = { max-threads = 1 }
+e2e-vault = { max-threads = 1 }
 
 # Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
 # server and manipulate its disk directories at runtime (crates/e2e_test:
@@ -54,6 +56,20 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# The production-handler relocation regression builds an isolated 8-disk,
+# 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
+# from overlapping the ecstore commit fixtures above.
+[[profile.default.overrides]]
+filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
+test-group = 'ecstore-serial-flaky'
+
+# Embedded integration-test binaries discover an ephemeral port and release
+# the probe listener before RustFS binds it. Serialize that cross-process
+# TOCTOU window; retries would only hide real startup failures.
+[[profile.default.overrides]]
+filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
+test-group = 'embedded-test-ports'
+
 # Serialize the durable manual-transition checkpoint test across nextest's
 # process boundary; it mutates bucket lifecycle metadata and is not quarantined.
 [[profile.default.overrides]]
@@ -81,6 +97,12 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
+# does not cross nextest process boundaries, so keep these tests in one group.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+test-group = 'e2e-vault'
+
 # ---------------------------------------------------------------------------
 # ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
 # ---------------------------------------------------------------------------
@@ -143,6 +165,16 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
+test-group = 'ecstore-serial-flaky'
+
+# Match the default-profile embedded test isolation without quarantining or
+# retrying failures in CI.
+[[profile.ci.overrides]]
+filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
+test-group = 'embedded-test-ports'
+
 # Serialize the durable manual-transition checkpoint test under the ci profile
 # too. No retries: failures stay visible.
 [[profile.ci.overrides]]
@@ -186,7 +218,7 @@ test-group = 'ecstore-serial-flaky'
 # the nightly profile derives its set as "the replication module MINUS this
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
-# regexes byte-identical. Count invariant: 20 here + 28 nightly = 48 total
+# regexes byte-identical. Count invariant: 20 here + 36 nightly = 56 total
 # (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
@@ -222,7 +254,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^reliant::tiering::/)
@@ -248,10 +280,10 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 # tests that are unfit for the per-PR e2e-smoke gate:
 #
 #   * 2 remote-target TLS validation tests.
-#   * 12 bucket-replication data-plane/helper tests — they PUT/delete objects
-#     and poll until source and target converge; two replicate over HTTPS, two
-#     pin active SSE failure contracts, and one guards event/history observers.
-#     The SSE-S3 contract remains ignored under backlog#1291.
+#   * 13 bucket-replication data-plane/helper tests — they PUT/delete objects
+#     and poll until source and target converge; two replicate over HTTPS,
+#     four pin active SSE fail-closed contracts (SSE-C, SSE-S3, SSE-KMS, and
+#     the SSE-S3 resync path), and one guards event/history observers.
 #   * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
 #   * 1 `_real_three_node` site-replication test.
@@ -317,9 +349,9 @@ path = "junit.xml"
 #
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
-# parallel-safe — the same property e2e-smoke relies on. The exception is the
-# 4-disk reliability / degraded-read fault-injection tests, serialized below
-# (identical to the ci profile) so several 4-disk servers never run at once.
+# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
+# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
+# Vault tests, both serialized below.
 # KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
 # 341 ran / 32 failed on the suites' first automated run ever). Deterministic
 # product failures cannot be quarantined away with retries, so each family is
@@ -355,3 +387,7 @@ test-group = 'e2e-reliability'
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
+
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
+test-group = 'e2e-vault'

.docker/observability/README.md

@@ -170,6 +170,10 @@ Important behavior notes:
 
 - Logs and metrics usually appear during startup, so seeing those two signals
   first is expected.
+- The OpenTelemetry bridge sends `tracing` fields as log attributes. Loki stores
+  those attributes as structured metadata, and the Collector also mirrors the
+  common troubleshooting fields into the log line so simple line filters can
+  find them.
 - Visible trace data usually requires real HTTP/S3/gRPC request traffic after
   startup, because request-path spans are created on demand.
 - `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters
@@ -195,6 +199,17 @@ curl -I http://127.0.0.1:9000/health/ready
 # Jaeger:  http://localhost:16686
 ```
 
+For a structured RustFS log such as an inter-node RPC authentication failure,
+the Loki line now includes fields such as `event`, `component`, `subsystem`,
+`failure_reason`, `rpc_service`, `rpc_method`, and `expected_audience`. Useful
+LogQL checks:
+
+```logql
+{service_name="RustFS"} |= "RPC signature verification failed"
+{service_name="RustFS"} |= "failure_reason="
+{service_name="RustFS"} | failure_reason != ""
+```
+
 If logs and metrics are present but traces are sparse, the most common cause is
 "no real request traffic yet" or "`info` level filtered nested spans", not an
 OTLP routing failure.

.docker/observability/README_ZH.md

@@ -169,6 +169,7 @@ RustFS 会自动在该基础 URL 后补全：
 需要注意：
 
 - 启动阶段通常会先看到日志和指标，因此“先有日志/指标、后有 trace”是正常现象。
+- OpenTelemetry bridge 会把 `tracing` 字段作为日志 attributes 发送。Loki 会将这些 attributes 存为 structured metadata，同时 Collector 会把常用排障字段镜像进日志行，方便用简单的行内容过滤直接查到。
 - 可见的 trace 数据通常依赖启动后的真实 HTTP/S3/gRPC 请求流量，因为请求路径上的 span 是按需创建的。
 - `RUSTFS_OBS_LOGGER_LEVEL=info` 会保留顶层请求 span，但会过滤掉很多 `debug` 级别的嵌套 span。
   如果 Tempo 或 Jaeger 中的 trace 看起来很稀疏，建议先改成 `RUSTFS_OBS_LOGGER_LEVEL=debug`，再判断是否是 collector 或 Tempo 问题。
@@ -192,6 +193,14 @@ curl -I http://127.0.0.1:9000/health/ready
 # Jaeger:  http://localhost:16686
 ```
 
+对于 RustFS 结构化日志，例如节点间 RPC 鉴权失败，Loki 日志行现在会包含 `event`、`component`、`subsystem`、`failure_reason`、`rpc_service`、`rpc_method`、`expected_audience` 等字段。常用 LogQL 检查：
+
+```logql
+{service_name="RustFS"} |= "RPC signature verification failed"
+{service_name="RustFS"} |= "failure_reason="
+{service_name="RustFS"} | failure_reason != ""
+```
+
 如果日志和指标已经正常，但 trace 仍然稀疏，最常见的原因通常是
 “还没有真实请求流量”或“`info` 级别过滤了嵌套 span”，而不是 OTLP 路由失败。
 

.docker/observability/otel-collector-config.yaml

@@ -29,11 +29,27 @@ processors:
     limit_mib: 1024
     spike_limit_mib: 256
   transform/logs:
+    error_mode: ignore
     log_statements:
       - context: log
         statements:
-          - set(attributes["message"], body.string)
-          - set(attributes["log.body"], body.string)
+          - set(attributes["message"], body.string) where IsString(body)
+          - set(attributes["log.body"], body.string) where IsString(body)
+          - set(body, Concat([body, " event=", attributes["event"]], "")) where IsString(body) and attributes["event"] != nil
+          - set(body, Concat([body, " component=", attributes["component"]], "")) where IsString(body) and attributes["component"] != nil
+          - set(body, Concat([body, " subsystem=", attributes["subsystem"]], "")) where IsString(body) and attributes["subsystem"] != nil
+          - set(body, Concat([body, " state=", attributes["state"]], "")) where IsString(body) and attributes["state"] != nil
+          - set(body, Concat([body, " result=", attributes["result"]], "")) where IsString(body) and attributes["result"] != nil
+          - set(body, Concat([body, " reason=", attributes["reason"]], "")) where IsString(body) and attributes["reason"] != nil
+          - set(body, Concat([body, " failure_reason=", attributes["failure_reason"]], "")) where IsString(body) and attributes["failure_reason"] != nil
+          - set(body, Concat([body, " rpc_path=", attributes["rpc_path"]], "")) where IsString(body) and attributes["rpc_path"] != nil
+          - set(body, Concat([body, " rpc_service=", attributes["rpc_service"]], "")) where IsString(body) and attributes["rpc_service"] != nil
+          - set(body, Concat([body, " rpc_method=", attributes["rpc_method"]], "")) where IsString(body) and attributes["rpc_method"] != nil
+          - set(body, Concat([body, " expected_audience=", attributes["expected_audience"]], "")) where IsString(body) and attributes["expected_audience"] != nil
+          - set(body, Concat([body, " peer_addr=", attributes["peer_addr"]], "")) where IsString(body) and attributes["peer_addr"] != nil
+          - set(body, Concat([body, " replay_scope_bootstrap_allowed=", attributes["replay_scope_bootstrap_allowed"]], "")) where IsString(body) and attributes["replay_scope_bootstrap_allowed"] != nil
+          - set(body, Concat([body, " error=", attributes["error"]], "")) where IsString(body) and attributes["error"] != nil
+          - set(body, Concat([body, " exception_message=", attributes["exception.message"]], "")) where IsString(body) and attributes["exception.message"] != nil
 
 exporters:
   otlp/tempo:

.github/workflows/ci-docs-only.yml

@@ -102,6 +102,9 @@ jobs:
       - name: Check architecture migration rules
         run: ./scripts/check_architecture_migration_rules.sh
 
+      - name: Check logging guardrails
+        run: ./scripts/check_logging_guardrails.sh
+
       - name: Check tokio io-uring feature guard
         run: ./scripts/check_no_tokio_io_uring.sh
 
@@ -111,6 +114,9 @@ jobs:
       - name: Check body-cache whitelist guard
         run: ./scripts/check_body_cache_whitelist.sh
 
+      - name: Check s3s footprint ratchet
+        run: ./scripts/check_s3s_footprint.sh
+
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
 

.github/workflows/ci.yml

@@ -137,6 +137,9 @@ jobs:
       - name: Check architecture migration rules
         run: ./scripts/check_architecture_migration_rules.sh
 
+      - name: Check logging guardrails
+        run: ./scripts/check_logging_guardrails.sh
+
       - name: Check tokio io-uring feature guard
         run: ./scripts/check_no_tokio_io_uring.sh
 
@@ -146,6 +149,9 @@ jobs:
       - name: Check body-cache whitelist guard
         run: ./scripts/check_body_cache_whitelist.sh
 
+      - name: Check s3s footprint ratchet
+        run: ./scripts/check_s3s_footprint.sh
+
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
 
@@ -764,6 +770,32 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
+      - name: Install Vault
+        run: |
+          VAULT_VERSION="1.17.6"
+          VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
+          curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
+          echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669  $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
+          unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
+          echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
+
+      - name: Verify Vault
+        run: |
+          "$RUSTFS_TEST_VAULT_BIN" version
+
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary

.github/workflows/package.yml

@@ -0,0 +1,463 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Package Workflow - Build DEB/RPM packages
+#
+# This workflow builds DEB and RPM packages from pre-built Linux binaries
+# and uploads them to Cloudflare R2.
+#
+# Trigger:
+# - release published: automatically package when a GitHub release is published
+# - workflow_dispatch: manual trigger with optional tag/run_id
+#
+# Flow:
+# 1. Find the Build workflow run for the release tag
+# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
+# 3. Build DEB packages for amd64 and arm64
+# 4. Build RPM packages for x86_64 and aarch64
+# 5. Upload all packages to Cloudflare R2
+
+name: Package DEB/RPM
+
+permissions:
+  contents: read
+  actions: read
+
+on:
+  release:
+    types: [ published ]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag to package (e.g. 1.0.0-beta.12). Leave empty for latest main build."
+        required: false
+        type: string
+      build_run_id:
+        description: "Build workflow run ID (overrides tag lookup)"
+        required: false
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Resolve which build run to use and extract version info
+  resolve:
+    name: Resolve Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      version: ${{ steps.resolve.outputs.version }}
+      build_type: ${{ steps.resolve.outputs.build_type }}
+      build_run_id: ${{ steps.resolve.outputs.build_run_id }}
+      tag: ${{ steps.resolve.outputs.tag }}
+    steps:
+      - name: Resolve build run
+        id: resolve
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          INPUT_TAG: ${{ github.event.inputs.tag }}
+          INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
+        run: |
+          set -euo pipefail
+
+          # Determine tag
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            TAG="${{ github.event.release.tag_name }}"
+          elif [[ -n "$INPUT_TAG" ]]; then
+            TAG="$INPUT_TAG"
+          else
+            TAG=""
+          fi
+
+          echo "Tag: ${TAG:-<none>}"
+
+          # Determine build run ID
+          BUILD_RUN_ID=""
+
+          if [[ -n "$INPUT_RUN_ID" ]]; then
+            # Explicit run ID takes priority
+            BUILD_RUN_ID="$INPUT_RUN_ID"
+            echo "Using explicit build run ID: $BUILD_RUN_ID"
+
+          elif [[ -n "$TAG" ]]; then
+            # Find the build run that produced this tag
+            echo "Looking for build run for tag: $TAG"
+            BUILD_RUN_ID=$(gh api \
+              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
+              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              # Tag might not be a branch; try event=push with head_branch matching
+              BUILD_RUN_ID=$(gh api \
+                "repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
+                --jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
+            fi
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              echo "❌ No successful build run found for tag: $TAG"
+              exit 1
+            fi
+            echo "Found build run: $BUILD_RUN_ID"
+
+          else
+            # No tag — latest successful main build
+            echo "No tag specified, looking for latest main build"
+            BUILD_RUN_ID=$(gh api \
+              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
+              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              echo "❌ No successful main build found"
+              exit 1
+            fi
+            echo "Latest main build: $BUILD_RUN_ID"
+          fi
+
+          # Determine version and build type
+          if [[ -n "$TAG" ]]; then
+            VERSION="$TAG"
+            if [[ "$TAG" == *"-preview"* ]]; then
+              BUILD_TYPE="preview"
+            elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
+              BUILD_TYPE="prerelease"
+            else
+              BUILD_TYPE="release"
+            fi
+          else
+            SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
+              --jq '.head_sha' 2>/dev/null | head -c 7)
+            VERSION="dev-${SHORT_SHA}"
+            BUILD_TYPE="development"
+          fi
+
+          {
+            echo "version=$VERSION"
+            echo "build_type=$BUILD_TYPE"
+            echo "build_run_id=$BUILD_RUN_ID"
+            echo "tag=${TAG}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "📊 Resolved:"
+          echo "  Version: $VERSION"
+          echo "  Build type: $BUILD_TYPE"
+          echo "  Build run ID: $BUILD_RUN_ID"
+
+  # Build DEB and RPM packages for each architecture
+  package:
+    name: Package (${{ matrix.arch }})
+    needs: resolve
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+            deb_arch: amd64
+            rpm_arch: x86_64
+            artifact_name: "rustfs-linux-x86_64-gnu"
+          - arch: aarch64
+            deb_arch: arm64
+            rpm_arch: aarch64
+            artifact_name: "rustfs-linux-aarch64-gnu"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Download binary artifact from build run
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          pattern: ${{ matrix.artifact_name }}*
+          path: ./binary-artifact
+          run-id: ${{ needs.resolve.outputs.build_run_id }}
+          github-token: ${{ github.token }}
+          merge-multiple: true
+
+      - name: Extract binary
+        id: binary
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
+          if [[ -z "$ZIP_FILE" ]]; then
+            echo "❌ No binary artifact found"
+            ls -la ./binary-artifact/ || true
+            exit 1
+          fi
+
+          echo "Found artifact: $ZIP_FILE"
+
+          mkdir -p ./bin
+          unzip -o "$ZIP_FILE" -d ./bin
+
+          if [[ ! -f ./bin/rustfs ]]; then
+            echo "❌ rustfs binary not found in archive"
+            exit 1
+          fi
+
+          chmod +x ./bin/rustfs
+          ls -lh ./bin/rustfs
+          echo "✅ Binary extracted"
+
+      - name: Build DEB package
+        id: deb
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          VERSION="${{ needs.resolve.outputs.version }}"
+          DEB_ARCH="${{ matrix.deb_arch }}"
+          # DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
+          DEB_VERSION="${VERSION/-/~}"
+          PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
+
+          echo "Building DEB: ${PKG_DIR}.deb"
+
+          mkdir -p "${PKG_DIR}/DEBIAN"
+          mkdir -p "${PKG_DIR}/usr/bin"
+          mkdir -p "${PKG_DIR}/etc/default"
+          mkdir -p "${PKG_DIR}/lib/systemd/system"
+          mkdir -p "${PKG_DIR}/usr/share/doc/rustfs"
+
+          cp ./bin/rustfs "${PKG_DIR}/usr/bin/"
+          chmod 755 "${PKG_DIR}/usr/bin/rustfs"
+
+          cp deploy/build/rustfs.service "${PKG_DIR}/lib/systemd/system/"
+
+          cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
+          # RustFS Environment Configuration
+          # See https://rustfs.com/docs/ for more information
+          # RUSTFS_VOLUMES=""
+          # RUSTFS_ROOT_USER=""
+          # RUSTFS_ROOT_PASSWORD=""
+          ENVEOF
+
+          cat > "${PKG_DIR}/DEBIAN/control" << EOF
+          Package: rustfs
+          Version: ${DEB_VERSION}
+          Section: utils
+          Priority: optional
+          Architecture: ${DEB_ARCH}
+          Depends: libc6 (>= 2.31)
+          Maintainer: RustFS Team <support@rustfs.com>
+          Description: High-performance distributed object storage
+           RustFS is a high-performance distributed object storage software
+           built using Rust. It is compatible with MinIO and S3 API.
+          Homepage: https://rustfs.com
+          EOF
+
+          cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
+          #!/bin/bash
+          set -e
+          if ! getent passwd rustfs > /dev/null 2>&1; then
+              useradd -r -s /bin/false -d /opt/rustfs rustfs
+          fi
+          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
+          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
+          POSTINST
+          chmod 755 "${PKG_DIR}/DEBIAN/postinst"
+
+          cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
+              systemctl stop rustfs
+          fi
+          PRERM
+          chmod 755 "${PKG_DIR}/DEBIAN/prerm"
+
+          cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          POSTRM
+          chmod 755 "${PKG_DIR}/DEBIAN/postrm"
+
+          cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
+          cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
+
+          fakeroot dpkg-deb --build "${PKG_DIR}"
+
+          DEB_FILE="${PKG_DIR}.deb"
+          ls -lh "$DEB_FILE"
+          echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
+          echo "✅ DEB built: $DEB_FILE"
+
+      - name: Build RPM package
+        id: rpm
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          VERSION="${{ needs.resolve.outputs.version }}"
+          RPM_ARCH="${{ matrix.rpm_arch }}"
+
+          echo "Building RPM for ${RPM_ARCH}"
+
+          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
+          sudo gem install fpm
+
+          fpm -s dir -t rpm \
+            --name rustfs \
+            --version "$VERSION" \
+            --architecture "$RPM_ARCH" \
+            --depends "glibc >= 2.31" \
+            --maintainer "RustFS Team <support@rustfs.com>" \
+            --description "High-performance distributed object storage" \
+            --url "https://rustfs.com" \
+            --license "Apache-2.0" \
+            --after-install <(cat <<'POSTINST'
+          #!/bin/bash
+          set -e
+          if ! getent passwd rustfs > /dev/null 2>&1; then
+              useradd -r -s /bin/false -d /opt/rustfs rustfs
+          fi
+          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
+          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          POSTINST
+            ) \
+            --before-remove <(cat <<'PRERM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
+              systemctl stop rustfs
+          fi
+          PRERM
+            ) \
+            --after-remove <(cat <<'POSTRM'
+          #!/bin/bash
+          set -e
+          if [ -d /run/systemd/system ]; then
+              systemctl daemon-reload
+          fi
+          POSTRM
+            ) \
+            --config-files /etc/default/rustfs \
+            ./bin/rustfs=/usr/bin/rustfs \
+            deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
+            LICENSE=/usr/share/doc/rustfs/LICENSE \
+            README.md=/usr/share/doc/rustfs/README.md
+
+          RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
+          if [[ -z "$RPM_FILE" ]]; then
+            echo "❌ RPM build failed"
+            exit 1
+          fi
+
+          ls -lh "$RPM_FILE"
+          echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
+          echo "✅ RPM built: $RPM_FILE"
+
+      - name: Upload packages to artifacts
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: packages-${{ matrix.arch }}
+          path: |
+            *.deb
+            *.rpm
+          retention-days: 30
+
+      - name: Upload packages to Cloudflare R2
+        if: env.R2_ACCESS_KEY_ID != ''
+        env:
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          R2_BUCKET: ${{ secrets.R2_BUCKET }}
+          AWS_EC2_METADATA_DISABLED: true
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "⚠️ R2 credentials missing, skipping upload"
+            exit 0
+          fi
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            R2_PREFIX="artifacts/rustfs/packages/dev"
+          else
+            R2_PREFIX="artifacts/rustfs/packages/release"
+          fi
+          R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
+
+          echo "📤 Uploading to $R2_PATH"
+
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
+
+          for f in "$DEB_FILE" "$RPM_FILE"; do
+            if [[ -n "$f" && -f "$f" ]]; then
+              echo "Uploading: $f"
+              aws s3 cp "$f" "$R2_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            fi
+          done
+
+          echo "✅ Upload complete"
+
+          # Also upload as latest for release/prerelease
+          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
+            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+            for f in "$DEB_FILE" "$RPM_FILE"; do
+              if [[ -n "$f" && -f "$f" ]]; then
+                echo "Uploading latest: $(basename "$f")"
+                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+              fi
+            done
+            echo "✅ Latest packages updated"
+          fi
+
+  # Summary
+  summary:
+    name: Summary
+    needs: [ resolve, package ]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Print summary
+        shell: bash
+        run: |
+          echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
+          echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"

AGENTS.md

@@ -322,6 +322,28 @@ High risk: all seven roles.
 - Use environment variables or vault tooling for sensitive configuration.
 - For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
 
+## Logging
+
+Applies to **every** `tracing` macro you add or edit, including a single line
+added in passing while fixing something else — not only to log-focused changes.
+
+- Fields first, message second: `event`, `component`, `subsystem`,
+  `result`/`state`, then key context. The message is a short label, not a
+  sentence with values interpolated into it.
+- Reuse the existing `EVENT_*` / `LOG_COMPONENT_*` / `LOG_SUBSYSTEM_*`
+  constants of the module you are editing; match the shape of the log sites
+  already in that file rather than introducing a second style next to them.
+- Level policy: `error` for behavior/security-affecting failures, `warn` for
+  degraded or fallback paths, `info` for low-frequency lifecycle, `debug` for
+  targeted diagnostics, `trace` for hot paths. Per-object and per-request
+  success paths are `trace`.
+- Never log secrets, tokens, credential payloads, or merged config dumps.
+- `scripts/check_logging_guardrails.sh` enforces a subset of this on the files
+  it lists; passing it is a floor, not evidence the log matches the house style.
+
+See `.agents/skills/rustfs-logging-governance/SKILL.md` for the full event
+model, level policy, and guardrail-update checklist.
+
 ## Tools
 
 ### xl.meta decode tool Quick Use

Cargo.toml

@@ -69,7 +69,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.97.1"
-version = "1.0.0-beta.12"
+version = "1.0.0-rc.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -86,52 +86,52 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
-rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
+rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
+rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -228,15 +228,15 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.10.1" }
 aws-credential-types = { version = "1.3.0" }
 aws-sdk-kms = { default-features = false, version = "1.114.0" }
-aws-sdk-s3 = { default-features = false, version = "1.140.0" }
+aws-sdk-s3 = { default-features = false, version = "1.141.0" }
 aws-sdk-sts = { default-features = false, version = "1.110.0" }
 aws-smithy-http-client = { default-features = false, version = "1.2.0" }
 aws-smithy-runtime-api = { version = "1.14.0" }
 aws-smithy-types = { version = "1.6.1" }
-base64 = "0.23.0"
+base64 = "0.23.1"
 base64-simd = "0.8.0"
 brotli = "8.0.4"
-clap = { version = "4.6.5" }
+clap = { version = "4.6.6" }
 const-str = { version = "1.1.0" }
 convert_case = "0.11.0"
 criterion = { version = "0.8" }
@@ -244,7 +244,7 @@ crossbeam-queue = "0.3.13"
 crossbeam-channel = "0.5.16"
 crossbeam-deque = "0.8.7"
 crossbeam-utils = "0.8.22"
-datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
+datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
 #datafusion = { default-features = false, version = "54.1.0" }
 derive_builder = "0.20.2"
 enumset = "1.1.14"
@@ -341,15 +341,15 @@ unftp-core = "0.1.0"
 suppaftp = { version = "10.0.1" }
 rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.62.5" }
-russh-sftp = "2.3.0"
+russh-sftp = "2.4.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
-libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
-hotpath = { version = "0.23.0", default-features = false }
+mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
+libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
+hotpath = { version = "0.23.1", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

README.md

@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：

crates/common/Cargo.toml

@@ -39,11 +39,15 @@ tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
 tonic = { workspace = true, features = ["gzip", "deflate"] }
 uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 chrono = { workspace = true, features = ["serde"] }
+jiff = { workspace = true, features = ["serde"] }
 metrics = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 rmp-serde = { workspace = true }
 s3s = { workspace = true, features = ["minio"] }
 tracing = { workspace = true }
 
+[dev-dependencies]
+serde_json = { workspace = true }
+
 [lib]
 doctest = false

crates/common/src/metrics.rs

@@ -15,6 +15,7 @@
 use crate::heal_channel::HealScanMode;
 use crate::last_minute::{AccElem, LastMinuteLatency};
 use chrono::{DateTime, Utc};
+use jiff::Timestamp;
 use serde::{Deserialize, Serialize};
 use std::{
     collections::{BTreeSet, HashMap},
@@ -669,7 +670,7 @@ impl LockedLastMinuteLatency {
 #[derive(Clone, Debug)]
 struct CurrentPathState {
     path: String,
-    updated_at: DateTime<Utc>,
+    updated_at: Timestamp,
 }
 
 struct CurrentPathTracker {
@@ -678,10 +679,10 @@ struct CurrentPathTracker {
 
 impl CurrentPathTracker {
     fn new(initial_path: String) -> Self {
-        Self::new_at(initial_path, Utc::now())
+        Self::new_at(initial_path, Timestamp::now())
     }
 
-    fn new_at(initial_path: String, updated_at: DateTime<Utc>) -> Self {
+    fn new_at(initial_path: String, updated_at: Timestamp) -> Self {
         Self {
             state: Arc::new(RwLock::new(CurrentPathState {
                 path: initial_path,
@@ -693,7 +694,7 @@ impl CurrentPathTracker {
     async fn update_path(&self, path: String) {
         let mut state = self.state.write().await;
         state.path = path;
-        state.updated_at = Utc::now();
+        state.updated_at = Timestamp::now();
     }
 
     async fn get_state(&self) -> CurrentPathState {
@@ -701,6 +702,36 @@ impl CurrentPathTracker {
     }
 }
 
+fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
+    let seconds = dt.timestamp();
+    let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
+        Ok(nanoseconds) => nanoseconds,
+        Err(_) => {
+            return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
+        }
+    };
+
+    match Timestamp::new(seconds, nanoseconds) {
+        Ok(timestamp) => timestamp,
+        Err(_) => {
+            if seconds < 0 {
+                Timestamp::MIN
+            } else {
+                Timestamp::MAX
+            }
+        }
+    }
+}
+
+fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
+    let duration = now.duration_since(earlier);
+    if duration.is_negative() {
+        return 0;
+    }
+
+    u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
+}
+
 #[derive(Clone, Copy, Debug, Default)]
 struct ScannerDiskBucketScanState {
     concurrency_limit: u64,
@@ -1166,12 +1197,12 @@ pub struct ScannerLastMinute {
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct ScannerMetricsReport {
-    pub collected_at: DateTime<Utc>,
+    pub collected_at: Timestamp,
     pub current_cycle: u64,
     #[serde(default)]
     pub current_cycle_active: bool,
-    pub current_started: DateTime<Utc>,
-    pub cycles_completed_at: Vec<DateTime<Utc>>,
+    pub current_started: Timestamp,
+    pub cycles_completed_at: Vec<Timestamp>,
     pub ongoing_buckets: usize,
     #[serde(default)]
     pub active_scan_paths: usize,
@@ -2988,8 +3019,8 @@ impl Metrics {
             let cycle = self.cycle_info.read().await;
             let has_cycle = if let Some(cycle) = cycle.as_ref() {
                 m.current_cycle = cycle.current;
-                m.cycles_completed_at = cycle.cycle_completed.clone();
-                m.current_started = cycle.started;
+                m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
+                m.current_started = chrono_to_jiff_timestamp(cycle.started);
                 true
             } else {
                 false
@@ -3024,15 +3055,15 @@ impl Metrics {
         };
 
         if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
-            m.current_started = init_time;
+            m.current_started = chrono_to_jiff_timestamp(init_time);
         }
 
-        m.collected_at = Utc::now();
+        m.collected_at = Timestamp::now();
         let current_path_snapshots = self.current_path_snapshots().await;
         m.active_scan_paths = current_path_snapshots.len();
         m.oldest_active_path_age_seconds = current_path_snapshots
             .iter()
-            .map(|(_, state)| m.collected_at.signed_duration_since(state.updated_at).num_seconds().max(0) as u64)
+            .map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at))
             .max()
             .unwrap_or_default();
         m.active_paths = current_path_snapshots
@@ -3308,6 +3339,22 @@ impl Drop for CloseDiskGuard {
 mod tests {
     use super::*;
 
+    #[test]
+    fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
+        let report = ScannerMetricsReport {
+            collected_at: Timestamp::constant(1_700_000_000, 123_456_000),
+            current_started: Timestamp::constant(1_699_999_940, 0),
+            cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)],
+            ..Default::default()
+        };
+
+        let value = serde_json::to_value(&report).expect("scanner metrics report should serialize");
+
+        assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z"));
+        assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z"));
+        assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z"));
+    }
+
     #[tokio::test]
     async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
         let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
@@ -3366,7 +3413,7 @@ mod tests {
     #[tokio::test]
     async fn report_counts_active_scan_paths() {
         let metrics = Metrics::new();
-        let updated_at = Utc::now() - chrono::Duration::seconds(12);
+        let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12);
         metrics.current_paths.write().await.insert(
             "disk-a".to_string(),
             Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)),
@@ -3388,7 +3435,7 @@ mod tests {
         let metrics = Metrics::new();
         let tracker = Arc::new(CurrentPathTracker::new_at(
             "bucket-a".to_string(),
-            Utc::now() - chrono::Duration::hours(1),
+            Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60),
         ));
         metrics
             .current_paths
@@ -4161,7 +4208,7 @@ mod tests {
         let report = metrics.report().await;
         *crate::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
 
-        assert_eq!(report.current_started, cycle_started);
+        assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
     }
 
     #[tokio::test]
@@ -4584,7 +4631,7 @@ mod tests {
         let active = metrics.report().await;
         assert!(active.current_cycle_active);
         assert_eq!(active.current_cycle, 12);
-        assert_eq!(active.current_started, cycle_started);
+        assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
 
         let idle_cycle = CurrentCycle {
             current: 0,

crates/config/src/constants/internode.rs

@@ -177,10 +177,9 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
 ///
 /// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
 /// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
-/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
-/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
-/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
-/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
+/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
+/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
+/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
 /// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
 /// the shared secret) — and increments
 /// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow

crates/data-usage/src/data_usage.rs

@@ -37,6 +37,10 @@ pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 *
 /// Keeping the existing object name preserves rolling-upgrade and rollback
 /// compatibility without allowing an ambiguous snapshot to become authoritative.
 pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
+/// Latest structurally complete scanner observation. Unlike
+/// [`DATA_USAGE_OBJECT_NAME`], this object is never authoritative for quota
+/// admission because namespace activity may have raced the scan.
+pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json";
 
 /// Usage snapshot written by scanner implementations predating distributed
 /// leadership fencing. It is read only when neither authoritative snapshot
@@ -218,6 +222,20 @@ pub struct DataUsageInfo {
     /// explicit entry for every bucket, including confirmed-empty buckets.
     #[serde(default)]
     pub usage_snapshot_complete: bool,
+    /// Whether no namespace activity or dirty-usage generation changed while
+    /// the coordinated snapshot was being produced.
+    ///
+    /// `false` still describes a structurally complete, useful point-in-time
+    /// usage view, but follow-up scanner work remains pending. `None` is kept
+    /// for snapshots written before this status became observable.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub usage_snapshot_converged: Option<bool>,
+    /// Identity of the authoritative snapshot from which a nonconverged
+    /// observation started. Admin readers require an exact match before using
+    /// the observation, so bucket namespace mutations fence old observations
+    /// without relying on synchronized clocks.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub usage_snapshot_authoritative_baseline: Option<DataUsageSnapshotIdentity>,
     /// Deprecated kept here for backward compatibility reasons
     pub bucket_sizes: HashMap<String, u64>,
     /// Per-disk snapshot information when available
@@ -225,6 +243,59 @@ pub struct DataUsageInfo {
     pub disk_usage_status: Vec<DiskUsageStatus>,
 }
 
+/// Stable identity fields changed by both coordinated scanner publication and
+/// backward-compatible bucket namespace cleanup.
+#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DataUsageSnapshotIdentity {
+    pub last_update: Option<SystemTime>,
+    pub scanner_cycle: Option<u64>,
+    pub scanner_epoch: Option<u64>,
+}
+
+impl DataUsageInfo {
+    pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity {
+        DataUsageSnapshotIdentity {
+            last_update: self.last_update,
+            scanner_cycle: self.scanner_cycle,
+            scanner_epoch: self.scanner_epoch,
+        }
+    }
+}
+
+/// Return whether `candidate` was produced after `baseline`.
+///
+/// New coordinated snapshots are ordered by leadership epoch and scanner
+/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
+/// predate those fields.
+pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
+    match (
+        candidate.scanner_epoch.zip(candidate.scanner_cycle),
+        baseline.scanner_epoch.zip(baseline.scanner_cycle),
+    ) {
+        (Some(candidate), Some(baseline)) => candidate > baseline,
+        (Some(_), None) => true,
+        (None, Some(_)) => false,
+        (None, None) => match (candidate.last_update, baseline.last_update) {
+            (Some(candidate), Some(baseline)) => candidate > baseline,
+            (Some(_), None) => true,
+            (None, Some(_) | None) => false,
+        },
+    }
+}
+
+/// Return whether a nonconverged observation may safely supersede the admin
+/// view of `authoritative`.
+///
+/// The exact baseline identity is independent of clock ordering. Older binaries
+/// already advance the authoritative timestamp when deleting a bucket, so a
+/// rollback delete/recreate fences the previous bucket incarnation too.
+pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
+    observed.usage_snapshot_converged == Some(false)
+        && observed.is_complete_bucket_usage_snapshot()
+        && observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
+        && data_usage_snapshot_is_newer(observed, authoritative)
+}
+
 /// Metadata describing the status of a disk-level data usage snapshot.
 #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DiskUsageStatus {
@@ -1783,6 +1854,8 @@ mod tests {
         let current = DataUsageInfo {
             last_update: Some(SystemTime::UNIX_EPOCH),
             usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(DataUsageSnapshotIdentity::default()),
             ..Default::default()
         };
         let encoded = rmp_serde::to_vec_named(&current).expect("encode current data usage snapshot");
@@ -1790,6 +1863,76 @@ mod tests {
 
         assert_eq!(legacy.buckets_count, 0);
         assert!(current.is_complete_bucket_usage_snapshot());
+        assert_eq!(current.usage_snapshot_converged, Some(false));
+    }
+
+    #[test]
+    fn convergence_marker_defaults_to_unknown_for_older_snapshots() {
+        let encoded = rmp_serde::to_vec_named(&DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        })
+        .expect("encode pre-convergence data usage snapshot");
+        let decoded: DataUsageInfo = rmp_serde::from_slice(&encoded).expect("decode older data usage snapshot");
+
+        assert!(decoded.is_complete_bucket_usage_snapshot());
+        assert_eq!(decoded.usage_snapshot_converged, None);
+    }
+
+    #[test]
+    fn observation_selection_is_clock_independent_and_baseline_fenced() {
+        let mut authoritative = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(600)),
+            scanner_epoch: Some(7),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        };
+        let observed = DataUsageInfo {
+            // A newer leader may have a slower wall clock.
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
+            scanner_epoch: Some(8),
+            scanner_cycle: Some(1),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
+            ..Default::default()
+        };
+
+        assert!(observed_data_usage_is_newer(&observed, &authoritative));
+
+        authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
+        assert!(
+            !observed_data_usage_is_newer(&observed, &authoritative),
+            "an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
+        );
+    }
+
+    #[test]
+    fn observation_selection_requires_nonconverged_complete_newer_data() {
+        let authoritative = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH),
+            scanner_epoch: Some(2),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        };
+        let baseline = Some(authoritative.snapshot_identity());
+        let candidate = |epoch, cycle, converged, complete| DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
+            scanner_epoch: Some(epoch),
+            scanner_cycle: Some(cycle),
+            usage_snapshot_complete: complete,
+            usage_snapshot_converged: converged,
+            usage_snapshot_authoritative_baseline: baseline,
+            ..Default::default()
+        };
+
+        assert!(observed_data_usage_is_newer(&candidate(2, 11, Some(false), true), &authoritative));
+        assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
+        assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
+        assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
     }
 
     #[test]

crates/e2e_test/src/bucket_logging_test.rs

@@ -16,91 +16,17 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::types::{
         AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
         RequestPaymentConfiguration, WebsiteConfiguration,
     };
+    use http::Method;
+    use http::header::CONTENT_TYPE;
     use serial_test::serial;
-    use std::path::PathBuf;
-    use std::process::Command;
     use tracing::info;
 
-    fn awscurl_binary_path() -> PathBuf {
-        std::env::var_os("AWSCURL_PATH")
-            .map(PathBuf::from)
-            .unwrap_or_else(|| PathBuf::from("awscurl"))
-    }
-
-    fn awscurl_available() -> bool {
-        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
-    }
-
-    fn execute_s3_awscurl(
-        method: &str,
-        url: &str,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-        let output = Command::new(awscurl_binary_path())
-            .args([
-                "--service",
-                "s3",
-                "--region",
-                "us-east-1",
-                "--access_key",
-                access_key,
-                "--secret_key",
-                secret_key,
-                "-i",
-                "-X",
-                method,
-                url,
-            ])
-            .output()?;
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
-        }
-        Ok(String::from_utf8_lossy(&output.stdout).to_string())
-    }
-
-    fn parse_status(raw: &str) -> Option<u16> {
-        raw.lines()
-            .filter_map(|line| {
-                if line.starts_with("HTTP/") {
-                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
-                } else {
-                    None
-                }
-            })
-            .next_back()
-    }
-
-    fn parse_body(raw: &str) -> String {
-        if let Some(pos) = raw.rfind("\r\n\r\n") {
-            return raw[pos + 4..].to_string();
-        }
-        if let Some(pos) = raw.rfind("\n\n") {
-            return raw[pos + 2..].to_string();
-        }
-        String::new()
-    }
-
-    fn parse_headers(raw: &str) -> String {
-        let start = raw.rfind("HTTP/").unwrap_or(0);
-        let tail = &raw[start..];
-        if let Some(pos) = tail.find("\r\n\r\n") {
-            return tail[..pos].to_string();
-        }
-        if let Some(pos) = tail.find("\n\n") {
-            return tail[..pos].to_string();
-        }
-        tail.to_string()
-    }
-
     #[tokio::test]
     #[serial]
     async fn test_dummy_bucket_compatibility_endpoints() {
@@ -470,10 +396,6 @@ mod tests {
     async fn test_dummy_bucket_endpoints_http_contracts() {
         init_logging();
         info!("Starting test: dummy-compat bucket API HTTP contracts");
-        if !awscurl_available() {
-            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
-            return;
-        }
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
         env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
@@ -488,56 +410,112 @@ mod tests {
             .await
             .expect("Failed to create bucket");
 
-        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketLogging HTTP request failed");
-        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
-        let logging_body = parse_body(&logging_raw);
+        let logging_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?logging=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200");
+        let logging_body = logging_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketLogging response body");
         assert!(
             logging_body.contains("<BucketLoggingStatus"),
             "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
         );
 
-        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketAccelerateConfiguration HTTP request failed");
-        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
-        let accel_body = parse_body(&accel_raw);
+        let accel_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?accelerate=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(accel_response.status(), 200, "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = accel_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketAccelerateConfiguration response body");
         assert!(
             accel_body.contains("<AccelerateConfiguration"),
             "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
         );
 
-        let payment_raw =
-            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
-                .expect("GetBucketRequestPayment HTTP request failed");
-        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
-        let payment_body = parse_body(&payment_raw);
+        let payment_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?requestPayment=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(payment_response.status(), 200, "GetBucketRequestPayment should return 200");
+        let payment_body = payment_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketRequestPayment response body");
         assert!(
             payment_body.contains("<Payer>BucketOwner</Payer>"),
             "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
         );
 
-        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
-            .expect("GetBucketWebsite HTTP request failed");
+        let website_response = signed_s3_request(
+            Method::GET,
+            &format!("{}/{bucket}?website=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("GetBucketWebsite HTTP request failed");
         assert_eq!(
-            parse_status(&website_raw),
-            Some(404),
+            website_response.status(),
+            404,
             "GetBucketWebsite should return 404 when website config is absent"
         );
-        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        let website_content_type = website_response
+            .headers()
+            .get(CONTENT_TYPE)
+            .expect("GetBucketWebsite response should include Content-Type")
+            .to_str()
+            .expect("GetBucketWebsite Content-Type should be valid ASCII")
+            .to_ascii_lowercase();
         assert!(
-            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            website_content_type.contains("xml"),
             "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
         );
-        let website_body = parse_body(&website_raw);
+        let website_body = website_response
+            .text()
+            .await
+            .expect("Failed to read GetBucketWebsite response body");
         assert!(
             website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
             "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
         );
 
-        let delete_raw =
-            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
-                .expect("DeleteBucketWebsite HTTP request failed");
-        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+        let delete_response = signed_s3_request(
+            Method::DELETE,
+            &format!("{}/{bucket}?website=", env.url),
+            None,
+            None,
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await
+        .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204");
 
         env.stop_server();
     }

crates/e2e_test/src/bucket_stats_regression_test.rs

@@ -31,7 +31,7 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, awscurl_get, init_logging};
+    use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, awscurl_get, init_logging};
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use rustfs_data_usage::DataUsageInfo;
@@ -65,7 +65,7 @@ mod tests {
         info!("RT-09: bucket object count updates after PUT");
 
         let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
-        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
+        env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
             .await
             .expect("start RustFS");
 
@@ -88,12 +88,21 @@ mod tests {
 
         // Wait for scanner to process (up to 90 seconds)
         let mut found_nonzero = false;
+        let mut last_query_error = None;
         for attempt in 0..18 {
             sleep(Duration::from_secs(5)).await;
 
-            if let Ok(usage) = get_data_usage(&env).await
-                && let Some(bucket_usage) = usage.buckets_usage.get(bucket)
-            {
+            let usage = match get_data_usage(&env).await {
+                Ok(usage) => {
+                    last_query_error = None;
+                    usage
+                }
+                Err(err) => {
+                    last_query_error = Some(err.to_string());
+                    continue;
+                }
+            };
+            if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
                 info!("  attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
                 if bucket_usage.objects_count >= 10 {
                     found_nonzero = true;
@@ -104,7 +113,8 @@ mod tests {
 
         assert!(
             found_nonzero,
-            "RT-09 FAIL: bucket object count did not update after PUT 10 objects (regression: stats stuck at 0)"
+            "RT-09 FAIL: bucket object count did not update after PUT 10 objects (regression: stats stuck at 0); last query error: {}",
+            last_query_error.as_deref().unwrap_or("none")
         );
 
         info!("RT-09 PASS: bucket object count updates after PUT");
@@ -122,7 +132,7 @@ mod tests {
         info!("RT-09b: bucket object count updates after DELETE");
 
         let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
-        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
+        env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
             .await
             .expect("start RustFS");
 
@@ -143,6 +153,22 @@ mod tests {
                 .expect("put object");
         }
 
+        let mut found_nonzero = false;
+        for attempt in 0..18 {
+            sleep(Duration::from_secs(5)).await;
+
+            if let Ok(usage) = get_data_usage(&env).await
+                && let Some(bucket_usage) = usage.buckets_usage.get(bucket)
+            {
+                info!("  baseline attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
+                if bucket_usage.objects_count >= 5 {
+                    found_nonzero = true;
+                    break;
+                }
+            }
+        }
+        assert!(found_nonzero, "RT-09b setup failed: scanner did not observe the 5 uploaded objects");
+
         // Delete all objects
         for i in 0..5 {
             client
@@ -156,12 +182,21 @@ mod tests {
 
         // Wait for scanner to update stats (up to 90 seconds)
         let mut found_zero = false;
+        let mut last_query_error = None;
         for attempt in 0..18 {
             sleep(Duration::from_secs(5)).await;
 
-            if let Ok(usage) = get_data_usage(&env).await
-                && let Some(bucket_usage) = usage.buckets_usage.get(bucket)
-            {
+            let usage = match get_data_usage(&env).await {
+                Ok(usage) => {
+                    last_query_error = None;
+                    usage
+                }
+                Err(err) => {
+                    last_query_error = Some(err.to_string());
+                    continue;
+                }
+            };
+            if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
                 info!("  attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
                 if bucket_usage.objects_count == 0 {
                     found_zero = true;
@@ -172,7 +207,8 @@ mod tests {
 
         assert!(
             found_zero,
-            "RT-09b FAIL: bucket object count did not update to 0 after deleting all objects (regression rustfs#5615)"
+            "RT-09b FAIL: bucket object count did not update to 0 after deleting all objects (regression rustfs#5615); last query error: {}",
+            last_query_error.as_deref().unwrap_or("none")
         );
 
         info!("RT-09b PASS: bucket object count updates to 0 after DELETE");

crates/e2e_test/src/common.rs

@@ -47,6 +47,8 @@ use walkdir::WalkDir;
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
 pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
 pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
+pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
+    &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
 const RUSTFS_FULL_FEATURE: &str = "full";
 
@@ -65,8 +67,14 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
     capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
 }
 
-fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
-    let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
+pub(crate) fn build_test_s3_config(
+    endpoint_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: Option<&str>,
+    provider_name: &'static str,
+) -> Config {
+    let credentials = Credentials::new(access_key, secret_key, session_token.map(str::to_owned), None, provider_name);
     let mut config = Config::builder()
         .credentials_provider(credentials)
         .region(Region::new("us-east-1"))
@@ -81,6 +89,33 @@ fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str,
     config.build()
 }
 
+pub(crate) fn build_test_sts_client(
+    endpoint_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: Option<&str>,
+    provider_name: &'static str,
+) -> aws_sdk_sts::Client {
+    let mut config = aws_sdk_sts::Config::builder()
+        .credentials_provider(aws_sdk_sts::config::Credentials::new(
+            access_key,
+            secret_key,
+            session_token.map(str::to_owned),
+            None,
+            provider_name,
+        ))
+        .region(aws_sdk_sts::config::Region::new("us-east-1"))
+        .endpoint_url(endpoint_url)
+        .retry_config(aws_sdk_sts::config::retry::RetryConfig::standard().with_max_attempts(1))
+        .behavior_version_latest();
+
+    if endpoint_url.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+
+    aws_sdk_sts::Client::from_conf(config.build())
+}
+
 pub fn workspace_root() -> PathBuf {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     path.pop(); // e2e_test
@@ -95,6 +130,38 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
+pub(crate) async fn signed_s3_request(
+    method: http::Method,
+    url: &str,
+    body: Option<String>,
+    content_type: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
+    let mut request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
+    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
+
+    let mut request = local_http_client().request(method, url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    Ok(request.send().await?)
+}
+
 /// Signs and sends an admin HTTP request with the given credentials.
 pub(crate) async fn admin_request(
     base_url: &str,
@@ -105,28 +172,8 @@ pub(crate) async fn admin_request(
     secret_key: &str,
 ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        request = request.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
-    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
-
-    let mut request = local_http_client().request(method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
+    let content_type = body.as_ref().map(|_| "application/json");
+    let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?;
     let status = response.status();
     let body = response.text().await?;
     Ok((status, body))
@@ -564,7 +611,12 @@ impl RustFSTestEnvironment {
 
     /// Create an AWS S3 client configured for this RustFS instance
     pub fn create_s3_client(&self) -> Client {
-        Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
+        self.create_s3_client_with_credentials(&self.access_key, &self.secret_key)
+    }
+
+    /// Create an AWS S3 client with explicit credentials for this RustFS instance.
+    pub fn create_s3_client_with_credentials(&self, access_key: &str, secret_key: &str) -> Client {
+        Client::from_conf(build_test_s3_config(&self.url, access_key, secret_key, None, "e2e-test"))
     }
 
     /// Create test bucket
@@ -1296,6 +1348,7 @@ impl RustFSTestClusterEnvironment {
             &self.nodes[node_idx].url,
             &self.access_key,
             &self.secret_key,
+            None,
             "cluster-test",
         )))
     }

crates/e2e_test/src/data_usage_test.rs

@@ -18,7 +18,7 @@ use rustfs_data_usage::DataUsageInfo;
 use serial_test::serial;
 use tokio::time::{Duration, sleep};
 
-use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
 
 async fn get_data_usage_info(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
@@ -35,16 +35,26 @@ where
     F: FnMut(&DataUsageInfo) -> bool,
 {
     let mut last_usage = DataUsageInfo::default();
+    let mut last_query_error = None;
     for _ in 0..45 {
-        let usage = get_data_usage_info(env).await?;
-        if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
-            return Ok(usage);
+        match get_data_usage_info(env).await {
+            Ok(usage) => {
+                last_query_error = None;
+                if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
+                    return Ok(usage);
+                }
+                last_usage = usage;
+            }
+            Err(err) => last_query_error = Some(err.to_string()),
         }
-        last_usage = usage;
         sleep(Duration::from_secs(2)).await;
     }
 
-    Err(format!("bucket usage did not converge for {bucket}; last usage: {last_usage:?}").into())
+    Err(format!(
+        "bucket usage did not converge for {bucket}; last usage: {last_usage:?}; last query error: {}",
+        last_query_error.as_deref().unwrap_or("none")
+    )
+    .into())
 }
 
 /// Regression test for data usage accuracy (issue #1012).
@@ -56,7 +66,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
 
     let client = env.create_s3_client();
 
@@ -74,8 +84,14 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
             .await?;
     }
 
-    // Query admin data usage API
-    let usage = get_data_usage_info(&env).await?;
+    let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
+        usage
+            .buckets_usage
+            .get(TEST_BUCKET)
+            .map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
+            .unwrap_or(false)
+    })
+    .await?;
 
     // Assert total object count and per-bucket count are not truncated
     let bucket_usage = usage
@@ -108,7 +124,7 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
 
     let client = env.create_s3_client();
     let bucket = "data-usage-versioned";
@@ -184,8 +200,8 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
     assert_eq!(usage.versions_total_count, 3, "total version count should match bucket usage");
     assert_eq!(usage.delete_markers_total_count, 1, "total delete marker count should match bucket usage");
 
-    env.stop_server();
-    env.start_rustfs_server(vec![]).await?;
+    env.restart_server_preserving_data(vec![], FAST_DATA_USAGE_SCANNER_ENV)
+        .await?;
 
     let restarted_usage = wait_for_bucket_usage(&env, bucket, |usage| {
         usage

crates/e2e_test/src/fake_s3_target/mod.rs

@@ -574,7 +574,8 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
         (&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
         (&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
         (&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
-        (&Method::PUT, true) if only_query_keys(&[]) => Operation::PutObject,
+        // A replication PUT addresses the source version via `?versionId=`.
+        (&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
         (&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
         (&Method::HEAD, true) if only_query_keys(&["versionId"]) => Operation::HeadObject,
         (&Method::DELETE, true) if only_query_keys(&["versionId"]) => Operation::DeleteObject,

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -2211,11 +2211,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
             "queue_snapshot.{field} must be readable in terminal status: {terminal}"
         );
     }
-    assert!(
-        cold_tier_object_count(&cold_client).await? < 64,
-        "queue pressure should leave at least one object untransitioned"
-    );
-
     Ok(())
 }
 

crates/e2e_test/src/kms/bucket_default_encryption_test.rs

@@ -21,9 +21,12 @@
 
 use super::common::LocalKMSTestEnvironment;
 use crate::common::{TEST_BUCKET, init_logging};
+use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
+    ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
+    ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
 };
+use rustfs_rio::{Checksum, ChecksumType};
 use serial_test::serial;
 use tracing::{debug, info, warn};
 
@@ -273,7 +276,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
 /// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
 #[tokio::test]
 #[serial]
-async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Testing bucket default encryption impact on create_multipart_upload");
 
@@ -309,15 +312,16 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         .await
         .expect("Failed to set bucket encryption");
 
-    // Step 2: Create multipart upload (without specifying encryption parameters)
-    info!("Creating multipart upload (without specifying encryption parameters, should use bucket default configuration)");
-    let test_key = "test-multipart-bucket-default.txt";
+    // Step 2: Declare CRC32 without specifying encryption parameters. The AWS SDK
+    // calculates each UploadPart checksum and sends it as a flexible checksum.
+    info!("Creating CRC32 multipart upload that should use bucket default encryption");
+    let test_key = "test-multipart-bucket-default-crc32.bin";
 
     let create_multipart_response = s3_client
         .create_multipart_upload()
         .bucket(TEST_BUCKET)
         .key(test_key)
-        // Note: No encryption parameters specified here, should use bucket default configuration
+        .checksum_algorithm(ChecksumAlgorithm::Crc32)
         .send()
         .await
         .expect("Failed to create multipart upload");
@@ -343,28 +347,61 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         "create_multipart_upload response should contain correct KMS key ID"
     );
 
-    // Step 3: Upload a part and complete multipart upload
-    info!("Uploading part and completing multipart upload");
-    let test_data = b"test-multipart-bucket-default-encryption-data";
+    // Step 3: Upload two parts. The first is exactly the S3 minimum size so this
+    // follows the same managed SSE-KMS multipart path as issue #5756.
+    const PART_SIZE: usize = 5 * 1024 * 1024;
+    let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
+    let part2: Vec<u8> = (0..1024 * 1024).map(|i| ((i + 17) % 251) as u8).collect();
+    let expected_body: Vec<u8> = part1.iter().chain(&part2).copied().collect();
 
-    // Upload part 1
-    let upload_part_response = s3_client
-        .upload_part()
-        .bucket(TEST_BUCKET)
-        .key(test_key)
-        .upload_id(upload_id)
-        .part_number(1)
-        .body(test_data.to_vec().into())
-        .send()
-        .await
-        .expect("Failed to upload part");
+    let upload_part = |part_number: i32, body: Vec<u8>| {
+        s3_client
+            .upload_part()
+            .bucket(TEST_BUCKET)
+            .key(test_key)
+            .upload_id(upload_id)
+            .part_number(part_number)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .body(ByteStream::from(body))
+            .send()
+    };
 
-    let etag = upload_part_response.e_tag().unwrap().to_string();
+    let expected_part1_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part1)
+        .expect("calculate part 1 CRC32")
+        .encoded;
+    let upload1 = upload_part(1, part1).await.expect("Failed to upload part 1 with CRC32");
+    assert_eq!(
+        upload1.checksum_crc32(),
+        Some(expected_part1_crc32.as_str()),
+        "UploadPart must return the CRC32 calculated over plaintext"
+    );
+
+    let expected_part2_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part2)
+        .expect("calculate part 2 CRC32")
+        .encoded;
+    let upload2 = upload_part(2, part2).await.expect("Failed to upload part 2 with CRC32");
+    assert_eq!(
+        upload2.checksum_crc32(),
+        Some(expected_part2_crc32.as_str()),
+        "UploadPart must return the CRC32 calculated over plaintext"
+    );
 
     // Complete multipart upload
-    let completed_part = aws_sdk_s3::types::CompletedPart::builder()
-        .part_number(1)
-        .e_tag(&etag)
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload1.e_tag().expect("No ETag for part 1"))
+                .checksum_crc32(upload1.checksum_crc32().expect("No CRC32 for part 1"))
+                .build(),
+        )
+        .parts(
+            CompletedPart::builder()
+                .part_number(2)
+                .e_tag(upload2.e_tag().expect("No ETag for part 2"))
+                .checksum_crc32(upload2.checksum_crc32().expect("No CRC32 for part 2"))
+                .build(),
+        )
         .build();
 
     let complete_multipart_response = s3_client
@@ -372,11 +409,7 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         .bucket(TEST_BUCKET)
         .key(test_key)
         .upload_id(upload_id)
-        .multipart_upload(
-            aws_sdk_s3::types::CompletedMultipartUpload::builder()
-                .parts(completed_part)
-                .build(),
-        )
+        .multipart_upload(completed_upload)
         .send()
         .await
         .expect("Failed to complete multipart upload");
@@ -400,6 +433,7 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         .get_object()
         .bucket(TEST_BUCKET)
         .key(test_key)
+        .checksum_mode(ChecksumMode::Enabled)
         .send()
         .await
         .expect("Failed to get object");
@@ -410,6 +444,13 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         Some(&ServerSideEncryption::AwsKms),
         "Final object should contain SSE-KMS encryption information"
     );
+    if let Some(completed_crc32) = complete_multipart_response.checksum_crc32() {
+        assert_eq!(
+            get_response.checksum_crc32(),
+            Some(completed_crc32),
+            "GetObject should return the persisted composite CRC32 when completion reports it"
+        );
+    }
 
     // Verify data integrity
     let downloaded_data = get_response
@@ -418,7 +459,11 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
         .await
         .expect("Failed to collect body")
         .into_bytes();
-    assert_eq!(&downloaded_data[..], test_data, "Downloaded data should match original data");
+    assert_eq!(
+        downloaded_data.as_ref(),
+        expected_body.as_slice(),
+        "Downloaded data should match the uploaded multipart body"
+    );
 
     // Cleanup is handled automatically when the test environment is dropped
     info!("Test passed: bucket default encryption correctly applied to multipart upload");

crates/e2e_test/src/lib.rs

@@ -290,6 +290,14 @@ mod overwrite_cleanup_regression_test;
 #[cfg(test)]
 mod list_buckets_double_slash_test;
 
+// Regression coverage for bucket-scoped ListBuckets authorization fallback.
+#[cfg(test)]
+mod list_buckets_auth_test;
+
+// ListBuckets visibility follows IAM authorization, not bucket policy.
+#[cfg(test)]
+mod list_buckets_iam_filter_test;
+
 // Regression test for backlog#629(b): region-aware CreateBucket SigV4.
 #[cfg(test)]
 mod create_bucket_region_test;

crates/e2e_test/src/list_buckets_auth_test.rs

@@ -0,0 +1,88 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression coverage for the MinIO-compatible filtered ListBuckets fallback.
+
+use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
+use std::error::Error;
+
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+#[tokio::test]
+async fn bucket_scoped_policy_returns_only_authorized_bucket() -> TestResult {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let root_client = env.create_s3_client();
+    let allowed_bucket = "list-buckets-authorized";
+    let hidden_bucket = "list-buckets-hidden";
+    let user = "listbucketsuser";
+    let secret = "listbucketssecret";
+    let policy = "list-buckets-scoped";
+
+    root_client.create_bucket().bucket(allowed_bucket).send().await?;
+    root_client.create_bucket().bucket(hidden_bucket).send().await?;
+
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy}"),
+        Some(
+            serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": [{
+                    "Effect": "Allow",
+                    "Action": ["s3:*"],
+                    "Resource": [
+                        format!("arn:aws:s3:::{allowed_bucket}"),
+                        format!("arn:aws:s3:::{allowed_bucket}/*")
+                    ]
+                }]
+            })
+            .to_string(),
+        ),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::POST,
+        "/rustfs/admin/v3/idp/builtin/policy/attach",
+        Some(serde_json::json!({ "policies": [policy], "user": user }).to_string()),
+    )
+    .await?;
+
+    let client = env.create_s3_client_with_credentials(user, secret);
+    // Capture ListBuckets first so the direct-access control cannot warm bucket metadata and mask the regression.
+    let listed = client.list_buckets().send().await;
+    client.list_objects_v2().bucket(allowed_bucket).send().await?;
+
+    let listed = listed?;
+    let names = listed
+        .buckets()
+        .iter()
+        .filter_map(|bucket| bucket.name().map(ToOwned::to_owned))
+        .collect::<Vec<_>>();
+    assert_eq!(names, vec![allowed_bucket]);
+
+    Ok(())
+}

crates/e2e_test/src/list_buckets_iam_filter_test.rs

@@ -0,0 +1,459 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use serial_test::serial;
+use tokio::time::{Duration, Instant};
+
+fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
+    Client::from_conf(build_test_s3_config(
+        &env.url,
+        access_key,
+        secret_key,
+        session_token,
+        "list-buckets-iam-filter",
+    ))
+}
+
+fn bucket_names(buckets: &[aws_sdk_s3::types::Bucket]) -> Vec<String> {
+    let mut names = buckets
+        .iter()
+        .filter_map(|bucket| bucket.name().map(str::to_owned))
+        .collect::<Vec<_>>();
+    names.sort();
+    names
+}
+
+async fn create_user(
+    env: &RustFSTestEnvironment,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
+    admin_ok(
+        env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={access_key}"),
+        Some(body),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn create_service_account(
+    env: &RustFSTestEnvironment,
+    target_user: &str,
+    policy: Option<&serde_json::Value>,
+) -> Result<(String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let request = match policy {
+        Some(policy) => serde_json::json!({ "targetUser": target_user, "policy": policy }),
+        None => serde_json::json!({ "targetUser": target_user }),
+    };
+    let response = admin_ok(env, http::Method::PUT, "/rustfs/admin/v3/add-service-accounts", Some(request.to_string())).await?;
+    let response: serde_json::Value = serde_json::from_str(&response)?;
+    let access_key = response["credentials"]["accessKey"]
+        .as_str()
+        .ok_or("service account response should contain credentials.accessKey")?
+        .to_owned();
+    let secret_key = response["credentials"]["secretKey"]
+        .as_str()
+        .ok_or("service account response should contain credentials.secretKey")?
+        .to_owned();
+    Ok((access_key, secret_key))
+}
+
+#[tokio::test]
+#[serial]
+async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.capture_log_path = Some(format!("{}/server.log", env.temp_dir));
+    env.start_rustfs_server_with_env(vec![], &[("RUST_LOG", "rustfs=debug,rustfs_notify=debug")])
+        .await?;
+
+    let admin_client = env.create_s3_client();
+    for bucket in [
+        "benchmark-artifacts",
+        "benchmark-denied",
+        "benchmark-location-only",
+        "benchmark-test1",
+        "testuser1-artifacts",
+    ] {
+        admin_client.create_bucket().bucket(bucket).send().await?;
+    }
+    assert_eq!(
+        bucket_names(admin_client.list_buckets().send().await?.buckets()),
+        vec![
+            "benchmark-artifacts",
+            "benchmark-denied",
+            "benchmark-location-only",
+            "benchmark-test1",
+            "testuser1-artifacts"
+        ]
+    );
+
+    let access_key = "benchmark";
+    let secret_key = "benchmark-secret-1234567890";
+    create_user(&env, access_key, secret_key).await?;
+
+    let policy_name = "benchmark-bucket-prefix";
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:*"],
+                "Resource": ["arn:aws:s3:::benchmark-*", "arn:aws:s3:::benchmark-*/*"],
+                "Condition": {
+                    "StringEquals": {
+                        "s3:prefix": [""],
+                        "s3:delimiter": ["/"]
+                    }
+                }
+            },
+            {
+                "Effect": "Deny",
+                "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
+                "Resource": ["arn:aws:s3:::benchmark-denied"]
+            },
+            {
+                "Effect": "Deny",
+                "Action": ["s3:ListBucket"],
+                "Resource": ["arn:aws:s3:::benchmark-location-only"]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"]
+            }
+        ]
+    })
+    .to_string();
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={access_key}&isGroup=false"),
+        Some(String::new()),
+    )
+    .await?;
+
+    let bucket_policy_allow = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Principal": { "AWS": [access_key] },
+            "Action": ["s3:ListBucket"],
+            "Resource": ["arn:aws:s3:::testuser1-artifacts"]
+        }]
+    })
+    .to_string();
+    admin_client
+        .put_bucket_policy()
+        .bucket("testuser1-artifacts")
+        .policy(bucket_policy_allow)
+        .send()
+        .await?;
+
+    let bucket_policy_deny = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Deny",
+            "Principal": { "AWS": [access_key] },
+            "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
+            "Resource": ["arn:aws:s3:::benchmark-artifacts"]
+        }]
+    })
+    .to_string();
+    admin_client
+        .put_bucket_policy()
+        .bucket("benchmark-artifacts")
+        .policy(bucket_policy_deny)
+        .send()
+        .await?;
+
+    let benchmark_client = user_client(&env, access_key, secret_key, None);
+    benchmark_client
+        .list_objects_v2()
+        .bucket("testuser1-artifacts")
+        .send()
+        .await?;
+
+    assert_eq!(
+        bucket_names(benchmark_client.list_buckets().send().await?.buckets()),
+        vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
+    );
+    let log_path = env.capture_log_path.as_deref().expect("server log path should be configured");
+    let deadline = Instant::now() + Duration::from_secs(5);
+    let audit_log = loop {
+        let audit_log = tokio::fs::read_to_string(log_path).await?;
+        if [
+            "iam_implicit_deny",
+            "s3_authorization_denied",
+            "ListAllMyBucketsAction",
+            "benchmark",
+            "DEBUG",
+        ]
+        .iter()
+        .all(|field| audit_log.contains(field))
+            || Instant::now() >= deadline
+        {
+            break audit_log;
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    };
+    assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}");
+    for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] {
+        assert!(audit_log.contains(field), "missing {field} in {audit_log}");
+    }
+
+    let denied_access_key = "no-bucket-access";
+    let denied_secret_key = "no-bucket-access-secret-1234567890";
+    create_user(&env, denied_access_key, denied_secret_key).await?;
+    let denied = user_client(&env, denied_access_key, denied_secret_key, None)
+        .list_buckets()
+        .send()
+        .await
+        .expect_err("a user without IAM bucket permissions must be denied");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+    let put_only_policy_name = "put-only-no-bucket-discovery";
+    let put_only_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:PutObject"],
+            "Resource": ["arn:aws:s3:::benchmark-*/*"]
+        }]
+    })
+    .to_string();
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={put_only_policy_name}"),
+        Some(put_only_policy),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!(
+            "/rustfs/admin/v3/set-user-or-group-policy?policyName={put_only_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
+        ),
+        Some(String::new()),
+    )
+    .await?;
+    let denied = user_client(&env, denied_access_key, denied_secret_key, None)
+        .list_buckets()
+        .send()
+        .await
+        .expect_err("an unrelated IAM action must not reveal bucket names");
+    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+    let list_all_policy_name = "list-all-buckets";
+    let list_all_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"]
+        }]
+    })
+    .to_string();
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={list_all_policy_name}"),
+        Some(list_all_policy),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!(
+            "/rustfs/admin/v3/set-user-or-group-policy?policyName={list_all_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
+        ),
+        Some(String::new()),
+    )
+    .await?;
+    assert_eq!(
+        bucket_names(
+            user_client(&env, denied_access_key, denied_secret_key, None)
+                .list_buckets()
+                .send()
+                .await?
+                .buckets()
+        ),
+        vec![
+            "benchmark-artifacts",
+            "benchmark-denied",
+            "benchmark-location-only",
+            "benchmark-test1",
+            "testuser1-artifacts"
+        ]
+    );
+
+    let group_user = "benchmark-group-user";
+    let group_secret = "benchmark-group-secret-1234567890";
+    let group_name = "benchmark-group";
+    create_user(&env, group_user, group_secret).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(
+            serde_json::json!({
+                "group": group_name,
+                "members": [group_user],
+                "isRemove": false,
+                "groupStatus": "enabled"
+            })
+            .to_string(),
+        ),
+    )
+    .await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
+
+    assert_eq!(
+        bucket_names(
+            user_client(&env, group_user, group_secret, None)
+                .list_buckets()
+                .send()
+                .await?
+                .buckets()
+        ),
+        vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
+    );
+
+    let (service_access_key, service_secret_key) = create_service_account(&env, group_user, None).await?;
+    assert_eq!(
+        bucket_names(
+            user_client(&env, &service_access_key, &service_secret_key, None)
+                .list_buckets()
+                .send()
+                .await?
+                .buckets()
+        ),
+        vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
+    );
+
+    let service_account_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
+            "Resource": ["arn:aws:s3:::benchmark-test1"],
+            "Condition": {
+                "StringEquals": {
+                    "s3:prefix": [""],
+                    "s3:delimiter": ["/"]
+                }
+            }
+        }]
+    });
+    let (restricted_service_access_key, restricted_service_secret_key) =
+        create_service_account(&env, group_user, Some(&service_account_policy)).await?;
+    assert_eq!(
+        bucket_names(
+            user_client(&env, &restricted_service_access_key, &restricted_service_secret_key, None,)
+                .list_buckets()
+                .send()
+                .await?
+                .buckets()
+        ),
+        vec!["benchmark-test1"]
+    );
+
+    let sts_client = build_test_sts_client(&env.url, group_user, group_secret, None, "list-buckets-iam-filter-sts");
+    let inherited = sts_client
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/list-buckets")
+        .role_session_name("list-buckets-iam-filter-inherited")
+        .send()
+        .await?;
+    let inherited = inherited
+        .credentials()
+        .ok_or("AssumeRole response should contain inherited temporary credentials")?;
+    assert_eq!(
+        bucket_names(
+            user_client(
+                &env,
+                inherited.access_key_id(),
+                inherited.secret_access_key(),
+                Some(inherited.session_token()),
+            )
+            .list_buckets()
+            .send()
+            .await?
+            .buckets()
+        ),
+        vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
+    );
+
+    let session_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
+            "Resource": ["arn:aws:s3:::benchmark-test1"],
+            "Condition": {
+                "StringEquals": {
+                    "s3:prefix": [""],
+                    "s3:delimiter": ["/"]
+                }
+            }
+        }]
+    })
+    .to_string();
+    let assumed = sts_client
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/list-buckets")
+        .role_session_name("list-buckets-iam-filter")
+        .policy(session_policy)
+        .send()
+        .await?;
+    let temporary = assumed
+        .credentials()
+        .ok_or("AssumeRole response should contain temporary credentials")?;
+    assert_eq!(
+        bucket_names(
+            user_client(
+                &env,
+                temporary.access_key_id(),
+                temporary.secret_access_key(),
+                Some(temporary.session_token()),
+            )
+            .list_buckets()
+            .send()
+            .await?
+            .buckets()
+        ),
+        vec!["benchmark-test1"]
+    );
+    Ok(())
+}

crates/e2e_test/src/multipart_auth_test.rs

@@ -6028,6 +6028,33 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     }
 
     let version_condition_client = restricted_user_client(&env, version_condition_user, version_condition_secret);
+    let mismatching_version_pax = HashMap::from([("minio.versionId", Uuid::new_v4().to_string())]);
+    let archive = make_tar_with_pax_entry("version-mismatch-entry.txt", b"must-not-write", None, &mismatching_version_pax).await;
+    let err = version_condition_client
+        .put_object()
+        .bucket(bucket)
+        .key("version-mismatch.tar")
+        .body(ByteStream::from(archive))
+        .customize()
+        .mutate_request(|req| {
+            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await
+        .expect_err("a mismatching PAX version ID must fail the replication condition");
+    assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
+    let err = admin_client
+        .head_object()
+        .bucket(bucket)
+        .key("version-mismatch-entry.txt")
+        .send()
+        .await
+        .expect_err("a denied PAX entry must not be written");
+    assert!(matches!(
+        err.as_service_error().and_then(|error| error.meta().code()),
+        Some("NoSuchKey" | "NotFound")
+    ));
+
     let matching_version_pax = HashMap::from([("minio.versionId", conditional_version_id)]);
     let archive = make_tar_with_pax_entry("condition-entry.txt", b"condition-body", None, &matching_version_pax).await;
     version_condition_client
@@ -6041,6 +6068,13 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
         })
         .send()
         .await?;
+    let stored = admin_client
+        .get_object()
+        .bucket(bucket)
+        .key("condition-entry.txt")
+        .send()
+        .await?;
+    assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
 
     let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
     let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);

crates/e2e_test/src/quota_test.rs

@@ -12,9 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
 use aws_sdk_s3::Client;
+use http::{Method, StatusCode};
 use serial_test::serial;
+use tokio::time::{Duration, sleep, timeout};
 use tracing::{debug, info};
 
 fn skip_without_awscurl() -> bool {
@@ -37,7 +39,8 @@ impl QuotaTestEnv {
     pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         let bucket_name = format!("quota-test-{}", uuid::Uuid::new_v4());
         let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
+        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")])
+            .await?;
         let client = env.create_s3_client();
 
         Ok(Self {
@@ -67,18 +70,7 @@ impl QuotaTestEnv {
     }
 
     pub async fn set_bucket_quota(&self, quota_bytes: u64) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, self.bucket_name);
-        let quota_config = serde_json::json!({
-            "quota": quota_bytes,
-            "quota_type": "HARD"
-        });
-
-        let response = awscurl_put(&url, &quota_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
-        if response.contains("error") {
-            Err(format!("Failed to set quota: {}", response).into())
-        } else {
-            Ok(())
-        }
+        self.set_bucket_quota_for(&self.bucket_name, quota_bytes).await
     }
 
     pub async fn get_bucket_quota(&self) -> Result<Option<u64>, Box<dyn std::error::Error + Send + Sync>> {
@@ -178,6 +170,29 @@ impl QuotaTestEnv {
         bucket: &str,
         quota_bytes: u64,
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
+        let readiness = async {
+            loop {
+                let (status, response) =
+                    admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
+                        .await?;
+                if status.is_success() {
+                    return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
+                }
+                if status != StatusCode::SERVICE_UNAVAILABLE {
+                    return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
+                }
+
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        match timeout(Duration::from_secs(30), readiness).await {
+            Ok(result) => result?,
+            Err(_) => {
+                return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
+            }
+        }
+
         let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
         let quota_config = serde_json::json!({
             "quota": quota_bytes,

crates/e2e_test/src/replication_extension_test.rs

@@ -16,6 +16,7 @@ use crate::common::{
     RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
     replication_fast_env, rustfs_binary_path,
 };
+use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
 use crate::kms::common::{create_key_with_specific_id, sse_customer_key_md5_base64};
 use crate::storage_api::replication_extension::BucketTargetSys;
 use aws_sdk_s3::config::{Credentials, Region};
@@ -362,19 +363,21 @@ impl Drop for SlowReplicationTargetGuard {
     }
 }
 
+// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
+// shape `mc replicate resync status` decodes.
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
-    #[serde(rename = "Targets", default)]
+    #[serde(rename = "target", default)]
     targets: Vec<ReplicationResetStatusTarget>,
 }
 
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusTarget {
-    #[serde(rename = "Arn", default)]
+    #[serde(rename = "arn", default)]
     arn: String,
-    #[serde(rename = "ResetID", default)]
+    #[serde(rename = "resetid", default)]
     reset_id: String,
-    #[serde(rename = "Status", default)]
+    #[serde(rename = "resyncStatus", default)]
     status: String,
 }
 
@@ -1654,19 +1657,30 @@ async fn wait_for_source_delete_marker_replication_failed(
         if response.status() != StatusCode::OK {
             return Err(format!("replication diff failed with status {}", response.status()).into());
         }
-        let diff: serde_json::Value = response.json().await?;
-        let failed = diff["Entries"].as_array().is_some_and(|entries| {
-            entries.iter().any(|entry| {
-                entry["Object"].as_str() == Some(key)
-                    && entry["IsDeleteMarker"].as_bool() == Some(true)
-                    && entry["ReplicationStatus"].as_str() == Some("FAILED")
-            })
+        // The default diff response is a madmin-style stream of bare DiffInfo
+        // JSON documents (one per line) with no envelope; assert the envelope
+        // is gone so an aggregate-shaped regression fails loudly here.
+        let body = response.text().await?;
+        let entries = body
+            .lines()
+            .filter(|line| !line.trim().is_empty())
+            .map(serde_json::from_str::<serde_json::Value>)
+            .collect::<Result<Vec<_>, _>>()?;
+        for entry in &entries {
+            if entry.get("Entries").is_some() {
+                return Err(format!("replication diff must stream bare DiffInfo documents, got envelope: {entry}").into());
+            }
+        }
+        let failed = entries.iter().any(|entry| {
+            entry["object"].as_str() == Some(key)
+                && entry["deletemarker"].as_bool() == Some(true)
+                && entry["rStatus"].as_str() == Some("FAILED")
         });
         if failed {
             return Ok(());
         }
         if tokio::time::Instant::now() >= deadline {
-            return Err(format!("source delete marker {key} never reported FAILED; last diff={diff}").into());
+            return Err(format!("source delete marker {key} never reported FAILED; last diff={body}").into());
         }
         sleep(Duration::from_millis(200)).await;
     }
@@ -2275,6 +2289,30 @@ async fn site_replication_state_edit(
     Ok(())
 }
 
+/// Start a bucket-level replication resync (`PUT ?replication-reset`) and
+/// return the target `(arn, reset_id)`, asserting the response carries the
+/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
+/// that `mc replicate resync start` decodes.
+async fn start_bucket_replication_reset(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication-reset", env.url);
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, None, None).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("replication reset start failed: {status} {body}").into());
+    }
+    let payload: serde_json::Value = response.json().await?;
+    let arn = payload["target"][0]["arn"].as_str().unwrap_or_default().to_string();
+    let reset_id = payload["target"][0]["resetid"].as_str().unwrap_or_default().to_string();
+    if arn.is_empty() || reset_id.is_empty() {
+        return Err(format!("replication reset response missing madmin target[0].arn/resetid: {payload}").into());
+    }
+    Ok((arn, reset_id))
+}
+
 async fn get_replication_reset_status(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -2435,6 +2473,107 @@ async fn build_replication_pair(
     Ok((source_env, target_env, source_bucket.to_string()))
 }
 
+/// P0-6: CopyObject creates a new object on the destination key, so it must be
+/// scheduled for bucket replication exactly like PutObject (MinIO
+/// CopyObjectHandler parity). Before the fix the copy path never consulted the
+/// replication config: the destination object stayed local forever (its status
+/// metadata was inherited wholesale from the source, so the scanner heal pass
+/// skipped it too — no PENDING/FAILED marker meant nothing to re-drive).
+#[tokio::test]
+#[serial]
+async fn test_copy_object_replicates_to_target() -> TestResult {
+    init_logging();
+
+    let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let target_bucket = "replication-check-dst";
+
+    let src_key = "copy-repl-source.txt";
+    let dst_key = "copy-repl-destination.txt";
+    let payload = b"copy object replication payload".to_vec();
+
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(src_key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+    assert_eq!(wait_for_object_on_target(&target_client, target_bucket, src_key).await?, payload);
+    // Wait for the source object's terminal COMPLETED status so the copy below
+    // starts from metadata that carries a stale terminal replication state; the
+    // copy must not inherit it (MinIO filterReplicationStatusMetadata parity)
+    // and must drive its own PENDING -> COMPLETED cycle.
+    wait_for_source_replication_status(&source_client, &source_bucket, src_key, "COMPLETED", false).await?;
+
+    source_client
+        .copy_object()
+        .bucket(&source_bucket)
+        .key(dst_key)
+        .copy_source(format!("{source_bucket}/{src_key}"))
+        .send()
+        .await?;
+
+    assert_eq!(
+        wait_for_object_on_target(&target_client, target_bucket, dst_key).await?,
+        payload,
+        "CopyObject destination must replicate to the remote target"
+    );
+    wait_for_source_replication_status(&source_client, &source_bucket, dst_key, "COMPLETED", false).await?;
+
+    Ok(())
+}
+
+/// P0-6 companion: snowball auto-extract writes each archive member as an
+/// independent object; every member must replicate to the remote target like a
+/// regular PUT (MinIO PutObjectExtract parity).
+#[tokio::test]
+#[serial]
+async fn test_snowball_extract_replicates_members_to_target() -> TestResult {
+    init_logging();
+
+    let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let target_bucket = "replication-check-dst";
+
+    let members: [(&str, &[u8]); 2] = [
+        ("snowball/member-one.txt", b"first member payload"),
+        ("snowball/member-two.txt", b"second member payload"),
+    ];
+
+    let mut builder = tokio_tar::Builder::new(std::io::Cursor::new(Vec::new()));
+    for (path, data) in members {
+        let mut header = tokio_tar::Header::new_gnu();
+        header.set_size(data.len() as u64);
+        header.set_mode(0o644);
+        header.set_cksum();
+        builder.append_data(&mut header, path, std::io::Cursor::new(data)).await?;
+    }
+    let archive = builder.into_inner().await?.into_inner();
+
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key("members.tar")
+        .metadata("Snowball-Auto-Extract", "true")
+        .body(ByteStream::from(archive))
+        .send()
+        .await?;
+
+    for (key, data) in members {
+        assert_eq!(
+            wait_for_object_on_target(&target_client, target_bucket, key).await?,
+            data,
+            "snowball-extracted member {key} must replicate to the remote target"
+        );
+        wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", false).await?;
+    }
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -2754,7 +2893,7 @@ async fn test_set_remote_target_update_requires_arn() -> Result<(), Box<dyn Erro
 
     assert_eq!(status, StatusCode::BAD_REQUEST);
     assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
-    assert!(body.to_ascii_lowercase().contains("arn is empty"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("arn is required"), "unexpected response: {body}");
 
     Ok(())
 }
@@ -2812,6 +2951,128 @@ async fn test_set_remote_target_update_rejects_missing_target() -> Result<(), Bo
     Ok(())
 }
 
+async fn send_set_replication_target_update_request(
+    source_env: &RustFSTestEnvironment,
+    source_bucket: &str,
+    ops: &[&str],
+    body: serde_json::Value,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let mut url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}&update=true",
+        source_env.url,
+        urlencoding::encode(source_bucket)
+    );
+    for op in ops {
+        url.push_str(&format!("&{op}=true"));
+    }
+    signed_request(
+        http::Method::PUT,
+        &url,
+        &source_env.access_key,
+        &source_env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await
+}
+
+async fn fetch_single_target(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+    let response = list_replication_targets_request(env, Some(bucket)).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    let mut targets: Vec<serde_json::Value> = response.json().await?;
+    assert_eq!(targets.len(), 1, "expected exactly one remote target");
+    Ok(targets.remove(0))
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_partial_update_preserves_credentials() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env
+        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
+        .await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-partial-update-src";
+    let target_bucket = "replication-partial-update-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+
+    // A sync-only update whose body omits credentials entirely must succeed and
+    // leave the stored connection settings untouched.
+    let response = send_set_replication_target_update_request(
+        &source_env,
+        source_bucket,
+        &["sync"],
+        serde_json::json!({
+            "arn": arn,
+            "type": "replication",
+            "replicationSync": true
+        }),
+    )
+    .await?;
+    assert_eq!(response.status(), StatusCode::OK, "sync-only update failed: {}", response.text().await?);
+
+    let target = fetch_single_target(&source_env, source_bucket).await?;
+    assert_eq!(target["replicationSync"], serde_json::json!(true));
+    assert_eq!(target["endpoint"], serde_json::json!(target_env.address));
+    assert_eq!(target["credentials"]["accessKey"], serde_json::json!(target_env.access_key));
+
+    // An update naming no field groups is a no-op: a body carrying a different
+    // endpoint and credentials must not leak into the stored target.
+    let response = send_set_replication_target_update_request(
+        &source_env,
+        source_bucket,
+        &[],
+        serde_json::json!({
+            "arn": arn,
+            "type": "replication",
+            "endpoint": "203.0.113.1:9000",
+            "credentials": { "accessKey": "other-access", "secretKey": "other-secret" },
+            "targetbucket": "elsewhere",
+            "secure": false,
+            "replicationSync": false
+        }),
+    )
+    .await?;
+    assert_eq!(response.status(), StatusCode::OK, "no-op update failed: {}", response.text().await?);
+
+    let target = fetch_single_target(&source_env, source_bucket).await?;
+    assert_eq!(
+        target["replicationSync"],
+        serde_json::json!(true),
+        "no-op update must not change sync mode"
+    );
+    assert_eq!(
+        target["endpoint"],
+        serde_json::json!(target_env.address),
+        "no-op update must not change endpoint"
+    );
+    assert_eq!(
+        target["credentials"]["accessKey"],
+        serde_json::json!(target_env.access_key),
+        "no-op update must not change credentials"
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -3714,7 +3975,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
   <Role></Role>
   <Rule>
     <ID>matrix-prefix</ID>
-    <Priority>100</Priority>
+    <Priority>110</Priority>
     <Status>Enabled</Status>
     <Filter><Prefix>prefix/</Prefix></Filter>
     <DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
@@ -3725,7 +3986,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
   </Rule>
   <Rule>
     <ID>matrix-both-prefix</ID>
-    <Priority>100</Priority>
+    <Priority>120</Priority>
     <Status>Enabled</Status>
     <Filter><Prefix>both/</Prefix></Filter>
     <DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
@@ -3735,7 +3996,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
   </Rule>
   <Rule>
     <ID>matrix-tag</ID>
-    <Priority>100</Priority>
+    <Priority>130</Priority>
     <Status>Enabled</Status>
     <Filter><Tag><Key>route</Key><Value>tagged</Value></Tag></Filter>
     <DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>
@@ -3745,7 +4006,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
   </Rule>
   <Rule>
     <ID>matrix-disabled</ID>
-    <Priority>100</Priority>
+    <Priority>140</Priority>
     <Status>Disabled</Status>
     <Filter><Prefix>disabled/</Prefix></Filter>
     <DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
@@ -4227,16 +4488,76 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
 }
 
 /// backlog#1147 repl-17 / backlog#1291: SSE-S3 must fail closed until managed
-/// encryption is supported on the target. The current plaintext replication is
-/// a known security bug, so this pins the required contract without blessing it.
+/// encryption is supported on the target. The silent plaintext replication
+/// that originally kept this test ignored was fixed by the fail-closed gate in
+/// `crates/ecstore/src/bucket/replication/replication_target_boundary.rs`
+/// (all replication modes route through it), so this now pins the current
+/// fail-closed contract: FAILED status, failure event, readable source, and a
+/// stable absence of all target versions.
 #[tokio::test]
 #[serial]
-#[ignore = "backlog#1291: SSE-S3 replication silently drops encryption"]
 async fn test_bucket_replication_sse_s3_contract() -> TestResult {
     init_logging();
     assert_managed_sse_replication_fails_explicitly("sse-s3", false).await
 }
 
+/// P1-22 stage 0: the existing-object resync path must fail closed for
+/// managed-SSE objects exactly like inline replication (which
+/// `test_bucket_replication_sse_s3_contract` pins, including the scanner heal
+/// re-drive). Resync re-drives every object version through the same
+/// fail-closed target boundary, so a resync over an encrypted bucket must
+/// terminate without ever materializing a plaintext (or unreadable) replica;
+/// the post-resync stays-absent window also spans further fast-scanner heal
+/// cycles.
+#[tokio::test]
+#[serial]
+async fn test_bucket_replication_sse_s3_resync_stays_fail_closed() -> TestResult {
+    init_logging();
+
+    let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("sse-resync", true).await?;
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let key = "sse-resync-contract.txt";
+    let body = b"repl-22 sse resync payload".to_vec();
+
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(key)
+        .body(ByteStream::from(body.clone()))
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    wait_for_source_replication_status(&source_client, &source_bucket, key, "FAILED", false).await?;
+
+    // Resync: drive the existing-object resync path over the failed object.
+    let (target_arn, reset_id) = start_bucket_replication_reset(&source_env, &source_bucket).await?;
+    let terminal = wait_for_replication_reset_target(&source_env, &source_bucket, &target_arn, |target| {
+        target.reset_id == reset_id && matches!(target.status.as_str(), "Completed" | "Failed")
+    })
+    .await?;
+    assert_eq!(terminal.reset_id, reset_id);
+
+    // The resync pass must have failed closed: still no target version (the
+    // window also spans further scanner heal cycles), and the source object
+    // stays readable and encrypted.
+    assert_failed_replication_stays_absent_for(
+        &source_client,
+        &source_bucket,
+        &target_client,
+        &target_bucket,
+        key,
+        false,
+        Duration::from_secs(5),
+    )
+    .await?;
+    let source = source_client.get_object().bucket(&source_bucket).key(key).send().await?;
+    assert_eq!(source.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+    assert_eq!(source.body.collect().await?.into_bytes().as_ref(), body.as_slice());
+
+    Ok(())
+}
+
 /// backlog#1147 repl-17: SSE-KMS currently fails closed rather than creating an
 /// unreadable replica; the shared helper verifies FAILED, the failure event,
 /// source readability, and a stable absence of all target versions.
@@ -6517,3 +6838,140 @@ async fn test_site_replication_replicates_service_accounts_created_from_sts_sess
 
     Ok(())
 }
+
+/// Poll the fake target journal until `operation` arrives for `key`, then
+/// return the `versionId` query value the request carried.
+async fn wait_for_target_request_version_id(
+    target: &FakeS3Target,
+    operation: FakeTargetOperation,
+    key: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
+    loop {
+        if let Some(record) = target
+            .requests()
+            .into_iter()
+            .find(|record| record.operation == operation && record.key.as_deref() == Some(key))
+        {
+            return Ok(record.version_id);
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("fake target never received {operation:?} for {key}; journal: {:?}", target.requests()).into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+/// P0-5: MinIO derives the replicated version exclusively from the `versionId`
+/// query parameter (`putOptsFromReq`); the internal x-*-source-version-id
+/// headers do not exist there. Without the query, a MinIO target mints fresh
+/// version ids and RustFS -> MinIO replication drifts. PutObject and
+/// CreateMultipartUpload (the version is decided at initiate time) must both
+/// carry the source version as `?versionId=`.
+#[tokio::test]
+#[serial]
+async fn test_replication_put_and_create_multipart_carry_source_version_id_query() -> TestResult {
+    init_logging();
+
+    let target = FakeS3Target::start().await?;
+    let target_bucket = "versionid-query-dst";
+    target.create_bucket(target_bucket);
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    let mut source_process_env = replication_fast_env();
+    source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+    source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
+    source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
+
+    let source_bucket = "versionid-query-src";
+    let source_client = source_env.create_s3_client();
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+
+    let target_arn = set_replication_target_with_options(
+        &source_env,
+        source_bucket,
+        ReplicationTargetOptions {
+            endpoint: &target.address(),
+            access_key: FAKE_ACCESS_KEY,
+            secret_key: FAKE_SECRET_KEY,
+            target_bucket,
+            secure: false,
+            skip_tls_verify: false,
+            ca_cert_pem: None,
+        },
+    )
+    .await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    // Small object -> replicated through a single PutObject.
+    let put = source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key("small.txt")
+        .body(ByteStream::from_static(b"versionid query payload"))
+        .send()
+        .await?;
+    let put_source_version = put
+        .version_id()
+        .ok_or("versioned source PUT must return a version id")?
+        .to_string();
+    let recorded = wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "small.txt").await?;
+    assert_eq!(
+        recorded.as_deref(),
+        Some(put_source_version.as_str()),
+        "replication PutObject must carry the source version in the versionId query"
+    );
+
+    // Multipart source object -> replicated through CreateMultipartUpload;
+    // the target version is fixed at initiate time.
+    let create = source_client
+        .create_multipart_upload()
+        .bucket(source_bucket)
+        .key("large.bin")
+        .send()
+        .await?;
+    let upload_id = create
+        .upload_id()
+        .ok_or("multipart initiate must return an upload id")?
+        .to_string();
+    let mut completed_parts = Vec::new();
+    for (part_number, body) in [(1, vec![b'a'; 5 * 1024 * 1024]), (2, vec![b'b'; 1024])] {
+        let uploaded = source_client
+            .upload_part()
+            .bucket(source_bucket)
+            .key("large.bin")
+            .upload_id(&upload_id)
+            .part_number(part_number)
+            .body(ByteStream::from(body))
+            .send()
+            .await?;
+        completed_parts.push(
+            CompletedPart::builder()
+                .part_number(part_number)
+                .e_tag(uploaded.e_tag().unwrap_or_default())
+                .build(),
+        );
+    }
+    let complete = source_client
+        .complete_multipart_upload()
+        .bucket(source_bucket)
+        .key("large.bin")
+        .upload_id(&upload_id)
+        .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
+        .send()
+        .await?;
+    let multipart_source_version = complete
+        .version_id()
+        .ok_or("versioned multipart completion must return a version id")?
+        .to_string();
+    let recorded = wait_for_target_request_version_id(&target, FakeTargetOperation::CreateMultipartUpload, "large.bin").await?;
+    assert_eq!(
+        recorded.as_deref(),
+        Some(multipart_source_version.as_str()),
+        "replication CreateMultipartUpload must carry the source version in the versionId query"
+    );
+
+    target.shutdown().await;
+    Ok(())
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -12,13 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
-use aws_sdk_sts::config::retry::RetryConfig;
-use aws_sdk_sts::config::{Credentials, Region};
+use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
+use aws_sdk_sts::Client;
 use aws_sdk_sts::error::ProvideErrorMetadata;
 use aws_sdk_sts::operation::RequestId;
-use aws_sdk_sts::{Client, Config};
-use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use bytes::Bytes;
 use http::header::{AUTHORIZATION, CONTENT_TYPE};
 use http::{Request, Response};
@@ -32,9 +29,8 @@ use serial_test::serial;
 use std::collections::BTreeSet;
 use std::convert::Infallible;
 use std::error::Error;
-use std::sync::Arc;
 use tokio::net::TcpListener;
-use tokio::sync::{Notify, mpsc};
+use tokio::sync::mpsc;
 use tokio::task::{JoinHandle, JoinSet};
 use tokio::time::{Duration, timeout};
 
@@ -43,22 +39,7 @@ type TestResult = Result<(), BoxError>;
 const OPA_AUTH_TOKEN: &str = "sts-opa-token";
 
 fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
-    let mut config = Config::builder()
-        .credentials_provider(Credentials::new(
-            access_key,
-            secret_key,
-            session_token.map(str::to_owned),
-            None,
-            "e2e-sts-query-compat",
-        ))
-        .region(Region::new("us-east-1"))
-        .endpoint_url(url)
-        .retry_config(RetryConfig::standard().with_max_attempts(1))
-        .behavior_version_latest();
-    if url.starts_with("http://") {
-        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
-    }
-    Client::from_conf(config.build())
+    build_test_sts_client(url, access_key, secret_key, session_token, "e2e-sts-query-compat")
 }
 
 async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
@@ -145,6 +126,52 @@ async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
     Ok(())
 }
 
+async fn assert_list_buckets_access_denied(
+    env: &RustFSTestEnvironment,
+    access_key: &str,
+    secret_key: &str,
+    context: &str,
+) -> TestResult {
+    let error = aws_sdk_s3::Client::from_conf(build_test_s3_config(
+        &env.url,
+        access_key,
+        secret_key,
+        None,
+        "e2e-list-buckets-opa-unavailable",
+    ))
+    .list_buckets()
+    .send()
+    .await
+    .expect_err("ListBuckets must be denied while OPA is unavailable");
+    let service_error = error
+        .as_service_error()
+        .ok_or_else(|| format!("{context} should deserialize as an S3 service error: {error:?}"))?;
+
+    assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
+    assert_eq!(service_error.code(), Some("AccessDenied"));
+    Ok(())
+}
+
+async fn assert_opa_unavailable_denies_sts_and_list_buckets(env: &RustFSTestEnvironment, context: &str) -> TestResult {
+    let user = "opaunavailable";
+    let secret = "stsOpaUnavailableSecret123";
+    create_user_with_policy(
+        env,
+        user,
+        secret,
+        "sts-opa-unavailable-local-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+
+    assert_access_denied(&sts_client(&env.url, user, secret, None), context).await?;
+    assert_list_buckets_access_denied(env, user, secret, context).await
+}
+
 async fn handle_opa_request(
     request: Request<Incoming>,
     requests: mpsc::UnboundedSender<Value>,
@@ -186,12 +213,15 @@ async fn handle_opa_request(
     };
     if payload.is_none() {
         let _ = validation_started.send(());
-        if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
-            release.notified().await;
-            return Ok(Response::builder()
-                .status(503)
-                .body(Full::new(Bytes::new()))
-                .expect("static OPA unavailable response must be valid"));
+        match validation_mode {
+            OpaValidationMode::Blocked => std::future::pending::<()>().await,
+            OpaValidationMode::Unavailable => {
+                return Ok(Response::builder()
+                    .status(503)
+                    .body(Full::new(Bytes::new()))
+                    .expect("static OPA unavailable response must be valid"));
+            }
+            OpaValidationMode::Ready => {}
         }
     }
     let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
@@ -201,6 +231,25 @@ async fn handle_opa_request(
             .and_then(Value::as_bool)
             .unwrap_or(false),
         Some(Value::String(account)) if account == "opadeny" => false,
+        Some(Value::String(account))
+            if account == "opaunavailable" && matches!(validation_mode, OpaValidationMode::Unavailable) =>
+        {
+            true
+        }
+        Some(Value::String(account)) if account == "opalistbuckets" => {
+            let action = payload
+                .as_ref()
+                .and_then(|value| value.pointer("/input/action"))
+                .and_then(Value::as_str);
+            let bucket = payload
+                .as_ref()
+                .and_then(|value| value.pointer("/input/resource/bucket"))
+                .and_then(Value::as_str);
+            matches!(
+                (action, bucket),
+                (Some("s3:ListBucket"), Some("opa-list-visible")) | (Some("s3:GetBucketLocation"), Some("opa-list-location"))
+            )
+        }
         None => true,
         _ => false,
     };
@@ -215,17 +264,17 @@ async fn handle_opa_request(
         .expect("static OPA response must be valid"))
 }
 
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 enum OpaValidationMode {
     Ready,
-    DelayedUnavailable(Arc<Notify>),
+    Blocked,
+    Unavailable,
 }
 
 struct OpaMock {
     url: String,
     requests: mpsc::UnboundedReceiver<Value>,
     validation_started: mpsc::UnboundedReceiver<()>,
-    validation_release: Option<Arc<Notify>>,
     task: JoinHandle<()>,
 }
 
@@ -234,9 +283,8 @@ impl OpaMock {
         Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
     }
 
-    async fn start_delayed_unavailable() -> Result<Self, BoxError> {
-        let release = Arc::new(Notify::new());
-        Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
+    async fn start_blocked() -> Result<Self, BoxError> {
+        Self::start_with_mode(OpaValidationMode::Blocked, None).await
     }
 
     async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
@@ -245,10 +293,6 @@ impl OpaMock {
         let (requests_tx, requests) = mpsc::unbounded_channel();
         let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
         let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
-        let validation_release = match &validation_mode {
-            OpaValidationMode::Ready => None,
-            OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
-        };
         let task = tokio::spawn(async move {
             let mut connections = JoinSet::new();
             loop {
@@ -257,7 +301,7 @@ impl OpaMock {
                         let Ok((stream, _)) = accepted else { break };
                         let requests = requests_tx.clone();
                         let validation_started = validation_started_tx.clone();
-                        let validation_mode = validation_mode.clone();
+                        let validation_mode = validation_mode;
                         let expected_authorization = expected_authorization.clone();
                         connections.spawn(async move {
                             let handler = service_fn(move |request| {
@@ -265,7 +309,7 @@ impl OpaMock {
                                     request,
                                     requests.clone(),
                                     validation_started.clone(),
-                                    validation_mode.clone(),
+                                    validation_mode,
                                     expected_authorization.clone(),
                                 )
                             });
@@ -282,7 +326,6 @@ impl OpaMock {
             url,
             requests,
             validation_started,
-            validation_release,
             task,
         })
     }
@@ -298,12 +341,6 @@ impl OpaMock {
             .await?
             .ok_or_else(|| "OPA validation channel closed".into())
     }
-
-    fn release_validation(&self) {
-        if let Some(release) = &self.validation_release {
-            release.notify_one();
-        }
-    }
 }
 
 impl Drop for OpaMock {
@@ -523,35 +560,119 @@ async fn test_sts_assume_role_opa_contract() -> TestResult {
 
 #[tokio::test]
 #[serial]
-async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
+async fn test_list_buckets_opa_contract() -> TestResult {
     init_logging();
 
-    let mut opa = OpaMock::start_delayed_unavailable().await?;
+    let mut opa = OpaMock::start().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+            ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
+        ],
+    )
+    .await?;
+
+    let admin_client = env.create_s3_client();
+    for bucket in ["opa-list-hidden", "opa-list-location", "opa-list-visible"] {
+        admin_client.create_bucket().bucket(bucket).send().await?;
+    }
+
+    let user = "opalistbuckets";
+    let secret = "opaListBucketsSecret123";
+    create_user(&env, user, secret).await?;
+
+    let output = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, user, secret, None, "e2e-list-buckets-opa"))
+        .list_buckets()
+        .send()
+        .await?;
+    let mut names = output
+        .buckets()
+        .iter()
+        .filter_map(|bucket| bucket.name().map(str::to_owned))
+        .collect::<Vec<_>>();
+    names.sort();
+    assert_eq!(names, ["opa-list-location", "opa-list-visible"]);
+
+    let mut evaluations = BTreeSet::new();
+    for _ in 0..6 {
+        let request = opa.next_request().await?;
+        assert_eq!(request.pointer("/input/identity/account").and_then(Value::as_str), Some(user));
+        assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(false));
+
+        let action = request
+            .pointer("/input/action")
+            .and_then(Value::as_str)
+            .ok_or("OPA ListBuckets input should include action")?;
+        let bucket = request
+            .pointer("/input/resource/bucket")
+            .and_then(Value::as_str)
+            .ok_or("OPA ListBuckets input should include resource.bucket")?;
+        if bucket.is_empty() {
+            assert_eq!(action, "s3:ListAllMyBuckets");
+            assert!(request.pointer("/input/context/conditions/prefix").is_none());
+            assert!(request.pointer("/input/context/conditions/delimiter").is_none());
+        } else {
+            let expected_arn = format!("arn:aws:s3:::{bucket}");
+            assert_eq!(request.pointer("/input/context/conditions/prefix"), Some(&serde_json::json!([""])));
+            assert_eq!(request.pointer("/input/context/conditions/delimiter"), Some(&serde_json::json!(["/"])));
+            assert_eq!(
+                request.pointer("/input/resource/arn").and_then(Value::as_str),
+                Some(expected_arn.as_str())
+            );
+        }
+        evaluations.insert((action.to_owned(), bucket.to_owned()));
+    }
+    assert_eq!(
+        evaluations,
+        BTreeSet::from([
+            ("s3:GetBucketLocation".to_owned(), "opa-list-hidden".to_owned()),
+            ("s3:GetBucketLocation".to_owned(), "opa-list-location".to_owned()),
+            ("s3:ListAllMyBuckets".to_owned(), String::new()),
+            ("s3:ListBucket".to_owned(), "opa-list-hidden".to_owned()),
+            ("s3:ListBucket".to_owned(), "opa-list-location".to_owned()),
+            ("s3:ListBucket".to_owned(), "opa-list-visible".to_owned()),
+        ])
+    );
+    assert!(
+        matches!(opa.requests.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
+        "ListBuckets should not make redundant OPA evaluations"
+    );
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start_blocked().await?;
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
         .await?;
     opa.wait_for_validation().await?;
 
-    let user = "opaunavailable";
-    let secret = "stsOpaUnavailableSecret123";
-    create_user_with_policy(
-        &env,
-        user,
-        secret,
-        "sts-opa-unavailable-local-policy",
-        serde_json::json!([{
-            "Effect": "Allow",
-            "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
+    assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA initialization").await?;
 
-    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
+    env.stop_server();
+    Ok(())
+}
 
-    opa.release_validation();
-    tokio::time::sleep(Duration::from_millis(200)).await;
-    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
+#[tokio::test]
+#[serial]
+async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start_with_mode(OpaValidationMode::Unavailable, None).await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
+        .await?;
+    opa.wait_for_validation().await?;
+
+    assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA validation failure").await?;
 
     env.stop_server();
     Ok(())

crates/ecstore/Cargo.toml

@@ -144,11 +144,13 @@ rustfs-lifecycle.workspace = true
 rustfs-s3-types = { workspace = true }
 rustfs-data-usage.workspace = true
 rustfs-object-capacity.workspace = true
+rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
 arc-swap.workspace = true
 async-trait.workspace = true
 bytes = { workspace = true, features = ["serde"] }
 byteorder = { workspace = true }
 chrono = { workspace = true, features = ["serde"] }
+jiff = { workspace = true, features = ["serde"] }
 glob = { workspace = true }
 thiserror.workspace = true
 flatbuffers.workspace = true

crates/ecstore/src/api/mod.rs

@@ -185,19 +185,19 @@ pub mod bucket {
             MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
             REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
             REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
-            ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
-            ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
-            ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
-            ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
-            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
+            ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
+            ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
+            ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
+            ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
+            ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
+            TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
             delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
             get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
             persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
             replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
             should_schedule_delete_replication, should_use_existing_delete_replication_info,
             should_use_existing_delete_replication_source, unsupported_replication_config_field,
-            validate_replication_config_target_arns, version_purge_status_to_filemeta,
+            validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
         };
     }
 
@@ -206,7 +206,9 @@ pub mod bucket {
     }
 
     pub mod target {
-        pub use crate::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat};
+        pub use crate::bucket::target::{
+            ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat, duration_from_secs_or_nanos,
+        };
     }
 
     pub mod utils {
@@ -308,7 +310,8 @@ pub mod config {
 pub mod data_usage {
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
-        init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
+        init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
+        invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
         load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
         record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
         record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
@@ -437,7 +440,7 @@ pub mod rpc {
         gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
         node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
         set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
-        tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, verify_tonic_boot_epoch_response,
         verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
         verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -1424,6 +1424,37 @@ fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObject
     }
 }
 
+/// Resolve the S3 `versionId` query parameter for a replication PUT /
+/// CreateMultipartUpload against a remote target.
+///
+/// MinIO reads the replicated version only from the query string
+/// (`putOptsFromReq`); the internal `x-*-source-version-id` headers do not
+/// exist there, so without the query a MinIO target mints fresh version ids
+/// and the deployments drift apart. RustFS represents the null version
+/// internally as the nil UUID while the S3 API addresses it as the literal
+/// "null" (the delete path already maps it via `target_delete_version_id`),
+/// and an empty id means the source object carries no version: send no query
+/// so an unversioned target stays valid.
+fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
+    if source_version_id.is_empty() {
+        None
+    } else if Uuid::parse_str(source_version_id).is_ok_and(|uuid| uuid.is_nil()) {
+        Some(rustfs_filemeta::NULL_VERSION_ID)
+    } else {
+        Some(source_version_id)
+    }
+}
+
+/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
+/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
+/// member, so the query is spliced in via `map_request`, which runs at
+/// `modify_before_signing`: the parameter becomes part of the SigV4 canonical
+/// request.
+fn append_version_id_query(uri: &str, version_id: &str) -> String {
+    let separator = if uri.contains('?') { '&' } else { '?' };
+    format!("{uri}{separator}versionId={}", urlencoding::encode(version_id))
+}
+
 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
     pub source_version_id: String,
@@ -1831,6 +1862,7 @@ impl TargetClient {
         if !version_id.is_empty() {
             insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
         }
+        let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned);
 
         match builder
             .bucket(bucket)
@@ -1845,6 +1877,11 @@ impl TargetClient {
                         req.headers_mut().insert(key_str, value_str);
                     }
                 }
+                if let Some(version_id) = &api_version_id {
+                    let uri = append_version_id_query(req.uri(), version_id);
+                    req.set_uri(uri)
+                        .map_err(aws_smithy_types::error::operation::BuildError::other)?;
+                }
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
@@ -1893,6 +1930,9 @@ impl TargetClient {
         if opts.internal.replication_request {
             insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
         }
+        // The remote version of a multipart replication is decided at initiate
+        // time; CompleteMultipartUpload does not read a versionId.
+        let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned);
 
         match self
             .client
@@ -1907,6 +1947,11 @@ impl TargetClient {
                         req.headers_mut().insert(key_str, value_str);
                     }
                 }
+                if let Some(version_id) = &api_version_id {
+                    let uri = append_version_id_query(req.uri(), version_id);
+                    req.set_uri(uri)
+                        .map_err(aws_smithy_types::error::operation::BuildError::other)?;
+                }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
             .send()
@@ -2679,6 +2724,91 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn put_object_sends_source_version_id_query_to_target() {
+        // MinIO reads the replicated version only from the `versionId` query
+        // parameter (its receive path ignores the x-*-source-version-id
+        // headers), so the query must carry the source version: a real UUID
+        // as-is, the internal nil-UUID null-version representation as the
+        // literal "null", and no query at all when the source object has no
+        // version (P0-5 RustFS->MinIO version drift).
+        let (client, request_uris) = recording_target_client();
+        let version_id = Uuid::new_v4().to_string();
+        let nil_version = Uuid::nil().to_string();
+        for source_version in [version_id.as_str(), nil_version.as_str(), ""] {
+            let mut opts = PutObjectOptions::default();
+            opts.internal.source_version_id = source_version.to_string();
+            opts.internal.replication_request = true;
+            client
+                .put_object("target-bucket", "object", 4, ByteStream::from_static(b"data"), &opts)
+                .await
+                .expect("recorded put_object should succeed");
+        }
+
+        let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
+        assert_eq!(request_uris.len(), 3);
+        assert!(
+            request_uris[0].contains(&format!("versionId={version_id}")),
+            "replication put_object must carry the source version as a versionId query: {}",
+            request_uris[0]
+        );
+        assert!(
+            request_uris[1].contains("versionId=null"),
+            "a nil-UUID (null) source version must be sent as the literal null: {}",
+            request_uris[1]
+        );
+        assert!(
+            !request_uris[2].contains("versionId="),
+            "put_object without a source version must omit the versionId query: {}",
+            request_uris[2]
+        );
+    }
+
+    #[tokio::test]
+    async fn create_multipart_upload_sends_source_version_id_query_to_target() {
+        // The remote version of a multipart replication is decided at initiate
+        // time: CreateMultipartUpload must carry the source version in the
+        // `versionId` query (CompleteMultipartUpload does not read one).
+        let (client, request_uris) = recording_target_client();
+        let version_id = Uuid::new_v4().to_string();
+        let nil_version = Uuid::nil().to_string();
+        for source_version in [version_id.as_str(), nil_version.as_str()] {
+            let mut opts = PutObjectOptions::default();
+            opts.internal.source_version_id = source_version.to_string();
+            opts.internal.replication_request = true;
+            let _ = client.create_multipart_upload("target-bucket", "object", &opts).await;
+        }
+
+        let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
+        assert_eq!(request_uris.len(), 2);
+        assert!(
+            request_uris[0].contains(&format!("versionId={version_id}")),
+            "replication create_multipart_upload must carry the source version as a versionId query: {}",
+            request_uris[0]
+        );
+        assert!(
+            request_uris[1].contains("versionId=null"),
+            "a nil-UUID (null) source version must be sent as the literal null: {}",
+            request_uris[1]
+        );
+    }
+
+    #[test]
+    fn put_object_headers_keep_source_version_id_for_legacy_receivers() {
+        // Older RustFS receivers have no versionId query support and fall back
+        // to the internal source-version-id headers (rolling-upgrade path);
+        // the query addition must never remove them.
+        let mut opts = PutObjectOptions::default();
+        let version_id = Uuid::new_v4().to_string();
+        opts.internal.source_version_id = version_id.clone();
+
+        assert_eq!(
+            rustfs_utils::http::get_header(&opts.header(), SUFFIX_SOURCE_VERSION_ID).as_deref(),
+            Some(version_id.as_str()),
+            "replication put requests must keep the internal source-version-id headers"
+        );
+    }
+
     #[test]
     fn put_object_headers_include_non_empty_source_etag_only() {
         let mut opts = PutObjectOptions::default();

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -10774,7 +10774,18 @@ mod tests {
     #[serial]
     async fn tier_free_version_recovery_real_enqueue_failure_retries_same_object() {
         let (disk_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("recovery-enqueue-failure-{}", Uuid::new_v4());
+        let object = "free-version-b";
+        let start_marker = "free-version-a0";
+        create_test_bucket(&ecstore, &bucket).await;
+        seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
+
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
+        let recovery_rx = {
+            let state = runtime_state.read().await;
+            Arc::clone(&state.tasks_rx[0])
+        };
+        let mut recovery_rx = recovery_rx.lock().await;
         assert!(
             super::enqueue_recovered_free_version(ObjectInfo {
                 bucket: "prefill".to_string(),
@@ -10784,20 +10795,29 @@ mod tests {
             .await,
             "the production recovery queue should accept its first task"
         );
-        let bucket = format!("recovery-enqueue-failure-{}", Uuid::new_v4());
-        let object = "free-version";
-        create_test_bucket(&ecstore, &bucket).await;
-        seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
 
-        let first = recover_tier_free_versions_with_cancel(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
-            .await
-            .expect("queue failure should return retry markers");
+        let first = recover_tier_free_versions_with_cancel(
+            Arc::clone(&ecstore),
+            1,
+            Some(bucket.clone()),
+            Some(start_marker.to_string()),
+            CancellationToken::new(),
+        )
+        .await
+        .expect("queue failure should return retry markers");
         assert_eq!(first.scanned, 1);
         assert_eq!(first.enqueued, 0);
         assert_eq!(first.failed, 1);
         assert!(first.truncated);
         assert_eq!(first.next_bucket_marker.as_deref(), Some(bucket.as_str()));
-        assert!(first.next_object_marker.is_none());
+        assert_eq!(first.next_object_marker.as_deref(), Some(start_marker));
+
+        drop(
+            recovery_rx
+                .try_recv()
+                .expect("the failed recovery attempt must leave the prefilled task queued")
+                .expect("the prefilled recovery queue entry should contain a task"),
+        );
 
         let retried = recover_tier_free_versions_with_cancel(
             Arc::clone(&ecstore),
@@ -10809,7 +10829,19 @@ mod tests {
         .await
         .expect("retry markers should revisit the failed free version");
         assert_eq!(retried.scanned, 1);
-        assert_eq!(retried.failed, 1);
+        assert_eq!(retried.enqueued, 1);
+        assert_eq!(retried.failed, 0);
+
+        let retried_task = recovery_rx
+            .try_recv()
+            .expect("the retry should enqueue the recovered free-version task")
+            .expect("the recovered queue entry should contain a task");
+        let retried_task = retried_task
+            .as_any()
+            .downcast_ref::<FreeVersionTask>()
+            .expect("the recovered queue entry should be a free-version task");
+        assert_eq!(retried_task.0.bucket, bucket);
+        assert_eq!(retried_task.0.name, object);
 
         remove_seeded_free_version(&disk_paths, &bucket, object).await;
         ecstore

crates/ecstore/src/bucket/quota/checker.rs

@@ -198,11 +198,36 @@ impl QuotaChecker {
     }
 
     pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
-        get_bucket_usage_memory(bucket)
-            .await
-            .ok_or_else(|| QuotaError::UsageUnavailable {
-                bucket: bucket.to_string(),
-            })
+        if let Some(usage) = get_bucket_usage_memory(bucket).await {
+            return Ok(usage);
+        }
+
+        // Degraded window (issue #5716): with no authoritative usage — most
+        // prominently after upgrading from a pre-v2 release, whose legacy
+        // `.usage.json` is demoted to non-authoritative until the scanner's
+        // first complete cycle persists `.usage.v2.json` — failing closed
+        // turned every write to a quota-enabled bucket into a retryable 503
+        // for the whole window. Quota admission instead degrades to the last
+        // persisted per-bucket size. That baseline is static between snapshot
+        // loads (live writes do not advance it), so hard-quota enforcement is
+        // advisory for the duration of the window: the overrun is bounded by
+        // the writes issued before the next complete scanner cycle. Buckets
+        // with no persisted baseline anywhere keep failing closed.
+        let store = self.metadata_sys.read().await.object_store();
+        // Box the fallback: it embeds the whole snapshot-load future, and every
+        // object write nests a quota check several futures deep, so keeping it
+        // inline would grow each write's state machine by the loader's full
+        // size — the debug-build 2MiB worker-stack overflow class fixed for
+        // bucket-config writes in #5648. The allocation only happens on the
+        // degraded path; the authoritative fast path returns above.
+        if let Some(baseline) = Box::pin(crate::data_usage::lookup_degraded_bucket_usage_baseline(store, bucket)).await {
+            debug!(bucket, baseline, "Bucket quota admission using degraded persisted usage baseline");
+            return Ok(baseline);
+        }
+
+        Err(QuotaError::UsageUnavailable {
+            bucket: bucket.to_string(),
+        })
     }
 }
 
@@ -232,6 +257,76 @@ mod tests {
         assert_eq!(result.quota_limit, None);
     }
 
+    /// Regression (issue #5716): an upgrade from a pre-v2 release leaves only
+    /// the legacy `.usage.json` snapshot, which has no completeness marker and
+    /// is demoted to non-authoritative, and the scanner's first complete cycle
+    /// can be a long way off. Quota admission must degrade to that persisted
+    /// baseline instead of failing every write to a quota-enabled bucket with
+    /// a retryable 503 for the whole window.
+    #[tokio::test]
+    #[serial]
+    async fn quota_admission_falls_back_to_legacy_snapshot_baseline() {
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
+        let checker = QuotaChecker::new(sys);
+        let bucket = format!("quota-legacy-{}", Uuid::new_v4().simple());
+
+        let mut legacy = rustfs_data_usage::DataUsageInfo {
+            last_update: Some(std::time::SystemTime::now()),
+            buckets_count: 1,
+            ..Default::default()
+        };
+        legacy.buckets_usage.insert(
+            bucket.clone(),
+            rustfs_data_usage::BucketUsageInfo {
+                size: 1_234,
+                ..Default::default()
+            },
+        );
+        legacy.bucket_sizes.insert(bucket.clone(), 1_234);
+        // usage_snapshot_complete stays false: pre-v2 snapshots do not carry
+        // the field at all, so they always deserialize as incomplete.
+        let legacy_path = format!("{}/{}", crate::disk::BUCKET_META_PREFIX, rustfs_data_usage::LEGACY_DATA_USAGE_OBJECT_NAME);
+        crate::config::com::save_config(
+            ecstore.clone(),
+            &legacy_path,
+            serde_json::to_vec(&legacy).expect("legacy snapshot should encode"),
+        )
+        .await
+        .expect("legacy snapshot fixture should be stored");
+        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+
+        let usage = checker
+            .get_real_time_usage(&bucket)
+            .await
+            .expect("quota admission must degrade to the persisted legacy baseline");
+        assert_eq!(usage, 1_234);
+
+        // A bucket absent from every persisted snapshot still has no grounded
+        // baseline and must keep failing closed.
+        let unknown = format!("quota-unknown-{}", Uuid::new_v4().simple());
+        assert!(matches!(
+            checker.get_real_time_usage(&unknown).await,
+            Err(QuotaError::UsageUnavailable { .. })
+        ));
+
+        // Deleting the bucket's usage from the backend must purge the
+        // baseline: a recreated bucket may not inherit the dead incarnation's
+        // size, so with no persisted trace left it fails closed again.
+        crate::data_usage::remove_bucket_usage_from_backend(ecstore.clone(), &bucket)
+            .await
+            .expect("bucket usage removal should succeed");
+        assert!(matches!(
+            checker.get_real_time_usage(&bucket).await,
+            Err(QuotaError::UsageUnavailable { .. })
+        ));
+
+        crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
+            .await
+            .expect("test usage cache cleanup should succeed");
+        crate::data_usage::invalidate_data_usage_snapshot_cache().await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_usage_rejects_an_unknown_mutation_baseline() {

crates/ecstore/src/bucket/replication/README.md

@@ -100,11 +100,41 @@ paths.
     behind the ECStore replication facade; only `rustfs/src/app/storage_api.rs`
     may retain direct object/delete replication helper calls.
 
-## First Code-Bearing Step
+## Completion Criteria
 
-Start with `ReplicationRuntime` or `ReplicationEventSink`. Both can be added as
-narrow internal contracts while keeping current queue, MRF, resync, and target
-behavior unchanged. Do not start with a crate move.
+The split is complete when the "Current dependency to remove" column in the
+Required Contracts table above is empty: every row is either deleted because
+the dependency is gone, or reduced to "none". No other signal — file count,
+boundary count, line count — measures completion.
+
+Target end state:
+
+- `replication_pool.rs`, `replication_resyncer.rs`, and `replication_state.rs`
+  move into `crates/replication` behind the contracts above;
+- the `*_boundary.rs` and `*_bridge.rs` micro-files dissolve naturally as the
+  code they fence moves across the crate boundary. They are the mechanical
+  seams of the migration ratchet — the architecture guard scripts anchor on
+  their file names — so batch-merging them beforehand is explicitly rejected:
+  it forces synchronized guard-script/mod/import churn with zero functional
+  gain;
+- the only module that can retire early is `datatypes.rs`: delete it once its
+  facade consumers import the resync status enums through `rustfs-replication`
+  directly.
+
+## Milestones
+
+| Milestone | Scope | Status |
+|---|---|---|
+| M0 | Record the completion criteria and end state (this section). | Done |
+| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
+| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
+| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
+| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
+
+The original first code-bearing step (narrow `ReplicationEventSink` /
+`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
+exists and runtime access goes through local boundary aliases — so new work
+starts from M2.
 
 Current compatibility guard: `crates/ecstore/tests/replication_facade_compat_test.rs`
 keeps the ECStore replication facade types covered while architecture rules

crates/ecstore/src/bucket/replication/mod.rs

@@ -47,9 +47,9 @@ pub use datatypes::ResyncStatusType;
 pub use replication_config_boundary::{
     ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
     REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
-    replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
-    validate_replication_config_target_arns,
+    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
+    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -15,7 +15,7 @@
 pub use rustfs_replication::{
     ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
     REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
-    replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
-    validate_replication_config_target_arns,
+    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
+    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs

@@ -89,3 +89,116 @@ pub fn replication_state_to_filemeta(state: &ReplicationState) -> rustfs_filemet
         target_delete_marker_version_ids_corrupt: state.target_delete_marker_version_ids_corrupt,
     }
 }
+
+// Reconciliation tests for the deliberately duplicated wire types.
+//
+// `rustfs-filemeta` (xl.meta disk format) and `rustfs-replication` (MRF/resync
+// persistence format) each own a copy of `ReplicationStatusType`,
+// `VersionPurgeStatusType` and `ReplicationState`; the conversions above hop
+// between them via `as_str()`, whose `From<&str>` impls fall back to `Empty`
+// on any unknown token. That fallback silently degrades data the moment one
+// side gains a variant the other lacks, so these tests pin the two sides
+// together:
+//
+// - the `match` statements are exhaustive with no `_` arm — adding a variant
+//   on either side fails compilation here until the mapping is reconsidered;
+// - the round-trips assert the string token survives both directions — a
+//   variant whose token the other side does not recognize fails the assert
+//   instead of quietly becoming `Empty`.
+//
+// Struct-shaped drift on `ReplicationState` is already compile-guarded by the
+// exhaustive struct literals in the two conversion functions above; the
+// round-trip test below additionally pins value fidelity for every field.
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    #[test]
+    fn replication_status_variants_round_trip_across_boundary() {
+        use rustfs_replication::ReplicationStatusType as Repl;
+
+        let all = [
+            Repl::Pending,
+            Repl::Completed,
+            Repl::CompletedLegacy,
+            Repl::Failed,
+            Repl::Replica,
+            Repl::Empty,
+        ];
+        for status in all {
+            // Exhaustive on the replication side: a new variant breaks this match.
+            match status {
+                Repl::Pending | Repl::Completed | Repl::CompletedLegacy | Repl::Failed | Repl::Replica | Repl::Empty => {}
+            }
+            let filemeta = replication_status_to_filemeta(status.clone());
+            assert_eq!(
+                filemeta.as_str(),
+                status.as_str(),
+                "replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
+            );
+            assert_eq!(replication_status_from_filemeta(filemeta), status);
+        }
+
+        // Exhaustive on the filemeta side: a new variant breaks this match.
+        fn _filemeta_side_is_covered(status: rustfs_filemeta::ReplicationStatusType) {
+            use rustfs_filemeta::ReplicationStatusType as Meta;
+            match status {
+                Meta::Pending | Meta::Completed | Meta::CompletedLegacy | Meta::Failed | Meta::Replica | Meta::Empty => {}
+            }
+        }
+    }
+
+    #[test]
+    fn version_purge_status_variants_round_trip_across_boundary() {
+        use rustfs_replication::VersionPurgeStatusType as Repl;
+
+        let all = [Repl::Pending, Repl::Complete, Repl::Failed, Repl::Empty];
+        for status in all {
+            // Exhaustive on the replication side: a new variant breaks this match.
+            match status {
+                Repl::Pending | Repl::Complete | Repl::Failed | Repl::Empty => {}
+            }
+            let filemeta = version_purge_status_to_filemeta(status.clone());
+            assert_eq!(
+                filemeta.as_str(),
+                status.as_str(),
+                "replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
+            );
+            assert_eq!(version_purge_status_from_filemeta(filemeta), status);
+        }
+
+        // Exhaustive on the filemeta side: a new variant breaks this match.
+        fn _filemeta_side_is_covered(status: rustfs_filemeta::VersionPurgeStatusType) {
+            use rustfs_filemeta::VersionPurgeStatusType as Meta;
+            match status {
+                Meta::Pending | Meta::Complete | Meta::Failed | Meta::Empty => {}
+            }
+        }
+    }
+
+    #[test]
+    fn replication_state_round_trips_every_field_across_boundary() {
+        let timestamp = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid timestamp");
+        let state = ReplicationState {
+            replica_timestamp: Some(timestamp),
+            replica_status: ReplicationStatusType::Replica,
+            delete_marker: true,
+            replication_timestamp: Some(timestamp),
+            replication_status_internal: Some("arn:a=PENDING;".to_string()),
+            version_purge_status_internal: Some("arn:a=FAILED;".to_string()),
+            replicate_decision_str: "arn:a=true;false;;".to_string(),
+            targets: HashMap::from([
+                ("arn:a".to_string(), ReplicationStatusType::Completed),
+                ("arn:b".to_string(), ReplicationStatusType::Failed),
+            ]),
+            purge_targets: HashMap::from([("arn:a".to_string(), VersionPurgeStatusType::Pending)]),
+            reset_statuses_map: HashMap::from([("reset-arn:a".to_string(), "reset-id;ts".to_string())]),
+            target_delete_marker_version_ids: HashMap::from([("arn:a".to_string(), "version-1".to_string())]),
+            target_delete_marker_version_ids_corrupt: true,
+        };
+
+        let round_tripped = replication_state_from_filemeta(&replication_state_to_filemeta(&state));
+        assert_eq!(round_tripped, state);
+    }
+}

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2550,6 +2550,12 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
             Ok((put_opts, is_mp)) => (put_opts, is_mp),
             Err(e) => {
+                // Unsupported source metadata (e.g. managed SSE) is a fail-closed
+                // condition: report FAILED so the composite status and the
+                // OperationFailedReplication event reflect that nothing reached
+                // the target, instead of leaking the optimistic Completed above.
+                rinfo.replication_status = ReplicationStatusType::Failed;
+                rinfo.error = Some(e.to_string());
                 warn!(
                     event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
                     component = LOG_COMPONENT_ECSTORE,
@@ -2954,6 +2960,11 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
             Ok((put_opts, is_mp)) => (put_opts, is_mp),
             Err(e) => {
+                // Unsupported source metadata (e.g. managed SSE) is a fail-closed
+                // condition: report FAILED so the composite status and the
+                // OperationFailedReplication event reflect that nothing reached
+                // the target, instead of leaking the optimistic Completed above.
+                rinfo.replication_status = ReplicationStatusType::Failed;
                 rinfo.error = Some(e.to_string());
                 warn!(
                     event = EVENT_RESYNC_TARGET_OPERATION_FAILED,

crates/ecstore/src/bucket/target/arn.rs

@@ -56,11 +56,59 @@ impl FromStr for ARN {
         if parts.len() != 6 {
             return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
         }
+        // Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}`; read the
+        // segments back in the same order so parse(display(a)) == a.
         Ok(ARN {
             arn_type: BucketTargetType::from_str(parts[2]).unwrap_or_default(),
-            id: parts[3].to_string(),
-            region: parts[4].to_string(),
+            region: parts[3].to_string(),
+            id: parts[4].to_string(),
             bucket: parts[5].to_string(),
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}` (madmin layout);
+    /// FromStr must read the same positions back so parse(display(a)) == a.
+    #[test]
+    fn from_str_round_trips_display_with_region_and_id() {
+        let arn = ARN::new(
+            BucketTargetType::ReplicationService,
+            "depl-123".to_string(),
+            "us-east-1".to_string(),
+            "bucket-a".to_string(),
+        );
+
+        let parsed = ARN::from_str(&arn.to_string()).expect("display output must parse");
+
+        assert_eq!(parsed.arn_type, arn.arn_type);
+        assert_eq!(parsed.region, arn.region, "region must survive display->parse round-trip");
+        assert_eq!(parsed.id, arn.id, "id must survive display->parse round-trip");
+        assert_eq!(parsed.bucket, arn.bucket);
+    }
+
+    #[test]
+    fn from_str_reads_region_then_id_in_display_order() {
+        let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("valid ARN must parse");
+
+        assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
+        assert_eq!(parsed.region, "us-east-1");
+        assert_eq!(parsed.id, "depl-123");
+        assert_eq!(parsed.bucket, "bucket-a");
+    }
+
+    /// RustFS commonly generates ARNs with an empty region:
+    /// `arn:rustfs:replication::<deployment_id>:<bucket>`.
+    #[test]
+    fn from_str_handles_empty_region_segment() {
+        let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
+
+        assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
+        assert_eq!(parsed.region, "", "region segment is empty in this form");
+        assert_eq!(parsed.id, "depl-123");
+        assert_eq!(parsed.bucket, "bucket-a");
+    }
+}

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::error::{Error, Result};
+use jiff::Timestamp;
 use rmp_serde::Serializer as rmpSerializer;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -32,7 +33,7 @@ pub struct Credentials {
     #[serde(rename = "secretKey")]
     pub secret_key: String,
     pub session_token: Option<String>,
-    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
+    pub expiration: Option<Timestamp>,
 }
 
 impl Credentials {
@@ -93,6 +94,21 @@ mod duration_milliseconds {
     }
 }
 
+/// Defensive decode for the two integer wire encodings of these duration
+/// fields: RustFS persists (and legacy RustFS clients sent) plain seconds,
+/// while Go `time.Duration` JSON — madmin/mc requests and MinIO-written
+/// bucket-targets metadata — is nanoseconds. No meaningful interval lies
+/// between 10^7 seconds (~115 days) and 10^7 nanoseconds (10ms), so the
+/// magnitude disambiguates the unit.
+pub fn duration_from_secs_or_nanos(value: u64) -> Duration {
+    const NANOS_THRESHOLD: u64 = 10_000_000;
+    if value < NANOS_THRESHOLD {
+        Duration::from_secs(value)
+    } else {
+        Duration::from_nanos(value)
+    }
+}
+
 mod duration_seconds {
     use serde::{Deserialize, Deserializer, Serializer};
     use std::time::Duration;
@@ -108,8 +124,8 @@ mod duration_seconds {
     where
         D: Deserializer<'de>,
     {
-        let secs = u64::deserialize(deserializer)?;
-        Ok(Duration::from_secs(secs))
+        let value = u64::deserialize(deserializer)?;
+        Ok(super::duration_from_secs_or_nanos(value))
     }
 }
 
@@ -408,7 +424,11 @@ mod tests {
         assert_eq!(credentials.access_key, "test-access-key");
         assert_eq!(credentials.secret_key, "test-secret-key");
         assert_eq!(credentials.session_token, Some("test-session-token".to_string()));
-        assert!(credentials.expiration.is_some());
+        assert_eq!(
+            serde_json::to_value(credentials.expiration.expect("expiration should parse"))
+                .expect("expiration should serialize to JSON"),
+            serde_json::json!("2024-12-31T23:59:59Z")
+        );
 
         // Verify latency statistics
         assert_eq!(target.latency.curr, Duration::from_millis(100));
@@ -484,6 +504,29 @@ mod tests {
         assert_eq!(original.offline_count, deserialized.offline_count);
     }
 
+    #[test]
+    fn bucket_target_reads_go_nanosecond_durations_defensively() {
+        // MinIO-written bucket-targets metadata and madmin clients encode
+        // these fields as Go `time.Duration` nanoseconds; RustFS has always
+        // persisted seconds. Both encodings must decode to the same interval.
+        let target: BucketTarget = serde_json::from_value(serde_json::json!({
+            "endpoint": "localhost:9000",
+            "targetbucket": "target",
+            "type": "replication",
+            "healthCheckDuration": 60_000_000_000u64,
+            "totalDowntime": 90_000_000_000u64
+        }))
+        .expect("nanosecond durations should deserialize");
+
+        assert_eq!(target.health_check_duration, Duration::from_secs(60));
+        assert_eq!(target.total_downtime, Duration::from_secs(90));
+
+        // The persisted wire format stays seconds for existing RustFS readers.
+        let value = serde_json::to_value(&target).expect("target should serialize");
+        assert_eq!(value["healthCheckDuration"], 60);
+        assert_eq!(value["totalDowntime"], 90);
+    }
+
     #[test]
     fn test_bucket_target_debug_redacts_credentials() {
         let target = BucketTarget {
@@ -562,12 +605,15 @@ mod tests {
                 .and_then(|credentials| credentials.session_token.as_deref()),
             Some("legacy-session-token")
         );
-        assert!(
+        assert_eq!(
             target
                 .credentials
                 .as_ref()
                 .and_then(|credentials| credentials.expiration)
-                .is_some()
+                .map(serde_json::to_value)
+                .transpose()
+                .expect("expiration should serialize to JSON"),
+            Some(serde_json::json!("2024-12-31T23:59:59Z"))
         );
     }
 
@@ -609,7 +655,11 @@ mod tests {
             credentials.session_token,
             Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string())
         );
-        assert!(credentials.expiration.is_some());
+        assert_eq!(
+            serde_json::to_value(credentials.expiration.expect("expiration should parse"))
+                .expect("expiration should serialize to JSON"),
+            serde_json::json!("2024-12-31T23:59:59Z")
+        );
     }
 
     #[test]

crates/ecstore/src/bucket/utils.rs

@@ -269,6 +269,32 @@ pub fn check_del_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }
 
+/// Filesystem `NAME_MAX`: every object-key path segment becomes one on-disk
+/// directory entry, so a longer segment can never be stored and previously
+/// escaped as an `ENAMETOOLONG` io error → `InternalError` 500 (rustfs#5785).
+const MAX_OBJECT_KEY_SEGMENT_BYTES: usize = 255;
+
+/// Reject object keys whose on-disk directory names would exceed `NAME_MAX`.
+///
+/// Middle segments map to their raw bytes; the final segment of a
+/// directory-object key (trailing `/`) is stored with the `__XLDIR__` suffix
+/// appended, shrinking its budget accordingly.
+fn object_key_segments_fit_on_disk(object: &str) -> bool {
+    let trailing_dir = object.ends_with('/');
+    let segments: Vec<&str> = object.split('/').collect();
+    let last_nonempty = segments.iter().rposition(|s| !s.is_empty());
+    for (index, segment) in segments.iter().enumerate() {
+        let mut budget = MAX_OBJECT_KEY_SEGMENT_BYTES;
+        if trailing_dir && Some(index) == last_nonempty {
+            budget = budget.saturating_sub(rustfs_utils::path::GLOBAL_DIR_SUFFIX.len());
+        }
+        if segment.len() > budget {
+            return false;
+        }
+    }
+    true
+}
+
 pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
     if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() {
         return Err(StorageError::BucketNameInvalid(bucket.to_string()));
@@ -282,6 +308,10 @@ pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
         return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
     }
 
+    if !object_key_segments_fit_on_disk(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
+    }
+
     // if cfg!(target_os = "windows") && object.contains('\\') {
     //     return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
     // }
@@ -379,6 +409,14 @@ pub fn check_put_object_args(bucket: &str, object: &str) -> Result<()> {
         return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
     }
 
+    // The write path validates arguments here rather than through
+    // check_bucket_and_object_names, so the on-disk segment budget has to be
+    // enforced in both places or an over-NAME_MAX key still reaches the disk
+    // layer and escapes as ENAMETOOLONG → InternalError 500 (rustfs#5785).
+    if !object_key_segments_fit_on_disk(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
+    }
+
     Ok(())
 }
 
@@ -387,6 +425,62 @@ mod tests {
     use super::*;
     use proptest::prelude::*;
 
+    /// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
+    /// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
+    /// ENAMETOOLONG as InternalError 500 from the disk layer.
+    #[test]
+    fn object_key_segment_name_max_budget() {
+        // 255-byte single segment: exactly at the on-disk limit.
+        assert!(check_bucket_and_object_names("bucket", &"a".repeat(255)).is_ok());
+        // 256 bytes: one over.
+        assert!(matches!(
+            check_bucket_and_object_names("bucket", &"a".repeat(256)),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+        // Long keys are fine as long as every segment fits.
+        let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
+        assert!(check_bucket_and_object_names("bucket", &segmented).is_ok());
+        // The budget counts bytes, not characters (100 CJK chars = 300 bytes).
+        assert!(matches!(
+            check_bucket_and_object_names("bucket", &"中".repeat(100)),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+        assert!(check_bucket_and_object_names("bucket", &"中".repeat(85)).is_ok());
+        // Directory-object keys spend GLOBAL_DIR_SUFFIX bytes of the final
+        // segment's budget on the on-disk __XLDIR__ encoding.
+        let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
+        assert!(check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
+        assert!(matches!(
+            check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+    }
+
+    /// rustfs#5785 follow-up: the write path validates through
+    /// check_put_object_args, not check_bucket_and_object_names, so the same
+    /// budget has to hold there — otherwise an over-NAME_MAX PUT still reached
+    /// the disk layer and came back as InternalError 500.
+    #[test]
+    fn put_object_args_enforce_the_same_segment_budget() {
+        assert!(check_put_object_args("bucket", &"a".repeat(255)).is_ok());
+        assert!(matches!(
+            check_put_object_args("bucket", &"a".repeat(256)),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+        assert!(matches!(
+            check_put_object_args("bucket", &"\u{4e2d}".repeat(100)),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+        let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
+        assert!(check_put_object_args("bucket", &segmented).is_ok());
+        let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
+        assert!(check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
+        assert!(matches!(
+            check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
+            Err(StorageError::ObjectNameInvalid(_, _))
+        ));
+    }
+
     // Test validation functions
     #[test]
     fn test_is_valid_object_name() {

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -36,15 +36,20 @@ use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
 use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
 use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
-use rustfs_io_metrics::internode_metrics::global_internode_metrics;
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
+};
+use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
 use std::collections::{HashSet, VecDeque};
 use std::sync::{LazyLock, Mutex, Once};
+use std::thread;
 use std::time::{Duration, Instant};
 use time::OffsetDateTime;
-use tracing::error;
+use tracing::{error, info, warn};
 use uuid::Uuid;
 
 type HmacSha256 = Hmac<Sha256>;
@@ -70,6 +75,11 @@ const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
 const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
 const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
 const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
+const REPLAY_CACHE_RETENTION_SECS: usize = 601;
+const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
+const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 8;
+const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 2048;
+const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 16_777_216;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
@@ -91,18 +101,211 @@ static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
     )
 });
 // Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
-// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
-// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
-static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
-    rustfs_utils::get_env_usize(
-        rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
-        rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
-    )
-    .max(1)
-});
+// active; overflow fails closed and increments the replay-cache overflow counter. Explicit operator
+// values and auto-sizing are both floored at the historical default so under-sizing cannot turn
+// legitimate high-throughput traffic into `No valid auth token` failures.
+static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(resolve_replay_cache_capacity);
 static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
 static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ReplayCacheCapacitySource {
+    Env,
+    EnvClampedToDefault,
+    Auto,
+    AutoClampedToDefault,
+    AutoInvalidEnv,
+    AutoInvalidEnvClampedToDefault,
+}
+
+impl ReplayCacheCapacitySource {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Env => "env",
+            Self::EnvClampedToDefault => "env_clamped_to_default",
+            Self::Auto => "auto",
+            Self::AutoClampedToDefault => "auto_clamped_to_default",
+            Self::AutoInvalidEnv => "auto_invalid_env",
+            Self::AutoInvalidEnvClampedToDefault => "auto_invalid_env_clamped_to_default",
+        }
+    }
+
+    fn is_env_clamped(self) -> bool {
+        matches!(self, Self::EnvClampedToDefault)
+    }
+
+    fn is_env(self) -> bool {
+        matches!(self, Self::Env)
+    }
+
+    fn is_invalid_env(self) -> bool {
+        matches!(self, Self::AutoInvalidEnv | Self::AutoInvalidEnvClampedToDefault)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ReplayCacheCapacityDecision {
+    capacity: usize,
+    source: ReplayCacheCapacitySource,
+    cpu_count: usize,
+    memory_limit_bytes: Option<u64>,
+    memory_basis: Option<MemoryBasis>,
+    memory_based_capacity: usize,
+    cpu_based_capacity: usize,
+}
+
+fn saturating_usize_from_u64(value: u64) -> usize {
+    usize::try_from(value).unwrap_or(usize::MAX)
+}
+
+fn replay_cache_capacity_from_resources(cpu_count: usize, memory_limit_bytes: Option<u64>) -> (usize, usize, usize) {
+    let cpu_count = cpu_count.max(1);
+    let cpu_based_capacity = cpu_count
+        .saturating_mul(REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU)
+        .saturating_mul(REPLAY_CACHE_RETENTION_SECS);
+    let memory_based_capacity = memory_limit_bytes
+        .map(|bytes| {
+            let budget = bytes.saturating_mul(REPLAY_CACHE_AUTO_MEMORY_PERCENT) / 100;
+            saturating_usize_from_u64(budget / REPLAY_CACHE_ENTRY_BYTES_ESTIMATE)
+        })
+        .unwrap_or(REPLAY_CACHE_AUTO_MAX_CAPACITY);
+    let capacity = memory_based_capacity
+        .min(cpu_based_capacity)
+        .clamp(rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, REPLAY_CACHE_AUTO_MAX_CAPACITY);
+    (capacity, memory_based_capacity, cpu_based_capacity)
+}
+
+fn replay_cache_capacity_decision(
+    env: rustfs_utils::EnvParseOutcome<usize>,
+    cpu_count: usize,
+    memory_limit_bytes: Option<u64>,
+    memory_basis: Option<MemoryBasis>,
+) -> ReplayCacheCapacityDecision {
+    let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
+    match env {
+        rustfs_utils::EnvParseOutcome::Parsed(configured) => {
+            let capacity = configured.max(default);
+            let source = if configured < default {
+                ReplayCacheCapacitySource::EnvClampedToDefault
+            } else {
+                ReplayCacheCapacitySource::Env
+            };
+            ReplayCacheCapacityDecision {
+                capacity,
+                source,
+                cpu_count: cpu_count.max(1),
+                memory_limit_bytes,
+                memory_basis,
+                memory_based_capacity: 0,
+                cpu_based_capacity: 0,
+            }
+        }
+        rustfs_utils::EnvParseOutcome::Absent | rustfs_utils::EnvParseOutcome::Invalid => {
+            let (capacity, memory_based_capacity, cpu_based_capacity) =
+                replay_cache_capacity_from_resources(cpu_count, memory_limit_bytes);
+            let clamped_to_default = capacity == default && memory_based_capacity.min(cpu_based_capacity) < default;
+            let invalid_env = matches!(env, rustfs_utils::EnvParseOutcome::Invalid);
+            let source = match (invalid_env, clamped_to_default) {
+                (true, true) => ReplayCacheCapacitySource::AutoInvalidEnvClampedToDefault,
+                (true, false) => ReplayCacheCapacitySource::AutoInvalidEnv,
+                (false, true) => ReplayCacheCapacitySource::AutoClampedToDefault,
+                (false, false) => ReplayCacheCapacitySource::Auto,
+            };
+            ReplayCacheCapacityDecision {
+                capacity,
+                source,
+                cpu_count: cpu_count.max(1),
+                memory_limit_bytes,
+                memory_basis,
+                memory_based_capacity,
+                cpu_based_capacity,
+            }
+        }
+    }
+}
+
+fn detected_replay_cache_resources() -> (usize, Option<u64>, Option<MemoryBasis>) {
+    let cpu_count = thread::available_parallelism().map(usize::from).unwrap_or(1).max(1);
+    let memory = resolve_effective_memory();
+    let memory_limit_bytes = (memory.total_bytes > 0).then_some(memory.total_bytes);
+    (cpu_count, memory_limit_bytes, Some(memory.basis))
+}
+
+fn log_replay_cache_capacity_decision(decision: ReplayCacheCapacityDecision) {
+    let source = decision.source.as_str();
+    if decision.source.is_env_clamped() {
+        warn!(
+            event = "internode_rpc_replay_cache_capacity_resolved",
+            component = "ecstore",
+            subsystem = "rpc_auth",
+            capacity = decision.capacity,
+            source,
+            default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+            env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+            "internode rpc replay cache capacity clamped to default"
+        );
+        return;
+    }
+    if decision.source.is_env() {
+        info!(
+            event = "internode_rpc_replay_cache_capacity_resolved",
+            component = "ecstore",
+            subsystem = "rpc_auth",
+            capacity = decision.capacity,
+            source,
+            default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+            env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+            "internode rpc replay cache capacity resolved from env"
+        );
+        return;
+    }
+    if decision.source.is_invalid_env() {
+        warn!(
+            event = "internode_rpc_replay_cache_capacity_resolved",
+            component = "ecstore",
+            subsystem = "rpc_auth",
+            capacity = decision.capacity,
+            source,
+            cpu_count = decision.cpu_count,
+            memory_limit_bytes = decision.memory_limit_bytes,
+            memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
+            memory_based_capacity = decision.memory_based_capacity,
+            cpu_based_capacity = decision.cpu_based_capacity,
+            auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
+            env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
+            "internode rpc replay cache capacity auto-sized after invalid env"
+        );
+        return;
+    }
+    info!(
+        event = "internode_rpc_replay_cache_capacity_resolved",
+        component = "ecstore",
+        subsystem = "rpc_auth",
+        capacity = decision.capacity,
+        source,
+        cpu_count = decision.cpu_count,
+        memory_limit_bytes = decision.memory_limit_bytes,
+        memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
+        memory_based_capacity = decision.memory_based_capacity,
+        cpu_based_capacity = decision.cpu_based_capacity,
+        auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
+        "internode rpc replay cache capacity resolved"
+    );
+}
+
+fn resolve_replay_cache_capacity() -> usize {
+    let (cpu_count, memory_limit_bytes, memory_basis) = detected_replay_cache_resources();
+    let decision = replay_cache_capacity_decision(
+        rustfs_utils::get_env_parse_outcome(rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY),
+        cpu_count,
+        memory_limit_bytes,
+        memory_basis,
+    );
+    global_internode_metrics().record_replay_cache_state(0, decision.capacity);
+    log_replay_cache_capacity_decision(decision);
+    decision.capacity
+}
+
 #[derive(Default)]
 struct RpcNonceCache {
     nonces: HashSet<Uuid>,
@@ -110,8 +313,50 @@ struct RpcNonceCache {
     max_wall_time: i64,
 }
 
+#[derive(Clone, Copy)]
+struct RpcReplayCacheMetricScope<'a> {
+    operation: &'static str,
+    backend: &'static str,
+    rpc_path: &'a str,
+}
+
+#[derive(Clone, Copy)]
+struct RpcNonceRecord<'a> {
+    nonce: Uuid,
+    signed_at: i64,
+    now: Instant,
+    wall_time: i64,
+    expires_at: Instant,
+    capacity: usize,
+    metric_scope: RpcReplayCacheMetricScope<'a>,
+}
+
+struct RpcNonceCacheMetrics<'a> {
+    expired: usize,
+    entries: usize,
+    capacity: usize,
+    overflow_scope: Option<RpcReplayCacheMetricScope<'a>>,
+}
+
+fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
+    let Some(metrics) = metrics else {
+        return;
+    };
+    let internode_metrics = global_internode_metrics();
+    internode_metrics.record_replay_cache_evictions("expired", metrics.expired);
+    internode_metrics.record_replay_cache_state(metrics.entries, metrics.capacity);
+    if let Some(scope) = metrics.overflow_scope {
+        internode_metrics.record_replay_cache_overflow_for_operation_and_backend_path(
+            scope.operation,
+            scope.backend,
+            scope.rpc_path,
+        );
+    }
+}
+
 impl RpcNonceCache {
-    fn remove_expired(&mut self, now: Instant, wall_time: i64) {
+    fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
+        let mut removed = 0;
         while matches!(
             self.expirations.front(),
             Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
@@ -120,37 +365,48 @@ impl RpcNonceCache {
                 break;
             };
             self.nonces.remove(&nonce);
+            removed += 1;
         }
+        removed
     }
 
-    fn check_and_record(
-        &mut self,
-        nonce: Uuid,
-        signed_at: i64,
-        now: Instant,
-        wall_time: i64,
-        expires_at: Instant,
-        capacity: usize,
-    ) -> std::io::Result<()> {
-        self.max_wall_time = self.max_wall_time.max(wall_time);
-        if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
-            return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
+    fn check_and_record<'a>(&mut self, record: RpcNonceRecord<'a>) -> (std::io::Result<()>, Option<RpcNonceCacheMetrics<'a>>) {
+        self.max_wall_time = self.max_wall_time.max(record.wall_time);
+        if self.max_wall_time.saturating_sub(record.signed_at) > SIGNATURE_VALID_DURATION {
+            return (Err(std::io::Error::other("RPC request timestamp expired after clock regression")), None);
         }
-        self.remove_expired(now, self.max_wall_time);
-        if self.nonces.contains(&nonce) {
-            return Err(std::io::Error::other("RPC request replay detected"));
+        let expired = self.remove_expired(record.now, self.max_wall_time);
+        let metrics = RpcNonceCacheMetrics {
+            expired,
+            entries: self.nonces.len(),
+            capacity: record.capacity,
+            overflow_scope: None,
+        };
+        if self.nonces.contains(&record.nonce) {
+            return (Err(std::io::Error::other("RPC request replay detected")), Some(metrics));
         }
-        if self.nonces.len() >= capacity {
+        if self.nonces.len() >= record.capacity {
             // Fail closed and alert: only legitimately signed traffic can fill the cache, so a
             // sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
             // for this node's peak mutation rate and writes are being refused.
-            global_internode_metrics().record_replay_cache_overflow();
-            return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
+            return (
+                Err(std::io::Error::other("RPC replay cache capacity exceeded")),
+                Some(RpcNonceCacheMetrics {
+                    overflow_scope: Some(record.metric_scope),
+                    ..metrics
+                }),
+            );
         }
-        self.nonces.insert(nonce);
+        self.nonces.insert(record.nonce);
         self.expirations
-            .push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
-        Ok(())
+            .push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
+        (
+            Ok(()),
+            Some(RpcNonceCacheMetrics {
+                entries: self.nonces.len(),
+                ..metrics
+            }),
+        )
     }
 }
 
@@ -541,18 +797,43 @@ fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
     Ok(())
 }
 
-fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
+fn tonic_rpc_metric_operation(path: &str) -> &'static str {
+    match parse_tonic_rpc_path(path).ok().map(|(_, rpc_method)| rpc_method) {
+        Some("ReadAll") => INTERNODE_OPERATION_GRPC_READ_ALL,
+        Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        _ => INTERNODE_OPERATION_GRPC_OTHER,
+    }
+}
+
+fn check_and_record_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::io::Result<()> {
     let wall_time = OffsetDateTime::now_utc().unix_timestamp();
-    let mut cache = LOCAL_RPC_NONCE_CACHE
-        .lock()
-        .map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
-    // Take the monotonic timestamp after acquiring the lock so expiration
-    // entries remain ordered by the same serialization point as insertion.
-    let now = Instant::now();
-    let expires_at = now
-        .checked_add(REPLAY_CACHE_RETENTION)
-        .ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
-    cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
+    let (result, metrics) = {
+        let mut cache = LOCAL_RPC_NONCE_CACHE
+            .lock()
+            .map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
+        // Take the monotonic timestamp after acquiring the lock so expiration
+        // entries remain ordered by the same serialization point as insertion.
+        let now = Instant::now();
+        let expires_at = now
+            .checked_add(REPLAY_CACHE_RETENTION)
+            .ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
+        cache.check_and_record(RpcNonceRecord {
+            nonce,
+            signed_at,
+            now,
+            wall_time,
+            expires_at,
+            capacity: *REPLAY_CACHE_CAPACITY,
+            metric_scope: RpcReplayCacheMetricScope {
+                operation: tonic_rpc_metric_operation(rpc_path),
+                backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
+                rpc_path,
+            },
+        })
+    };
+    publish_nonce_cache_metrics(metrics);
+    result
 }
 
 /// Build headers with authentication signature
@@ -814,7 +1095,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_nonce(nonce, signed_at)
+    check_and_record_nonce(nonce, signed_at, path)
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -847,6 +1128,46 @@ pub fn verify_tonic_rpc_signature_with_bootstrap(
     )
 }
 
+pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
+    match error.to_string().as_str() {
+        "Missing RPC audience" => "missing_audience",
+        "Invalid RPC request path" => "invalid_request_path",
+        "RPC replay-scoped authentication required" => "replay_scope_required",
+        "Missing RPC replay scope version" => "missing_replay_scope_version",
+        "Unsupported RPC replay scope version" => "unsupported_replay_scope_version",
+        "Missing RPC replay scope signature" => "missing_replay_scope_signature",
+        "Missing RPC replay scope nonce" => "missing_replay_scope_nonce",
+        "Invalid RPC replay scope nonce" => "invalid_replay_scope_nonce",
+        "Missing RPC boot epoch" => "missing_boot_epoch",
+        "Invalid RPC boot epoch" => "invalid_boot_epoch",
+        "Invalid RPC replay scope signature" => "invalid_replay_scope_signature",
+        "RPC boot epoch is stale" => "stale_boot_epoch",
+        "RPC request replay detected" => "replay_detected",
+        "RPC replay cache capacity exceeded" => "replay_cache_capacity",
+        "RPC replay cache unavailable" => "replay_cache_unavailable",
+        "RPC replay expiry overflow" => "replay_expiry_overflow",
+        "RPC request timestamp expired after clock regression" => "timestamp_expired_after_clock_regression",
+        "RPC v2 authentication required" => "v2_required",
+        "Missing RPC auth version" => "missing_v2_auth_version",
+        "Unsupported RPC auth version" => "unsupported_v2_auth_version",
+        "Missing RPC v2 signature" => "missing_v2_signature",
+        "Invalid RPC v2 signature" => "invalid_v2_signature",
+        "Missing timestamp header" => "missing_timestamp",
+        "Invalid timestamp format" => "invalid_timestamp",
+        "Request timestamp expired" => "timestamp_expired",
+        "Missing RPC nonce" => "missing_v2_nonce",
+        "Invalid RPC nonce" => "invalid_v2_nonce",
+        "Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce",
+        "Missing RPC content SHA-256" => "missing_content_sha256",
+        "Invalid RPC content SHA-256" => "invalid_content_sha256",
+        "Missing signature header" => "missing_v1_signature",
+        "Invalid signature" => "invalid_v1_signature",
+        "Invalid RPC HMAC key" => "invalid_hmac_key",
+        message if message.contains(RPC_SECRET_REQUIRED_OPERATOR_MESSAGE) => "missing_rpc_secret",
+        _ => "unknown",
+    }
+}
+
 fn verify_tonic_rpc_signature_with_policy(
     audience: &str,
     path: &str,
@@ -965,7 +1286,7 @@ fn verify_tonic_rpc_signature_with_strictness(
         return Err(std::io::Error::other("Invalid RPC v2 signature"));
     }
     if let Some(nonce) = parsed_nonce {
-        check_and_record_nonce(nonce, timestamp)?;
+        check_and_record_nonce(nonce, timestamp, path)?;
     }
     Ok(())
 }
@@ -1699,6 +2020,31 @@ mod tests {
         assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
     }
 
+    #[test]
+    fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
+        for (message, reason) in [
+            ("Invalid RPC v2 signature", "invalid_v2_signature"),
+            ("RPC replay-scoped authentication required", "replay_scope_required"),
+            ("Missing RPC replay scope signature", "missing_replay_scope_signature"),
+            ("RPC boot epoch is stale", "stale_boot_epoch"),
+            ("RPC request replay detected", "replay_detected"),
+            ("Request timestamp expired", "timestamp_expired"),
+            ("Missing RPC content SHA-256", "missing_content_sha256"),
+            ("Invalid RPC content SHA-256", "invalid_content_sha256"),
+        ] {
+            assert_eq!(
+                tonic_rpc_auth_failure_reason(&std::io::Error::other(message)),
+                reason,
+                "message {message:?} should map to a stable low-cardinality reason"
+            );
+        }
+    }
+
+    #[test]
+    fn tonic_rpc_auth_failure_reason_falls_back_for_unclassified_errors() {
+        assert_eq!(tonic_rpc_auth_failure_reason(&std::io::Error::other("opaque failure")), "unknown");
+    }
+
     #[test]
     fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
         ensure_test_rpc_secret();
@@ -1903,6 +2249,126 @@ mod tests {
         assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
     }
 
+    #[test]
+    fn tonic_rpc_metric_operation_classifies_get_hot_path_methods() {
+        assert_eq!(
+            tonic_rpc_metric_operation("/node_service.NodeService/ReadAll"),
+            INTERNODE_OPERATION_GRPC_READ_ALL
+        );
+        assert_eq!(
+            tonic_rpc_metric_operation("/node_service.NodeService/ReadMultiple"),
+            INTERNODE_OPERATION_GRPC_READ_MULTIPLE
+        );
+        assert_eq!(
+            tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
+            INTERNODE_OPERATION_GRPC_WRITE_ALL
+        );
+        assert_eq!(
+            tonic_rpc_metric_operation("/node_service.NodeService/SignalService"),
+            INTERNODE_OPERATION_GRPC_OTHER
+        );
+        assert_eq!(tonic_rpc_metric_operation("not-a-grpc-path"), INTERNODE_OPERATION_GRPC_OTHER);
+    }
+
+    #[test]
+    fn replay_cache_capacity_uses_env_with_default_floor() {
+        let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
+
+        let high = replay_cache_capacity_decision(
+            rustfs_utils::EnvParseOutcome::Parsed(default * 16),
+            2,
+            Some(512 * 1024 * 1024),
+            Some(MemoryBasis::Host),
+        );
+        assert_eq!(high.capacity, default * 16);
+        assert_eq!(high.source, ReplayCacheCapacitySource::Env);
+
+        let low = replay_cache_capacity_decision(
+            rustfs_utils::EnvParseOutcome::Parsed(1),
+            64,
+            Some(128 * 1024 * 1024 * 1024),
+            Some(MemoryBasis::Host),
+        );
+        assert_eq!(low.capacity, default);
+        assert_eq!(low.source, ReplayCacheCapacitySource::EnvClampedToDefault);
+    }
+
+    #[test]
+    fn replay_cache_capacity_auto_sizes_from_cpu_and_memory() {
+        let gib = 1024_u64 * 1024 * 1024;
+        let decision =
+            replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 8, Some(16 * gib), Some(MemoryBasis::Host));
+
+        assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
+        assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
+        assert_eq!(decision.memory_based_capacity, 10_737_418);
+        assert_eq!(decision.cpu_based_capacity, 9_846_784);
+        assert_eq!(decision.capacity, 9_846_784);
+    }
+
+    #[test]
+    fn replay_cache_capacity_auto_reaches_hotpath_verified_capacity_on_larger_nodes() {
+        let gib = 1024_u64 * 1024 * 1024;
+        let decision =
+            replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
+
+        assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
+        assert_eq!(decision.memory_based_capacity, 21_474_836);
+        assert_eq!(decision.cpu_based_capacity, 19_693_568);
+        assert_eq!(decision.capacity, 16_777_216);
+    }
+
+    #[test]
+    fn replay_cache_capacity_auto_keeps_default_floor_for_small_nodes() {
+        let decision = replay_cache_capacity_decision(
+            rustfs_utils::EnvParseOutcome::Absent,
+            1,
+            Some(512 * 1024 * 1024),
+            Some(MemoryBasis::Host),
+        );
+
+        assert_eq!(decision.capacity, rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
+        assert_eq!(decision.source, ReplayCacheCapacitySource::AutoClampedToDefault);
+        assert!(decision.memory_based_capacity < rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
+    }
+
+    #[test]
+    fn replay_cache_capacity_invalid_env_uses_auto_sizing() {
+        let decision = replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Invalid, 8, None, None);
+
+        assert_eq!(decision.source, ReplayCacheCapacitySource::AutoInvalidEnv);
+        assert_eq!(decision.capacity, 9_846_784);
+    }
+
+    fn check_test_nonce_record(cache: &mut RpcNonceCache, record: RpcNonceRecord<'_>) -> std::io::Result<()> {
+        let (result, metrics) = cache.check_and_record(record);
+        publish_nonce_cache_metrics(metrics);
+        result
+    }
+
+    fn test_nonce_record(
+        nonce: Uuid,
+        signed_at: i64,
+        now: Instant,
+        wall_time: i64,
+        expires_at: Instant,
+        capacity: usize,
+    ) -> RpcNonceRecord<'static> {
+        RpcNonceRecord {
+            nonce,
+            signed_at,
+            now,
+            wall_time,
+            expires_at,
+            capacity,
+            metric_scope: RpcReplayCacheMetricScope {
+                operation: INTERNODE_OPERATION_GRPC_READ_ALL,
+                backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
+                rpc_path: "/node_service.NodeService/ReadAll",
+            },
+        }
+    }
+
     #[test]
     fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
         let now = Instant::now();
@@ -1912,15 +2378,12 @@ mod tests {
         let nonce_b = Uuid::new_v4();
         let mut cache = RpcNonceCache::default();
 
-        cache
-            .check_and_record(nonce_a, 100, now, 100, expiry, 1)
+        check_test_nonce_record(&mut cache, test_nonce_record(nonce_a, 100, now, 100, expiry, 1))
             .expect("first nonce should be recorded");
-        let capacity = cache
-            .check_and_record(nonce_b, 100, now, 100, expiry, 1)
+        let capacity = check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 100, now, 100, expiry, 1))
             .expect_err("a full replay cache must fail closed");
         assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
-        cache
-            .check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
+        check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 702, after_expiry, 702, after_expiry, 1))
             .expect("expired nonce should release capacity");
         assert!(!cache.nonces.contains(&nonce_a));
         assert!(cache.nonces.contains(&nonce_b));
@@ -2123,17 +2586,15 @@ mod tests {
         let nonce = Uuid::new_v4();
         let mut cache = RpcNonceCache::default();
 
-        cache
-            .check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
+        check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, now, 1_000, expiry, 2))
             .expect("first nonce should be recorded");
-        let replay = cache
-            .check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
+        let replay = check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, after_expiry, 900, after_expiry, 2))
             .expect_err("wall clock regression must not make an old signature reusable");
         assert_eq!(replay.to_string(), "RPC request replay detected");
 
-        let stale = cache
-            .check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
-            .expect_err("the monotonic wall-clock high-water mark must fail closed");
+        let stale =
+            check_test_nonce_record(&mut cache, test_nonce_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2))
+                .expect_err("the monotonic wall-clock high-water mark must fail closed");
         assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
     }
 }

crates/ecstore/src/cluster/rpc/mod.rs

@@ -34,8 +34,8 @@ pub use client::{
 pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
     normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
-    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
-    verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+    verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
     verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
     verify_tonic_rpc_signature_with_bootstrap,
 };

crates/ecstore/src/core/pools.rs

@@ -3017,7 +3017,14 @@ impl ECStore {
                 &cleanup_preflight_allowed_missing,
                 "decommission",
             )
-            .await;
+            .await
+            .map_err(|err| match err {
+                data_movement::SourceCleanupError::SourceChanged => Error::other(format!(
+                    "decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started",
+                    bucket, entry.name
+                )),
+                data_movement::SourceCleanupError::Storage(err) => err,
+            });
             resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
         } else if decommissioned != fivs.versions.len() || expired > 0 {
             warn!(

crates/ecstore/src/core/sets.rs

@@ -749,7 +749,7 @@ impl crate::storage_api_contracts::list::ListOperations for Sets {
     type WalkCancellation = CancellationToken;
     type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         bucket: &str,
@@ -1093,7 +1093,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
 
         Ok(result)
     }
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -1749,7 +1749,19 @@ mod tests {
             upload_id_marker = page.next_upload_id_marker;
         }
 
-        assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
+        // Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
+        // upload id embeds the process-global deployment id, which a
+        // concurrently running test can swap between create and list time.
+        let normalize = |uploads: &[(String, String)]| {
+            let mut normalized = uploads
+                .iter()
+                .map(|(key, upload_id)| (key.clone(), runtime_sources::upload_uuid_suffix(upload_id)))
+                .collect::<Vec<_>>();
+            normalized.sort();
+            normalized
+        };
+        let actual = normalize(&actual);
+        assert_eq!(actual, normalize(&expected), "set-level merge must return every upload exactly once");
         let mut deduped = actual.clone();
         deduped.dedup();
         assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");

crates/ecstore/src/data_movement/mod.rs

@@ -25,7 +25,7 @@ use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::{
     multipart::{CompletePart, MultipartOperations as _},
     namespace::NamespaceLocking as _,
-    object::{ObjectIO as _, ObjectOperations as _},
+    object::{HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
 };
 use crate::store::ECStore;
 use bytes::Bytes;
@@ -228,6 +228,7 @@ fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx:
     ObjectOptions {
         versioned: object_info.version_id.is_some(),
         version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
+        http_preconditions: data_movement_unversioned_target_precondition(object_info),
         data_movement: true,
         mod_time: object_info.mod_time,
         preserve_etag: object_info.etag.clone(),
@@ -242,6 +243,7 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
         src_pool_idx,
         data_movement: true,
         version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
+        http_preconditions: data_movement_unversioned_target_precondition(object_info),
         mod_time: object_info.mod_time,
         user_defined: data_movement_user_defined(object_info),
         preserve_etag: object_info.etag.clone(),
@@ -249,6 +251,17 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
     }
 }
 
+fn is_unversioned_data_movement_object(object_info: &ObjectInfo) -> bool {
+    object_info.version_id.is_none_or(|version_id| version_id.is_nil())
+}
+
+fn data_movement_unversioned_target_precondition(object_info: &ObjectInfo) -> Option<HTTPPreconditions> {
+    is_unversioned_data_movement_object(object_info).then(|| HTTPPreconditions {
+        if_none_match: Some("*".to_string()),
+        ..Default::default()
+    })
+}
+
 fn data_movement_put_object_reader(
     bucket: &str,
     object_info: &ObjectInfo,
@@ -337,7 +350,7 @@ fn schedule_data_movement_multipart_abort_cleanup(
 }
 
 fn should_check_data_movement_overwrite_resume(err: &Error) -> bool {
-    is_err_data_movement_overwrite(err)
+    is_err_data_movement_overwrite(err) || matches!(err, Error::PreconditionFailed)
 }
 
 fn effective_actual_size(info: &ObjectInfo) -> Option<i64> {
@@ -403,6 +416,16 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
         && are_equivalent_data_movement_parts(&source.parts, &target.parts)
 }
 
+fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
+    is_unversioned_data_movement_object(source)
+        && is_unversioned_data_movement_object(target)
+        && !target.delete_marker
+        && source
+            .mod_time
+            .zip(target.mod_time)
+            .is_some_and(|(source_time, target_time)| target_time > source_time)
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 struct SourceCleanupPartIdentity {
     number: usize,
@@ -414,6 +437,15 @@ struct SourceCleanupPartIdentity {
     checksums: BTreeMap<String, String>,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+struct SourceCleanupErasureIdentity {
+    algorithm: String,
+    data_blocks: usize,
+    parity_blocks: usize,
+    block_size: usize,
+    distribution: Vec<usize>,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub(crate) struct SourceCleanupVersionIdentity {
     name: String,
@@ -424,10 +456,28 @@ pub(crate) struct SourceCleanupVersionIdentity {
     etag: Option<String>,
     checksum: Option<Vec<u8>>,
     data_dir: Option<uuid::Uuid>,
+    transition_status: String,
+    transitioned_objname: String,
+    transition_tier: String,
+    transition_version_id: Option<uuid::Uuid>,
+    transition_version: Option<String>,
+    transition_version_state: u8,
+    expire_restored: bool,
+    erasure: SourceCleanupErasureIdentity,
     metadata: BTreeMap<String, String>,
     parts: Vec<SourceCleanupPartIdentity>,
 }
 
+fn source_cleanup_erasure_identity(erasure: &rustfs_filemeta::ErasureInfo) -> SourceCleanupErasureIdentity {
+    SourceCleanupErasureIdentity {
+        algorithm: erasure.algorithm.clone(),
+        data_blocks: erasure.data_blocks,
+        parity_blocks: erasure.parity_blocks,
+        block_size: erasure.block_size,
+        distribution: erasure.distribution.clone(),
+    }
+}
+
 fn source_cleanup_part_identity(part: &ObjectPartInfo) -> SourceCleanupPartIdentity {
     SourceCleanupPartIdentity {
         number: part.number,
@@ -457,6 +507,19 @@ pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceClean
         etag: version.get_etag(),
         checksum: version.checksum.as_ref().map(|checksum| checksum.to_vec()),
         data_dir: version.data_dir,
+        transition_status: version.transition_status.clone(),
+        transitioned_objname: version.transitioned_objname.clone(),
+        transition_tier: version.transition_tier.clone(),
+        transition_version_id: version.transition_version_id,
+        transition_version: version.transition_version.clone(),
+        transition_version_state: match version.transition_version_state {
+            rustfs_filemeta::TransitionVersionState::Unknown => 0,
+            rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
+            rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
+            rustfs_filemeta::TransitionVersionState::Exact => 3,
+        },
+        expire_restored: version.expire_restored,
+        erasure: source_cleanup_erasure_identity(&version.erasure),
         metadata: version
             .metadata
             .iter()
@@ -472,10 +535,6 @@ fn source_cleanup_version_identities(fivs: &FileInfoVersions) -> Vec<SourceClean
     identities
 }
 
-fn source_cleanup_versions_match(expected: &FileInfoVersions, current: &FileInfoVersions) -> bool {
-    source_cleanup_versions_match_with_allowed_missing(expected, current, &[])
-}
-
 fn source_cleanup_versions_match_with_allowed_missing(
     expected: &FileInfoVersions,
     current: &FileInfoVersions,
@@ -507,6 +566,26 @@ fn source_cleanup_versions_match_with_allowed_missing(
         .all(|(identity, count)| allowed_counts.get(&identity).copied().unwrap_or_default() >= count)
 }
 
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum SourceCleanupError {
+    #[error("source versions changed after migration started")]
+    SourceChanged,
+    #[error(transparent)]
+    Storage(#[from] Error),
+}
+
+fn ensure_source_cleanup_versions_match(
+    expected: &FileInfoVersions,
+    current: &FileInfoVersions,
+    allowed_missing: &[SourceCleanupVersionIdentity],
+) -> std::result::Result<(), SourceCleanupError> {
+    if source_cleanup_versions_match_with_allowed_missing(expected, current, allowed_missing) {
+        Ok(())
+    } else {
+        Err(SourceCleanupError::SourceChanged)
+    }
+}
+
 fn source_cleanup_preflight_error(op_label: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
     Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}"))
 }
@@ -529,21 +608,87 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
     expected: &FileInfoVersions,
     allowed_missing: &[SourceCleanupVersionIdentity],
     op_label: &str,
-) -> Result<()> {
+) -> std::result::Result<(), SourceCleanupError> {
     let Some(current) = load_source_cleanup_versions(set, bucket, object, op_label).await? else {
         return Ok(());
     };
 
-    if source_cleanup_versions_match_with_allowed_missing(expected, &current, allowed_missing) {
-        return Ok(());
+    ensure_source_cleanup_versions_match(expected, &current, allowed_missing)
+}
+
+#[cfg(test)]
+struct SourceCleanupDeleteBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct SourceCleanupDeleteBarrier {
+    state: Arc<SourceCleanupDeleteBarrierState>,
+}
+
+#[cfg(test)]
+static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl SourceCleanupDeleteBarrier {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(SourceCleanupDeleteBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("source cleanup delete barrier mutex should not poison");
+        assert!(slot.is_none(), "source cleanup delete barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
     }
 
-    Err(source_cleanup_preflight_error(
-        op_label,
-        bucket,
-        object,
-        "source versions changed after migration started",
-    ))
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("source cleanup should reach the pre-delete barrier");
+    }
+
+    pub(crate) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for SourceCleanupDeleteBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("source cleanup delete barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("source cleanup delete barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
 }
 
 pub(crate) async fn cleanup_source_entry_if_unchanged(
@@ -553,30 +698,32 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
     expected: &FileInfoVersions,
     allowed_missing: &[SourceCleanupVersionIdentity],
     op_label: &str,
-) -> Result<ObjectInfo> {
+) -> std::result::Result<ObjectInfo, SourceCleanupError> {
     let cleanup_key = encode_dir_object(object);
     let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
-    let _guard = ns_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+    let _guard = ns_lock
+        .get_write_lock(get_lock_acquire_timeout())
+        .await
+        .map_err(Error::from)?;
 
     ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
 
-    let result = set
-        .delete_object(
-            bucket,
-            cleanup_key.as_str(),
-            ObjectOptions {
-                delete_prefix: true,
-                delete_prefix_object: true,
-                data_movement: true,
-                no_lock: true,
-                ..Default::default()
-            },
-        )
-        .await;
+    #[cfg(test)]
+    pause_source_cleanup_before_delete(bucket, object).await;
+
+    let mut opts = ObjectOptions {
+        delete_prefix: true,
+        delete_prefix_object: true,
+        data_movement: true,
+        no_lock: true,
+        ..Default::default()
+    };
+    opts.add_namespace_lock_guard(&_guard);
+    let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await;
     if result.is_ok() {
         crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
     }
-    result
+    result.map_err(SourceCleanupError::from)
 }
 
 fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
@@ -627,7 +774,11 @@ fn resolve_data_movement_overwrite_resume_result(
         return Ok(false);
     };
 
-    Ok(is_equivalent_data_movement_object(source, &target))
+    if is_equivalent_data_movement_object(source, &target) {
+        return Ok(true);
+    }
+
+    Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
 }
 
 async fn should_treat_data_movement_overwrite_as_complete(
@@ -838,7 +989,6 @@ pub(crate) async fn migrate_object(
                         bucket.as_str(),
                         object_info.name.as_str()
                     );
-                    mark_multipart_upload_completed(&abort_multipart_flag);
                     return Ok(());
                 }
 
@@ -857,6 +1007,32 @@ pub(crate) async fn migrate_object(
         }
         .await;
 
+        if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
+            let abort_result = match store.pools.get(target_pool_idx) {
+                Some(pool) => {
+                    pool.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default())
+                        .await
+                }
+                None => Err(Error::other(format!(
+                    "{op_label}: target pool {target_pool_idx} is out of range while aborting superseded multipart upload"
+                ))),
+            };
+            if let Err(abort_err) = abort_result
+                && !is_err_invalid_upload_id(&abort_err)
+            {
+                error!("{op_label}: abort superseded multipart upload err {:?}", &abort_err);
+                schedule_data_movement_multipart_abort_cleanup(
+                    store.clone(),
+                    target_pool_idx,
+                    bucket.clone(),
+                    object_info.name.clone(),
+                    res.upload_id.clone(),
+                    op_label,
+                );
+            }
+            return Ok(());
+        }
+
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
                 return match store
@@ -1056,7 +1232,7 @@ mod tests {
         let expected = cleanup_test_versions(vec![first.clone(), second.clone()]);
         let current = cleanup_test_versions(vec![second, first]);
 
-        assert!(source_cleanup_versions_match(&expected, &current));
+        assert!(source_cleanup_versions_match_with_allowed_missing(&expected, &current, &[]));
     }
 
     #[test]
@@ -1064,7 +1240,40 @@ mod tests {
         let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
         let current = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "changed")]);
 
-        assert!(!source_cleanup_versions_match(&expected, &current));
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &[])
+            .expect_err("changed source metadata must defer cleanup");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
+    }
+
+    #[test]
+    fn test_source_cleanup_preflight_rejects_changed_transition_or_erasure() {
+        let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
+        let mut current = expected.clone();
+        current.versions[0].transition_tier = "COLD".to_string();
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &[])
+            .expect_err("transition metadata changes must defer cleanup");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
+
+        let mut current = expected.clone();
+        current.versions[0].erasure.algorithm = "changed".to_string();
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &[])
+            .expect_err("erasure metadata changes must defer cleanup");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
+    }
+
+    #[test]
+    fn test_source_cleanup_preflight_ignores_per_disk_erasure_fields() {
+        let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
+        expected.versions[0].erasure.checksums = vec![rustfs_filemeta::ChecksumInfo {
+            part_number: 1,
+            hash: Bytes::from_static(b"disk-a-checksum"),
+            ..Default::default()
+        }];
+        let mut current = expected.clone();
+        current.versions[0].erasure.index = 7;
+        current.versions[0].erasure.checksums[0].hash = Bytes::from_static(b"disk-b-checksum");
+
+        assert!(source_cleanup_versions_match_with_allowed_missing(&expected, &current, &[]));
     }
 
     #[test]
@@ -1075,7 +1284,9 @@ mod tests {
             cleanup_test_file_info("object.txt", Uuid::from_u128(2), "new-version"),
         ]);
 
-        assert!(!source_cleanup_versions_match(&expected, &current));
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &[])
+            .expect_err("an added source version must defer cleanup");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
     }
 
     #[test]
@@ -1096,7 +1307,9 @@ mod tests {
         let expected = cleanup_test_versions(vec![migrated.clone(), protected]);
         let current = cleanup_test_versions(vec![migrated]);
 
-        assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &current, &[]));
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &[])
+            .expect_err("an unexpected missing version must defer cleanup");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
     }
 
     #[test]
@@ -1108,7 +1321,9 @@ mod tests {
         let current = cleanup_test_versions(vec![migrated, new_version]);
         let allowed_missing = vec![source_cleanup_version_identity(&expired)];
 
-        assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, &current, &allowed_missing));
+        let err = ensure_source_cleanup_versions_match(&expected, &current, &allowed_missing)
+            .expect_err("a new source version must defer cleanup even when an expired version may be missing");
+        assert!(matches!(err, SourceCleanupError::SourceChanged));
     }
 
     #[test]
@@ -1171,12 +1386,13 @@ mod tests {
     }
 
     #[test]
-    fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() {
+    fn test_should_check_data_movement_overwrite_resume_accepts_conflict_errors() {
         assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr(
             "bucket-a".to_string(),
             "object-a".to_string(),
             "version-a".to_string(),
         )));
+        assert!(should_check_data_movement_overwrite_resume(&Error::PreconditionFailed));
         assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown));
     }
 
@@ -1551,7 +1767,7 @@ mod tests {
     #[test]
     fn test_data_movement_complete_multipart_opts_preserves_mod_time_version_and_etag() {
         let mod_time = OffsetDateTime::now_utc();
-        let version_id = Uuid::nil();
+        let version_id = Uuid::from_u128(7);
         let object_info = ObjectInfo {
             version_id: Some(version_id),
             mod_time: Some(mod_time),
@@ -1567,11 +1783,12 @@ mod tests {
         assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str()));
         assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value"));
         assert_eq!(opts.src_pool_idx, 7);
+        assert!(opts.http_preconditions.is_none());
     }
 
     #[test]
     fn test_data_movement_put_object_opts_preserves_version_and_etag() {
-        let version_id = Uuid::nil();
+        let version_id = Uuid::from_u128(9);
         let object_info = ObjectInfo {
             version_id: Some(version_id),
             mod_time: Some(OffsetDateTime::UNIX_EPOCH),
@@ -1589,6 +1806,35 @@ mod tests {
         assert_eq!(opts.src_pool_idx, 9);
         assert!(opts.data_movement);
         assert_eq!(opts.mod_time, object_info.mod_time);
+        assert!(opts.http_preconditions.is_none());
+    }
+
+    #[test]
+    fn test_data_movement_unversioned_put_and_complete_require_absent_target() {
+        for version_id in [None, Some(Uuid::nil())] {
+            let object_info = ObjectInfo {
+                version_id,
+                ..Default::default()
+            };
+
+            let put_opts = data_movement_put_object_opts(&object_info, 9);
+            let complete_opts = data_movement_complete_multipart_opts(&object_info, 9);
+
+            assert_eq!(
+                put_opts
+                    .http_preconditions
+                    .as_ref()
+                    .and_then(HTTPPreconditions::if_none_match_value),
+                Some("*")
+            );
+            assert_eq!(
+                complete_opts
+                    .http_preconditions
+                    .as_ref()
+                    .and_then(HTTPPreconditions::if_none_match_value),
+                Some("*")
+            );
+        }
     }
 
     #[test]
@@ -1837,6 +2083,154 @@ mod tests {
         assert!(should_resume);
     }
 
+    #[test]
+    fn test_precondition_conflict_accepts_newer_unversioned_target() {
+        for version_id in [None, Some(Uuid::nil())] {
+            let source = ObjectInfo {
+                version_id,
+                size: 128,
+                etag: Some("etag-source".to_string()),
+                mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            let target = ObjectInfo {
+                etag: Some("etag-client-write".to_string()),
+                mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+                ..source.clone()
+            };
+
+            let should_resume =
+                resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                    .expect("precondition conflict target should be evaluated");
+
+            assert!(should_resume);
+        }
+    }
+
+    #[test]
+    fn test_precondition_conflict_accepts_equivalent_target() {
+        let source = ObjectInfo {
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let should_resume =
+            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(source.clone())), &source, 0, 1)
+                .expect("equivalent precondition target should be evaluated");
+
+        assert!(should_resume);
+    }
+
+    #[test]
+    fn test_precondition_conflict_rejects_non_newer_unversioned_target() {
+        let source = ObjectInfo {
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            etag: Some("etag-conflict".to_string()),
+            ..source.clone()
+        };
+
+        let should_resume =
+            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                .expect("precondition conflict target should be evaluated");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn test_precondition_conflict_rejects_newer_delete_marker() {
+        let source = ObjectInfo {
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            delete_marker: true,
+            etag: None,
+            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+            ..source.clone()
+        };
+
+        let should_resume =
+            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                .expect("delete marker conflict should be evaluated");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn test_overwrite_error_rejects_newer_unversioned_target() {
+        let source = ObjectInfo {
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            etag: Some("etag-client-write".to_string()),
+            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+            ..source.clone()
+        };
+        let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
+
+        let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1)
+            .expect("pool-selection overwrite must require target equivalence");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn test_precondition_conflict_rejects_newer_versioned_target() {
+        let source = ObjectInfo {
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(Uuid::from_u128(2)),
+            etag: Some("etag-conflict".to_string()),
+            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+            ..source.clone()
+        };
+
+        let should_resume =
+            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                .expect("versioned conflict target should be evaluated");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn test_precondition_conflict_rejects_versioned_source_with_unversioned_target() {
+        let source = ObjectInfo {
+            version_id: Some(Uuid::from_u128(1)),
+            size: 128,
+            etag: Some("etag-source".to_string()),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: None,
+            etag: Some("etag-conflict".to_string()),
+            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+            ..source.clone()
+        };
+
+        let should_resume =
+            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                .expect("versioned source conflict should be evaluated");
+
+        assert!(!should_resume);
+    }
+
     #[test]
     fn test_rebalance_overwrite_resume_accepts_equivalent_target_version() {
         let source = ObjectInfo {

crates/ecstore/src/data_usage/mod.rs

@@ -20,7 +20,7 @@ pub mod local_snapshot;
 use crate::storage_api_contracts::{
     bucket::{BucketOperations as _, BucketOptions},
     list::{ListOperations as _, StorageListObjectVersionsInfo},
-    object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _},
+    object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
 };
 use crate::{
     bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys},
@@ -33,8 +33,9 @@ use crate::{
 };
 pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
 use rustfs_data_usage::{
-    BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DataUsageCache, DataUsageEntry,
-    DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, VersionsHistogram,
+    BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
+    DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
+    VersionsHistogram, observed_data_usage_is_newer,
 };
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -85,12 +86,57 @@ static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
 static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
 static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
 
+/// Best-available persisted usage for `bucket` when no authoritative source
+/// exists yet (issue #5716): after an upgrade from a pre-v2 release the only
+/// persisted usage data is the legacy `.usage.json`, which is demoted to
+/// non-authoritative, and the authoritative caches stay empty until the
+/// scanner's first complete cycle lands. Quota admission degrades to the
+/// pre-discard per-bucket sizes retained on the cached snapshot instead of
+/// failing every write closed.
+///
+/// The baseline is static between snapshot loads — live writes do not advance
+/// it — so hard-quota enforcement during the degraded window is advisory: the
+/// overrun is bounded only by the writes issued before the next complete
+/// scanner cycle replaces the baseline with authoritative usage. That is
+/// strictly tighter than beta.11 (usage treated as 0) and strictly more
+/// available than a blanket 503. The fallback applies to any window without
+/// authoritative usage, not only pre-v2 upgrades; the values always come from
+/// the last persisted scanner output. Loads go through the TTL-bounded
+/// snapshot cache, so the quota path adds at most one backend read per
+/// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
+/// from every persisted snapshot — those still fail closed.
+pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
+    let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+    {
+        let cache = data_usage_snapshot_cache().read().await;
+        if let Some(cached) = cache
+            .as_ref()
+            .filter(|cached| tokio::time::Instant::now().duration_since(cached.loaded_at) < ttl)
+        {
+            return cached.degraded_baseline.get(bucket).copied();
+        }
+    }
+
+    // Stale or empty cache: refresh through the TTL-bounded loader. A failed
+    // refresh carries the previous baseline forward, so quota admission keeps
+    // its last grounded values through a backend read outage.
+    let _ = load_data_usage_from_backend_cached(store).await;
+    let cache = data_usage_snapshot_cache().read().await;
+    cache
+        .as_ref()
+        .and_then(|cached| cached.degraded_baseline.get(bucket).copied())
+}
+
 /// Cached copy of the last persisted data usage snapshot, served to admin
 /// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
 #[derive(Debug, Clone)]
 struct CachedDataUsageSnapshot {
     info: Option<DataUsageInfo>,
     loaded_at: tokio::time::Instant,
+    /// Pre-discard per-bucket sizes from the same load, retained even when the
+    /// snapshot is incomplete and its bucket data is discarded. Consumed only
+    /// by [`lookup_degraded_bucket_usage_baseline`] for quota admission.
+    degraded_baseline: HashMap<String, u64>,
 }
 
 impl CachedDataUsageSnapshot {
@@ -114,24 +160,34 @@ fn fresh_cached_data_usage_snapshot(
 
 fn cache_data_usage_snapshot_result(
     cache: &mut Option<CachedDataUsageSnapshot>,
-    result: Result<DataUsageInfo, Error>,
+    result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
     loaded_at: tokio::time::Instant,
     refresh_generation: u64,
+    current_generation: u64,
 ) -> Option<Result<DataUsageInfo, Error>> {
-    if data_usage_snapshot_generation() != refresh_generation {
+    if current_generation != refresh_generation {
         return None;
     }
 
     Some(match result {
-        Ok(info) => {
+        Ok((info, degraded_baseline)) => {
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info.clone()),
                 loaded_at,
+                degraded_baseline,
             });
             Ok(info)
         }
         Err(e) => {
-            *cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
+            // Keep the previous baseline through a failed refresh: quota
+            // admission must not lose its last grounded values because one
+            // backend read errored.
+            let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
+            *cache = Some(CachedDataUsageSnapshot {
+                info: None,
+                loaded_at,
+                degraded_baseline,
+            });
             Err(e)
         }
     })
@@ -142,6 +198,9 @@ type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
 static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
 static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
 static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
+static ADMIN_DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
+static ADMIN_DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
+static ADMIN_DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
 
 // Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
 // full-bucket version listing is negligible and lets tests prove that admin
@@ -200,11 +259,24 @@ fn data_usage_snapshot_generation() -> u64 {
     DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
 }
 
+fn admin_data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
+    ADMIN_DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
+}
+
+fn admin_data_usage_snapshot_generation() -> u64 {
+    ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
+}
+
 fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
     DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
     *cache = None;
 }
 
+fn clear_admin_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
+    ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
+    *cache = None;
+}
+
 fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
     LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
         moka::future::Cache::builder()
@@ -229,6 +301,11 @@ lazy_static::lazy_static! {
         SLASH_SEPARATOR,
         DATA_USAGE_OBJECT_NAME
     );
+    pub static ref DATA_USAGE_OBSERVED_OBJ_NAME_PATH: String = format!("{}{}{}",
+        crate::disk::BUCKET_META_PREFIX,
+        SLASH_SEPARATOR,
+        DATA_USAGE_OBSERVED_OBJECT_NAME
+    );
     static ref DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
     static ref LEGACY_DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
         crate::disk::BUCKET_META_PREFIX,
@@ -303,6 +380,11 @@ fn stale_data_usage_persist_reason_for_source(
 /// Store data usage info to backend storage
 #[instrument(skip(store))]
 pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
+    if data_usage_info.usage_snapshot_converged == Some(false) {
+        return Err(Error::other(
+            "nonconverged data usage observations cannot replace the quota-authoritative snapshot",
+        ));
+    }
     // Prevent older data from overwriting newer persisted stats
     if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
         && source.is_authoritative()
@@ -323,10 +405,12 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
         serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
 
     // Save to backend using the same mechanism as original code
-    crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
+    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
         .await
         .map_err(Error::other)?;
 
+    cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
+
     // Invalidate the cached snapshot so readers observe the new save on their
     // next request instead of waiting out the remaining TTL. The next cached
     // read reloads through `load_data_usage_from_backend`, keeping its
@@ -336,6 +420,64 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
     Ok(())
 }
 
+#[async_trait::async_trait]
+trait ObservedDataUsageSnapshotCleanup {
+    async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error>;
+}
+
+#[async_trait::async_trait]
+impl ObservedDataUsageSnapshotCleanup for ECStore {
+    async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
+        self.delete_object(
+            RUSTFS_META_BUCKET,
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+            ObjectOptions {
+                delete_prefix: true,
+                delete_prefix_object: true,
+                http_preconditions: Some(HTTPPreconditions {
+                    if_match: Some(revision.to_string()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        )
+        .await
+        .map(|_| ())
+    }
+}
+
+async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
+where
+    S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
+{
+    let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
+        Ok(Some(snapshot)) => snapshot,
+        Ok(None) => return,
+        Err(err) => {
+            record_usage_snapshot_failure(
+                "read_observed_before_authoritative_cleanup",
+                DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                &err,
+            );
+            return;
+        }
+    };
+    if observed_data_usage_is_newer(&observed, authoritative) {
+        return;
+    }
+
+    match store.delete_observed_data_usage_snapshot(&revision).await {
+        Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
+        Err(err) => {
+            record_usage_snapshot_failure(
+                "delete_observed_after_authoritative_save",
+                DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                &err,
+            );
+        }
+    }
+}
+
 fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
     data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX);
 }
@@ -389,6 +531,11 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
     let mut snapshot_cache = data_usage_snapshot_cache().write().await;
     ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
     clear_data_usage_snapshot_cache(&mut snapshot_cache);
+    drop(snapshot_cache);
+
+    let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
+    ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache cleanup")?;
+    clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
     Ok(())
 }
 
@@ -404,6 +551,10 @@ where
     let mut snapshot_cache = data_usage_snapshot_cache().write().await;
     ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
     clear_data_usage_snapshot_cache(&mut snapshot_cache);
+    drop(snapshot_cache);
+    let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
+    ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
+    clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
     result
 }
 
@@ -497,6 +648,23 @@ where
     )
     .await?;
 
+    ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
+    if let Err(err) = remove_bucket_usage_from_object_with_retries(
+        store,
+        DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+        bucket,
+        DATA_USAGE_REMOVE_CAS_RETRIES,
+        None,
+        guard,
+    )
+    .await
+    {
+        // The authoritative timestamp was already advanced above, so admin
+        // selection rejects this observation even if optional cleanup fails.
+        // Never make an admin-only freshness artifact block DeleteBucket.
+        record_usage_snapshot_failure("remove_bucket_from_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
+    }
+
     for object in [
         LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
         LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
@@ -761,10 +929,72 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
 /// Load data usage info from backend storage
 #[instrument(skip(store))]
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
+}
+
+/// Like [`load_data_usage_from_backend`], but also returns the pre-discard
+/// per-bucket sizes so the cached loader can retain them as the degraded
+/// quota-admission baseline (issue #5716).
+async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
     let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
     Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
 }
 
+async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
+    let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
+        Ok(data) => data,
+        Err(Error::ConfigNotFound) => return None,
+        Err(err) => {
+            record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
+            return None;
+        }
+    };
+
+    match parse_usage_snapshot(&data) {
+        Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
+        Ok(_) => {
+            error!(
+                event = "data_usage_snapshot_load_failed",
+                component = "ecstore",
+                subsystem = "data_usage",
+                state = "invalid_observed_snapshot",
+                object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                "observed data usage snapshot was not a structurally complete nonconverged view"
+            );
+            None
+        }
+        Err(err) => {
+            record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
+            None
+        }
+    }
+}
+
+fn select_admin_data_usage_snapshot(
+    mut authoritative: DataUsageInfo,
+    authoritative_format: bool,
+    observed: Option<DataUsageInfo>,
+) -> (DataUsageInfo, bool) {
+    if authoritative_format
+        && authoritative.is_complete_bucket_usage_snapshot()
+        && authoritative.usage_snapshot_converged.is_none()
+    {
+        authoritative.usage_snapshot_converged = Some(true);
+    }
+    match observed {
+        Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
+        _ => (authoritative, authoritative_format),
+    }
+}
+
+async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
+    let observed = load_observed_data_usage_snapshot(store).await;
+    let (selected, selected_is_current_format) =
+        select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
+    Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
+}
+
 fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
     if !data_usage_info.is_complete_bucket_usage_snapshot() {
         data_usage_info.usage_snapshot_complete = false;
@@ -807,7 +1037,13 @@ fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo)
     }
 }
 
-async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
+/// Returns the normalized snapshot plus the pre-discard per-bucket sizes: the
+/// degraded quota-admission baseline captured before an incomplete snapshot
+/// drops its bucket data (issue #5716).
+async fn normalize_loaded_data_usage(
+    mut data_usage_info: DataUsageInfo,
+    authoritative_format: bool,
+) -> (DataUsageInfo, HashMap<String, u64>) {
     info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
 
     if !authoritative_format {
@@ -815,6 +1051,7 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authori
     }
     populate_backward_compatible_usage_maps(&mut data_usage_info);
     validate_complete_usage_snapshot(&mut data_usage_info);
+    let degraded_baseline = data_usage_info.bucket_sizes.clone();
     discard_incomplete_bucket_usage(&mut data_usage_info);
 
     // Handle replication info
@@ -840,7 +1077,7 @@ async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authori
         }
     }
 
-    data_usage_info
+    (data_usage_info, degraded_baseline)
 }
 
 /// Load the persisted data usage snapshot through a small in-process cache.
@@ -873,10 +1110,58 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
         }
 
         let refresh_generation = data_usage_snapshot_generation();
-        let result = load_data_usage_from_backend(store.clone()).await;
+        let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
         let loaded_at = tokio::time::Instant::now();
         let mut cache = data_usage_snapshot_cache().write().await;
-        if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
+        if let Some(result) =
+            cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
+        {
+            return result;
+        }
+        drop(cache);
+        drop(refresh_guard);
+    }
+}
+
+/// Load the freshest structurally complete snapshot for authenticated admin
+/// observability. A scan raced by namespace activity may be selected here, but
+/// never by [`load_data_usage_from_backend_cached`], which remains the
+/// converged source for quota admission.
+pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+
+    loop {
+        {
+            let cache = admin_data_usage_snapshot_cache().read().await;
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+                return result;
+            }
+        }
+
+        let refresh_guard = ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
+            .get_or_init(|| Arc::new(TokioMutex::new(())))
+            .lock()
+            .await;
+        {
+            let cache = admin_data_usage_snapshot_cache().read().await;
+            if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
+                return result;
+            }
+        }
+
+        let refresh_generation = admin_data_usage_snapshot_generation();
+        let result = load_admin_data_usage_from_backend(store.clone())
+            .await
+            .map(|info| (info, HashMap::new()));
+        let loaded_at = tokio::time::Instant::now();
+        let mut cache = admin_data_usage_snapshot_cache().write().await;
+        if let Some(result) = cache_data_usage_snapshot_result(
+            &mut cache,
+            result,
+            loaded_at,
+            refresh_generation,
+            admin_data_usage_snapshot_generation(),
+        ) {
             return result;
         }
         drop(cache);
@@ -889,6 +1174,16 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
 pub async fn invalidate_data_usage_snapshot_cache() {
     let mut cache = data_usage_snapshot_cache().write().await;
     clear_data_usage_snapshot_cache(&mut cache);
+
+    let mut admin_cache = admin_data_usage_snapshot_cache().write().await;
+    clear_admin_data_usage_snapshot_cache(&mut admin_cache);
+}
+
+/// Invalidate only the admin/console view after an observational save. Quota
+/// admission continues to use the independently cached converged snapshot.
+pub async fn invalidate_admin_data_usage_snapshot_cache() {
+    let mut cache = admin_data_usage_snapshot_cache().write().await;
+    clear_admin_data_usage_snapshot_cache(&mut cache);
 }
 
 /// Aggregate usage information from local disk snapshots.
@@ -2012,6 +2307,7 @@ mod tests {
     struct UsageCasState {
         object: Option<(Vec<u8>, u64)>,
         backup_object: Option<(Vec<u8>, u64)>,
+        observed_object: Option<(Vec<u8>, u64)>,
         legacy_object: Option<(Vec<u8>, u64)>,
         legacy_backup_object: Option<(Vec<u8>, u64)>,
         interleaving_snapshot: Option<Vec<u8>>,
@@ -2031,10 +2327,24 @@ mod tests {
         state: Mutex<UsageCasState>,
     }
 
+    #[async_trait::async_trait]
+    impl ObservedDataUsageSnapshotCleanup for UsageCasStore {
+        async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
+            let mut state = self.state.lock().await;
+            let current = state.observed_object.as_ref().ok_or(Error::FileNotFound)?.1;
+            if revision != format!("usage-{current}") {
+                return Err(Error::PreconditionFailed);
+            }
+            state.observed_object = None;
+            Ok(())
+        }
+    }
+
     #[derive(Clone, Copy, Debug, PartialEq, Eq)]
     enum UsageObjectSlot {
         Primary,
         Backup,
+        Observed,
         LegacyPrimary,
         LegacyBackup,
     }
@@ -2063,6 +2373,7 @@ mod tests {
             let slot = match object {
                 object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
                 object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
+                object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
                 object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
                 object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
                 _ => return Err(Error::FileNotFound),
@@ -2071,6 +2382,7 @@ mod tests {
             let stored = match slot {
                 UsageObjectSlot::Primary => &state.object,
                 UsageObjectSlot::Backup => &state.backup_object,
+                UsageObjectSlot::Observed => &state.observed_object,
                 UsageObjectSlot::LegacyPrimary => &state.legacy_object,
                 UsageObjectSlot::LegacyBackup => &state.legacy_backup_object,
             };
@@ -2110,6 +2422,7 @@ mod tests {
             let slot = match object {
                 object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
                 object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
+                object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
                 object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
                 object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
                 _ => return Err(Error::FileNotFound),
@@ -2145,6 +2458,9 @@ mod tests {
                 let revision = state.backup_object.as_ref().map_or(1, |(_, revision)| revision + 1);
                 state.backup_object = Some((interleaving, revision));
             }
+            if slot == UsageObjectSlot::Observed {
+                return Err(Error::other("observed test fixture writes are injected directly"));
+            }
             if slot == UsageObjectSlot::LegacyPrimary
                 && let Some(interleaving) = state.interleaving_legacy_snapshot.take()
             {
@@ -2160,6 +2476,7 @@ mod tests {
             let current_revision = match slot {
                 UsageObjectSlot::Primary => state.object.as_ref(),
                 UsageObjectSlot::Backup => state.backup_object.as_ref(),
+                UsageObjectSlot::Observed => state.observed_object.as_ref(),
                 UsageObjectSlot::LegacyPrimary => state.legacy_object.as_ref(),
                 UsageObjectSlot::LegacyBackup => state.legacy_backup_object.as_ref(),
             }
@@ -2190,6 +2507,7 @@ mod tests {
             match slot {
                 UsageObjectSlot::Primary => state.object = Some((buf, revision)),
                 UsageObjectSlot::Backup => state.backup_object = Some((buf, revision)),
+                UsageObjectSlot::Observed => state.observed_object = Some((buf, revision)),
                 UsageObjectSlot::LegacyPrimary => state.legacy_object = Some((buf, revision)),
                 UsageObjectSlot::LegacyBackup => state.legacy_backup_object = Some((buf, revision)),
             }
@@ -2361,7 +2679,7 @@ mod tests {
         legacy.bucket_sizes.insert("large".to_string(), 0);
         legacy.buckets_count = 2;
 
-        let normalized = normalize_loaded_data_usage(legacy, false).await;
+        let (normalized, degraded_baseline) = normalize_loaded_data_usage(legacy, false).await;
 
         assert_eq!(normalized.buckets_count, 0);
         assert!(normalized.buckets_usage.is_empty());
@@ -2369,6 +2687,10 @@ mod tests {
         assert_eq!(normalized.objects_total_count, 0);
         assert_eq!(normalized.objects_total_size, 0);
         assert!(!normalized.usage_snapshot_complete);
+        // Issue #5716: the discarded sizes must survive as the degraded
+        // quota-admission baseline.
+        assert_eq!(degraded_baseline.get("control").copied(), Some(10_285));
+        assert_eq!(degraded_baseline.get("large").copied(), Some(0));
     }
 
     #[test]
@@ -2404,7 +2726,7 @@ mod tests {
         info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
         info.buckets_count = 2;
 
-        let normalized = normalize_loaded_data_usage(info, true).await;
+        let (normalized, _) = normalize_loaded_data_usage(info, true).await;
 
         assert_eq!(normalized.buckets_count, 2);
         assert!(normalized.usage_snapshot_complete);
@@ -2439,7 +2761,7 @@ mod tests {
         info.bucket_sizes.insert("partial".to_string(), 196_870_144);
         info.buckets_count = 1;
 
-        let normalized = normalize_loaded_data_usage(info, true).await;
+        let (normalized, _) = normalize_loaded_data_usage(info, true).await;
 
         assert_eq!(normalized.buckets_count, 0);
         assert!(!normalized.buckets_usage.contains_key("control"));
@@ -2454,7 +2776,7 @@ mod tests {
         info.buckets_count = 2;
 
         assert!(!data_usage_contains_bucket(&info, "missing"));
-        let normalized = normalize_loaded_data_usage(info, true).await;
+        let (normalized, _) = normalize_loaded_data_usage(info, true).await;
 
         assert!(!normalized.usage_snapshot_complete);
         assert!(normalized.buckets_usage.is_empty());
@@ -2463,7 +2785,7 @@ mod tests {
 
     #[tokio::test]
     async fn complete_empty_snapshot_remains_authoritative() {
-        let normalized = normalize_loaded_data_usage(
+        let (normalized, _) = normalize_loaded_data_usage(
             DataUsageInfo {
                 last_update: Some(SystemTime::UNIX_EPOCH),
                 usage_snapshot_complete: true,
@@ -2478,6 +2800,71 @@ mod tests {
         assert!(normalized.buckets_usage.is_empty());
     }
 
+    #[test]
+    fn admin_snapshot_selection_requires_the_current_authoritative_baseline() {
+        let authoritative = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        };
+        let observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(11),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
+            ..Default::default()
+        };
+
+        let (selected, _) = select_admin_data_usage_snapshot(authoritative.clone(), true, Some(observed.clone()));
+        assert_eq!(selected.usage_snapshot_converged, Some(false));
+
+        let mut namespace_changed = authoritative;
+        namespace_changed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let (selected, _) = select_admin_data_usage_snapshot(namespace_changed, true, Some(observed));
+        assert_eq!(selected.usage_snapshot_converged, Some(true));
+    }
+
+    #[tokio::test]
+    async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
+        let store = UsageCasStore::default();
+        let authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let stale_observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            ..Default::default()
+        };
+        store.state.lock().await.observed_object =
+            Some((serde_json::to_vec(&stale_observed).expect("observed snapshot should encode"), 1));
+
+        cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
+        assert!(store.state.lock().await.observed_object.is_none());
+
+        cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
+        assert!(store.state.lock().await.observed_object.is_none());
+
+        let newer_observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(3)),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(11),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
+            ..Default::default()
+        };
+        store.state.lock().await.observed_object =
+            Some((serde_json::to_vec(&newer_observed).expect("observed snapshot should encode"), 2));
+
+        cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
+        assert!(store.state.lock().await.observed_object.is_some());
+    }
+
     #[test]
     #[serial]
     fn cached_snapshot_failure_is_reused_until_ttl_expires() {
@@ -2485,8 +2872,14 @@ mod tests {
         let mut cache = None;
         let refresh_generation = data_usage_snapshot_generation();
 
-        let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
-            .expect("an uninterrupted refresh should populate the cache");
+        let first = cache_data_usage_snapshot_result(
+            &mut cache,
+            Err(Error::ErasureReadQuorum),
+            loaded_at,
+            refresh_generation,
+            data_usage_snapshot_generation(),
+        )
+        .expect("an uninterrupted refresh should populate the cache");
         assert!(matches!(first, Err(Error::ErasureReadQuorum)));
 
         let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2504,9 +2897,15 @@ mod tests {
         let mut cache = None;
         let refresh_generation = data_usage_snapshot_generation();
 
-        let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
-            .expect("an uninterrupted refresh should populate the cache")
-            .expect("successful load must be returned");
+        let first = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok((expected, HashMap::new())),
+            loaded_at,
+            refresh_generation,
+            data_usage_snapshot_generation(),
+        )
+        .expect("an uninterrupted refresh should populate the cache")
+        .expect("successful load must be returned");
         assert_snapshot_bucket(&first, "bucket");
 
         let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
@@ -2523,14 +2922,16 @@ mod tests {
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
             loaded_at,
+            degraded_baseline: HashMap::new(),
         });
         clear_data_usage_snapshot_cache(&mut cache);
 
         let stale_result = cache_data_usage_snapshot_result(
             &mut cache,
-            Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
+            Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
             loaded_at,
             refresh_generation,
+            data_usage_snapshot_generation(),
         );
 
         assert!(stale_result.is_none());
@@ -3533,6 +3934,7 @@ mod tests {
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
             loaded_at: tokio::time::Instant::now(),
+            degraded_baseline: HashMap::new(),
         });
         memory_cache()
             .write()
@@ -3595,6 +3997,7 @@ mod tests {
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
             loaded_at: tokio::time::Instant::now(),
+            degraded_baseline: HashMap::new(),
         });
 
         let store_for_cleanup = store.clone();
@@ -3646,6 +4049,7 @@ mod tests {
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(stale),
             loaded_at: tokio::time::Instant::now(),
+            degraded_baseline: HashMap::new(),
         });
 
         remove_bucket_usage_from_backend_with_guard(&store, BUCKET, None)

crates/ecstore/src/disk/local.rs

@@ -119,7 +119,7 @@ fn read_all_data_std(path: &Path) -> core::result::Result<(Vec<u8>, Option<Offse
     Ok((bytes, modtime))
 }
 
-fn inline_metadata_rollback_dir(version_id: Uuid, meta: &FileMeta) -> Uuid {
+pub(crate) fn inline_metadata_rollback_dir(version_id: Uuid, meta: &FileMeta) -> Uuid {
     let used_data_dirs: HashSet<Uuid> = meta.get_data_dirs().unwrap_or_default().into_iter().flatten().collect();
     let base = version_id.as_u128() ^ INLINE_METADATA_ROLLBACK_DIR_XOR;
     let mut salt = 0u128;
@@ -240,8 +240,15 @@ async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, d
 }
 
 async fn restore_metadata_backup(object_dir: &Path, xl_path: &Path, rollback_dir: Uuid) -> Result<()> {
-    let backup_path = object_dir.join(rollback_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
-    rename_all(&backup_path, xl_path, object_dir).await
+    let rollback_path = object_dir.join(rollback_dir.to_string());
+    let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP);
+    rename_all(&backup_path, xl_path, object_dir).await?;
+    // A synthetic inline rollback dir held only the backup the rename above
+    // just consumed; reclaim it so the object dir can empty out. A real data
+    // dir still holds its parts, so the non-recursive remove is a benign
+    // no-op there (mirrors restore_delete_rollback).
+    let _ = fs::remove_dir(&rollback_path).await;
+    Ok(())
 }
 
 async fn restore_delete_rollback(object_dir: &Path, xl_path: &Path, rollback_dir: Uuid) -> Result<()> {
@@ -684,6 +691,10 @@ const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
 const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
 const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
 const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_failed";
+/// A healing commit could not trash the stale destination data dir it is about
+/// to replace. Best effort — the rename that follows fails closed — but a
+/// recurring signal means heal is stuck on that drive.
+const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed";
 const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total";
 const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total";
 // io_uring read-backend gray-release observability (rustfs/backlog#1172).
@@ -7846,6 +7857,9 @@ impl DiskAPI for LocalDisk {
         check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?;
 
         let no_inline = fi.data.is_none() && fi.size > 0;
+        // Captured before `fi` is consumed by add_version; gates the stale
+        // destination purge below.
+        let fi_healing = fi.is_healing();
 
         // Resolved once for the whole commit so a concurrent configuration
         // change can never leave a single rename_data half-synced. The tier is
@@ -7962,6 +7976,26 @@ impl DiskAPI for LocalDisk {
             shard_sync_res?;
             remove_dst_base_before_commit(dst_path).map_err(to_file_error)?;
 
+            // Heal reuses the version's data_dir, so for in-place corruption
+            // the destination dir still exists — and rename(2) cannot replace
+            // a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge
+            // it first, healing commits only; fresh PUTs mint a new data_dir
+            // and never collide. Best effort: a real failure surfaces in the
+            // rename below.
+            if fi_healing
+                && let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
+                && let Err(err) = self.move_to_trash(dst_data_path, true, false).await
+            {
+                warn!(
+                    event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
+                    dst_path = ?dst_data_path,
+                    error = ?err,
+                    "Healing commit could not purge the stale destination data dir"
+                );
+            }
+
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref()
                 && let Err(err) = rename_all(src_data_path, dst_data_path, &skip_parent).await
             {
@@ -11220,6 +11254,86 @@ mod test {
         (disk, dir)
     }
 
+    // Stage the bitrot-heal collision: a committed version whose data_dir is
+    // present and non-empty, plus a replacement shard staged in tmp for the
+    // SAME data_dir (heal repairs in place, it does not mint a new data_dir).
+    async fn stage_healing_collision(
+        bucket: &str,
+        object: &str,
+        tmp_object: &str,
+    ) -> (LocalDisk, tempfile::TempDir, std::path::PathBuf, FileInfo) {
+        use tempfile::tempdir;
+        let dir = tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        ensure_test_volume(&disk, bucket).await;
+        ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+
+        let version_id = Uuid::parse_str("dddddddd-dddd-dddd-dddd-dddddddddddd").expect("version id should parse");
+        let data_dir = Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("data dir should parse");
+
+        let object_dir = dir.path().join(bucket).join(object);
+        let dst_data_dir = object_dir.join(data_dir.to_string());
+        fs::create_dir_all(&dst_data_dir)
+            .await
+            .expect("dst data dir should be created");
+        fs::write(dst_data_dir.join("part.1"), b"stale-corrupt-shard")
+            .await
+            .expect("stale shard should be written");
+        let old_fi = test_file_info(object, version_id, Some(data_dir), None);
+        fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(old_fi))
+            .await
+            .expect("old metadata should be written");
+
+        let tmp_data_dir = dir
+            .path()
+            .join(RUSTFS_META_TMP_BUCKET)
+            .join(tmp_object)
+            .join(data_dir.to_string());
+        fs::create_dir_all(&tmp_data_dir)
+            .await
+            .expect("tmp data dir should be created");
+        fs::write(tmp_data_dir.join("part.1"), b"healed-shard")
+            .await
+            .expect("healed shard should be written");
+
+        let new_fi = test_file_info(object, version_id, Some(data_dir), None);
+        (disk, dir, dst_data_dir.join("part.1"), new_fi)
+    }
+
+    // A healing commit must replace a still-existing destination data dir;
+    // without the purge it failed on every attempt and bitrot was never
+    // repaired.
+    #[tokio::test]
+    async fn rename_data_healing_commit_replaces_stale_destination_data_dir() {
+        let (disk, _dir, dst_part, mut new_fi) = stage_healing_collision("bucket", "bitrot-object", "tmp-heal-object").await;
+        new_fi.set_healing();
+
+        disk.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-heal-object", new_fi, "bucket", "bitrot-object")
+            .await
+            .expect("a healing rename_data must replace the stale destination data dir");
+
+        let content = fs::read(&dst_part).await.expect("healed shard should be readable");
+        assert_eq!(content, b"healed-shard", "the healed shard must replace the stale corrupt content");
+    }
+
+    // The purge is healing-gated: an ordinary commit colliding with a
+    // non-empty data dir must keep failing loudly.
+    #[tokio::test]
+    async fn rename_data_non_healing_destination_collision_still_fails() {
+        let (disk, _dir, dst_part, new_fi) = stage_healing_collision("bucket", "collision-object", "tmp-collision-object").await;
+
+        disk.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-collision-object", new_fi, "bucket", "collision-object")
+            .await
+            .expect_err("a non-healing rename_data onto a non-empty destination data dir must fail");
+
+        let content = fs::read(&dst_part).await.expect("stale shard should still be readable");
+        assert_eq!(
+            content, b"stale-corrupt-shard",
+            "a failed non-healing commit must leave the existing content untouched"
+        );
+    }
+
     #[tokio::test]
     async fn test_rename_data_new_object_fsyncs_new_ancestor_dirs() {
         // A first PUT under a new prefix must fsync every newly created ancestor
@@ -12228,6 +12342,54 @@ mod test {
         );
     }
 
+    // The undo_write restore consumes `<rollback>/xl.meta.bkp` by rename; a
+    // synthetic rollback dir is then empty and must be reclaimed so the object
+    // dir can empty out (BucketNotEmpty leak). A real data dir still holds its
+    // parts and must survive the non-recursive remove.
+    #[tokio::test]
+    async fn restore_metadata_backup_reclaims_empty_rollback_dir_only() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("temp dir should be created");
+        let object_dir = dir.path().join("bucket").join("obj");
+        let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
+        let rollback_dir = Uuid::new_v4();
+        let rollback_path = object_dir.join(rollback_dir.to_string());
+        fs::create_dir_all(&rollback_path)
+            .await
+            .expect("rollback dir should be created");
+        fs::write(rollback_path.join(STORAGE_FORMAT_FILE_BACKUP), b"old-meta")
+            .await
+            .expect("backup should be written");
+
+        restore_metadata_backup(&object_dir, &xl_path, rollback_dir)
+            .await
+            .expect("restore should succeed");
+        assert_eq!(
+            fs::read(&xl_path).await.expect("xl.meta should be restored"),
+            b"old-meta",
+            "restore must move the backup back onto xl.meta"
+        );
+        assert!(!rollback_path.exists(), "an emptied synthetic rollback dir must be reclaimed");
+
+        // Real data dir: parts remain, the dir must survive.
+        let real_dir = Uuid::new_v4();
+        let real_path = object_dir.join(real_dir.to_string());
+        fs::create_dir_all(&real_path).await.expect("real data dir should be created");
+        fs::write(real_path.join(STORAGE_FORMAT_FILE_BACKUP), b"older-meta")
+            .await
+            .expect("backup should be written");
+        fs::write(real_path.join("part.1"), b"data")
+            .await
+            .expect("part should be written");
+
+        restore_metadata_backup(&object_dir, &xl_path, real_dir)
+            .await
+            .expect("restore should succeed");
+        assert!(real_path.join("part.1").exists(), "a real data dir must keep its parts");
+        assert!(real_path.exists(), "a non-empty data dir must not be removed");
+    }
+
     #[tokio::test]
     async fn rename_commit_failure_cleans_local_rollback_backup() {
         use tempfile::tempdir;

crates/ecstore/src/disk/mod.rs

@@ -42,6 +42,10 @@ pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
 pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
 pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
 
+const LOG_COMPONENT_ECSTORE: &str = "ecstore";
+const LOG_SUBSYSTEM_DISK: &str = "disk";
+const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
+
 pub fn part_transaction_path(part_path: &str) -> String {
     match part_path.rsplit_once('/') {
         Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
@@ -1196,7 +1200,13 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
         Some(DiskError::DiskNotFound) => CHECK_PART_DISK_NOT_FOUND,
         None => CHECK_PART_SUCCESS,
         _ => {
-            tracing::warn!("conv_part_err_to_int: unknown error: {err:?}");
+            tracing::warn!(
+                event = EVENT_DISK_PART_ERR_UNCLASSIFIED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_DISK,
+                error = ?err,
+                "Part error has no check-part code and degrades to unknown"
+            );
             CHECK_PART_UNKNOWN
         }
     }

crates/ecstore/src/ecstore_validation_blackbox.rs

@@ -7,6 +7,7 @@ use crate::io_support::rio::HashReader;
 use crate::object_api::{BLOCK_SIZE_V2, ObjectLockConfigSnapshot, ObjectOptions, PutObjReader};
 use crate::set_disk::SetDisks;
 use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations as _};
 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
 use crate::storage_api_contracts::range::HTTPRangeSpec;
 use crate::store::init_format::save_format_file;
@@ -206,8 +207,28 @@ async fn blackbox_get_restores_body_after_one_shard_file_is_removed() {
 #[tokio::test]
 // Serialized: forces the reader-setup strategy through a process-global env var.
 #[serial_test::serial]
-async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard() {
-    use rustfs_common::heal_channel::{HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealRequestSource};
+async fn blackbox_heal_requests_preserve_repair_scope() {
+    use rustfs_common::heal_channel::{
+        HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealRequestSource,
+    };
+
+    async fn receive_matching_heal(rx: &mut HealChannelReceiver, bucket: &str, object: &str) -> HealChannelRequest {
+        tokio::time::timeout(std::time::Duration::from_secs(30), async {
+            loop {
+                match rx.recv().await.expect("heal channel should stay open") {
+                    HealChannelCommand::Start { request, response_tx }
+                        if request.bucket == bucket && request.object_prefix.as_deref() == Some(object) =>
+                    {
+                        let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                        break request;
+                    }
+                    _ => continue,
+                }
+            }
+        })
+        .await
+        .expect("matching heal request should be submitted")
+    }
 
     // Own the process-global heal channel so the read path's repair submission
     // becomes observable. init_heal_channel() succeeds exactly once per test
@@ -220,6 +241,126 @@ async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard(
     let mut heal_rx = rustfs_common::heal_channel::init_heal_channel()
         .expect("this must be the only ecstore test that owns the heal channel receiver");
 
+    // Ordinary PUTs use the same admission channel as read repair. A single
+    // rename target failure still satisfies write quorum, so the committed
+    // version must be queued for convergence without delaying the PUT ACK.
+    let (_put_dirs, put_set) = make_local_set_disks(4, 2).await;
+    let put_bucket = "bb-put-partial-convergence";
+    let put_object = "object.bin";
+    put_set
+        .make_bucket(put_bucket, &MakeBucketOptions::default())
+        .await
+        .expect("PUT bucket should be created");
+    let offline_disk = {
+        let mut disks = put_set.disks.write().await;
+        disks[0].take()
+    };
+    let mut put_reader = PutObjReader::from_vec(vec![0x42; BLOCK_SIZE_V2 + 1024]);
+    let committed = put_set
+        .put_object(
+            put_bucket,
+            put_object,
+            &mut put_reader,
+            &ObjectOptions {
+                no_lock: true,
+                versioned: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("partial ordinary PUT should succeed at write quorum");
+    let committed_version = committed
+        .version_id
+        .expect("versioned PUT should return a version id")
+        .to_string();
+
+    let request = tokio::time::timeout(std::time::Duration::from_secs(30), async {
+        loop {
+            match heal_rx.recv().await.expect("heal channel should stay open") {
+                HealChannelCommand::Start { request, response_tx }
+                    if request.bucket == put_bucket && request.object_prefix.as_deref() == Some(put_object) =>
+                {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                    break request;
+                }
+                HealChannelCommand::Start { response_tx, .. } => {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                }
+                _ => {}
+            }
+        }
+    })
+    .await
+    .expect("partial ordinary PUT should enqueue convergence heal");
+    assert_eq!(request.object_version_id.as_deref(), Some(committed_version.as_str()));
+    assert_eq!(request.pool_index, Some(0));
+    assert_eq!(request.set_index, Some(0));
+
+    let duplicate_request = tokio::time::timeout(std::time::Duration::from_millis(100), async {
+        loop {
+            match heal_rx.recv().await.expect("heal channel should stay open") {
+                HealChannelCommand::Start { request, response_tx }
+                    if request.bucket == put_bucket && request.object_prefix.as_deref() == Some(put_object) =>
+                {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                    break Some(request);
+                }
+                HealChannelCommand::Start { response_tx, .. } => {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                }
+                _ => {}
+            }
+        }
+    })
+    .await
+    .ok()
+    .flatten();
+    assert!(duplicate_request.is_none(), "partial ordinary PUT must enqueue exactly one heal request");
+
+    {
+        let mut disks = put_set.disks.write().await;
+        disks[0] = offline_disk;
+    }
+
+    let healthy_bucket = "bb-put-healthy-convergence";
+    put_set
+        .make_bucket(healthy_bucket, &MakeBucketOptions::default())
+        .await
+        .expect("healthy PUT bucket should be created");
+    let mut healthy_reader = PutObjReader::from_vec(b"healthy".to_vec());
+    put_set
+        .put_object(
+            healthy_bucket,
+            put_object,
+            &mut healthy_reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("healthy ordinary PUT should succeed");
+    let healthy_request = tokio::time::timeout(std::time::Duration::from_millis(100), async {
+        loop {
+            match heal_rx.recv().await.expect("heal channel should stay open") {
+                HealChannelCommand::Start { request, response_tx }
+                    if request.bucket == healthy_bucket && request.object_prefix.as_deref() == Some(put_object) =>
+                {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                    break Some(request);
+                }
+                HealChannelCommand::Start { response_tx, .. } => {
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
+                }
+                _ => {}
+            }
+        }
+    })
+    .await
+    .ok()
+    .flatten();
+    assert!(healthy_request.is_none(), "fully converged ordinary PUT must not enqueue heal");
+
     // Keep data-blocks-first reader setup explicit for this deterministic
     // repair assertion (see ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP in
     // set_disk/core/io_primitives.rs): if a caller opts back into all-shards,
@@ -276,19 +417,7 @@ async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard(
 
         assert_eq!(restored, payload);
 
-        let request = tokio::time::timeout(std::time::Duration::from_secs(30), async {
-            loop {
-                match heal_rx.recv().await.expect("heal channel should stay open") {
-                    HealChannelCommand::Start { request, response_tx } if request.bucket == bucket => {
-                        let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
-                        break request;
-                    }
-                    _ => continue,
-                }
-            }
-        })
-        .await
-        .expect("corrupt-shard GET should enqueue a read-repair heal request");
+        let request = receive_matching_heal(&mut heal_rx, bucket, object).await;
 
         assert_eq!(request.source, HealRequestSource::ReadRepair);
         assert_eq!(request.object_prefix.as_deref(), Some(object));
@@ -297,6 +426,142 @@ async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard(
         assert_eq!(request.set_index, Some(0));
         assert_eq!(request.priority, HealChannelPriority::Low);
         assert_eq!(request.recreate_missing, Some(true));
+
+        let mpu_bucket = "bb-mpu-convergence-heal";
+        let partial_object = "partial.bin";
+        let suspended_object = "suspended.bin";
+        let payload = vec![0x5a; 1 << 20];
+        let mpu_opts = ObjectOptions {
+            no_lock: true,
+            versioned: true,
+            ..Default::default()
+        };
+
+        set_disks
+            .make_bucket(mpu_bucket, &MakeBucketOptions::default())
+            .await
+            .expect("multipart bucket should be created");
+
+        let stage_upload = async |object: &str| {
+            let upload = set_disks
+                .new_multipart_upload(mpu_bucket, object, &mpu_opts)
+                .await
+                .expect("multipart upload should be created");
+            let mut reader = PutObjReader::new(
+                HashReader::from_stream(
+                    Cursor::new(payload.clone()),
+                    payload.len() as i64,
+                    payload.len() as i64,
+                    None,
+                    None,
+                    false,
+                )
+                .expect("multipart reader should be constructed"),
+            );
+            let part = set_disks
+                .put_object_part(mpu_bucket, object, &upload.upload_id, 1, &mut reader, &mpu_opts)
+                .await
+                .expect("multipart part should be written");
+            (
+                upload.upload_id,
+                vec![CompletePart {
+                    part_num: part.part_num,
+                    etag: part.etag,
+                    ..Default::default()
+                }],
+            )
+        };
+
+        let (partial_upload_id, partial_parts) = stage_upload(partial_object).await;
+        let offline_disk = {
+            let mut disks = set_disks.disks.write().await;
+            disks[3].take().expect("fourth disk should be online before completion")
+        };
+        crate::crash_inject::arm(crate::crash_inject::CrashPoint::MultipartAfterCommitBeforePartsCleanup, partial_object);
+        let completed = set_disks
+            .clone()
+            .complete_multipart_upload(mpu_bucket, partial_object, &partial_upload_id, partial_parts, &mpu_opts)
+            .await;
+        assert!(
+            matches!(completed, Err(Error::Unexpected)),
+            "partial multipart completion should reach the post-commit crash point, got {completed:?}"
+        );
+        crate::crash_inject::disarm(crate::crash_inject::CrashPoint::MultipartAfterCommitBeforePartsCleanup, partial_object);
+
+        let request = receive_matching_heal(&mut heal_rx, mpu_bucket, partial_object).await;
+
+        let completed_version_id = request
+            .object_version_id
+            .clone()
+            .expect("versioned multipart convergence heal must bind a version id");
+        assert_eq!(request.pool_index, Some(0));
+        assert_eq!(request.set_index, Some(0));
+        assert_eq!(request.priority, HealChannelPriority::Normal);
+
+        let duplicate = tokio::time::timeout(std::time::Duration::from_millis(250), async {
+            loop {
+                match heal_rx.recv().await.expect("heal channel should stay open") {
+                    HealChannelCommand::Start { request, .. }
+                        if request.bucket == mpu_bucket && request.object_prefix.as_deref() == Some(partial_object) =>
+                    {
+                        break request;
+                    }
+                    _ => continue,
+                }
+            }
+        })
+        .await;
+        assert!(duplicate.is_err(), "partial multipart completion must enqueue exactly one heal request");
+
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[3] = Some(offline_disk);
+        }
+        let committed = set_disks
+            .get_object_info(
+                mpu_bucket,
+                partial_object,
+                &ObjectOptions {
+                    no_lock: true,
+                    versioned: true,
+                    version_id: Some(completed_version_id.clone()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("heal-bound multipart version should be committed and addressable");
+        assert_eq!(committed.version_id.map(|version_id| version_id.to_string()), Some(completed_version_id));
+
+        let (suspended_upload_id, suspended_parts) = stage_upload(suspended_object).await;
+        let offline_disk = {
+            let mut disks = set_disks.disks.write().await;
+            disks[3]
+                .take()
+                .expect("fourth disk should be online before suspended completion")
+        };
+        let suspended_opts = ObjectOptions {
+            no_lock: true,
+            version_suspended: true,
+            ..Default::default()
+        };
+        let suspended = set_disks
+            .clone()
+            .complete_multipart_upload(mpu_bucket, suspended_object, &suspended_upload_id, suspended_parts, &suspended_opts)
+            .await
+            .expect("suspended multipart completion should succeed at write quorum");
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[3] = Some(offline_disk);
+        }
+        assert!(
+            suspended.version_id.is_some_and(|version_id| version_id.is_nil()),
+            "suspended multipart completion should publish the null version"
+        );
+
+        let request = receive_matching_heal(&mut heal_rx, mpu_bucket, suspended_object).await;
+
+        let null_version_id = uuid::Uuid::nil().to_string();
+        assert_eq!(request.object_version_id.as_deref(), Some(null_version_id.as_str()));
     })
     .await;
 }

crates/ecstore/src/erasure/coding/erasure.rs

@@ -20,12 +20,21 @@ use bytes::{Bytes, BytesMut};
 use reed_solomon_erasure::galois_8::ReedSolomon;
 use reed_solomon_simd;
 use smallvec::SmallVec;
-use std::io;
+use std::{
+    collections::HashMap,
+    io,
+    sync::{Arc, OnceLock, RwLock},
+};
 use tokio::io::AsyncRead;
 use tracing::warn;
 use uuid::Uuid;
 
 const MODERN_MAX_TOTAL_SHARDS: usize = <reed_solomon_erasure::galois_8::Field as reed_solomon_erasure::Field>::ORDER;
+const MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 64;
+
+type ModernReedSolomonCache = RwLock<HashMap<(usize, usize), Arc<ReedSolomon>>>;
+
+static MODERN_REED_SOLOMON_CACHE: OnceLock<ModernReedSolomonCache> = OnceLock::new();
 
 /// Errors returned when constructing an [`Erasure`] codec.
 #[derive(Debug, thiserror::Error)]
@@ -275,7 +284,7 @@ impl LegacyReedSolomonEncoder {
 pub struct ReedSolomonEncoder {
     data_shards: usize,
     parity_shards: usize,
-    encoder: Option<ReedSolomon>,
+    encoder: Option<Arc<ReedSolomon>>,
 }
 
 impl Clone for ReedSolomonEncoder {
@@ -291,7 +300,7 @@ impl Clone for ReedSolomonEncoder {
 impl ReedSolomonEncoder {
     fn try_new_typed(data_shards: usize, parity_shards: usize) -> Result<Self, reed_solomon_erasure::Error> {
         let encoder = if parity_shards > 0 {
-            Some(ReedSolomon::new(data_shards, parity_shards)?)
+            Some(cached_modern_reed_solomon(data_shards, parity_shards)?)
         } else {
             None
         };
@@ -362,6 +371,30 @@ impl ReedSolomonEncoder {
     }
 }
 
+fn cached_modern_reed_solomon(data_shards: usize, parity_shards: usize) -> Result<Arc<ReedSolomon>, reed_solomon_erasure::Error> {
+    let key = (data_shards, parity_shards);
+    let cache = MODERN_REED_SOLOMON_CACHE.get_or_init(|| RwLock::new(HashMap::new()));
+
+    if let Some(encoder) = cache
+        .read()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+        .get(&key)
+        .cloned()
+    {
+        return Ok(encoder);
+    }
+
+    let encoder = Arc::new(ReedSolomon::new(data_shards, parity_shards)?);
+    let mut cache = cache.write().unwrap_or_else(|poisoned| poisoned.into_inner());
+    if let Some(existing) = cache.get(&key) {
+        return Ok(Arc::clone(existing));
+    }
+    if cache.len() < MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES {
+        cache.insert(key, Arc::clone(&encoder));
+    }
+    Ok(encoder)
+}
+
 fn encode_parity_shards<F>(shards: &mut [Option<Vec<u8>>], data_shards: usize, parity_shards: usize, encode: F) -> io::Result<()>
 where
     F: FnOnce(SmallVec<[&mut [u8]; 16]>) -> io::Result<()>,
@@ -1272,6 +1305,16 @@ mod tests {
         assert!(legacy.legacy_encoder.is_some());
     }
 
+    #[test]
+    fn modern_encoder_construction_reuses_cached_codec() {
+        let first = ReedSolomonEncoder::try_new_typed(31, 7).expect("modern codec should construct");
+        let second = ReedSolomonEncoder::try_new_typed(31, 7).expect("modern codec should construct");
+
+        let first = first.encoder.as_ref().expect("modern codec should initialize an encoder");
+        let second = second.encoder.as_ref().expect("modern codec should initialize an encoder");
+        assert!(Arc::ptr_eq(first, second));
+    }
+
     #[test]
     fn construction_errors_preserve_encoder_sources() {
         let modern = ErasureConstructionError::ModernEncoder {

crates/ecstore/src/erasure/coding/heal.rs

@@ -24,7 +24,11 @@ use std::io;
 use std::io::ErrorKind;
 use std::time::Duration;
 use tokio::io::AsyncRead;
-use tracing::{info, warn};
+use tracing::{trace, warn};
+
+const LOG_COMPONENT_ECSTORE: &str = "ecstore";
+const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
+const EVENT_ERASURE_HEAL_STARTED: &str = "erasure_heal_started";
 
 async fn read_heal_shards<R>(
     readers: &mut [Option<BitrotReader<R>>],
@@ -115,11 +119,14 @@ impl super::Erasure {
     where
         R: AsyncRead + Unpin + Send + Sync,
     {
-        info!(
-            "Erasure heal, writers len: {}, readers len: {}, total_length: {}",
-            writers.len(),
-            readers.len(),
-            total_length
+        trace!(
+            event = EVENT_ERASURE_HEAL_STARTED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_ERASURE,
+            writer_count = writers.len(),
+            reader_count = readers.len(),
+            total_length,
+            "Erasure heal started"
         );
         if writers.len() != self.parity_shards + self.data_shards {
             return Err(Error::other("invalid argument"));

crates/ecstore/src/io_support/rio.rs

@@ -380,6 +380,12 @@ impl WritePlan {
     }
 
     pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
+        // Transformations create new HashReaders around the plaintext reader. Keep
+        // the request checksum metadata on the final reader for multipart/single
+        // PUT persistence, but leave verification to the plaintext reader.
+        let checksum = reader.content_hash().clone();
+        let trailer = reader.get_trailer().cloned();
+
         let encrypted = self.encryption.is_some();
         if let Some(algorithm) = self.compression {
             reader = HashReader::from_reader(
@@ -438,6 +444,12 @@ impl WritePlan {
             };
         }
 
+        // `ignore_value` deliberately avoids a second hasher over compressed or
+        // encrypted bytes. The inner reader still validates the plaintext request
+        // checksum while this outer reader exposes the request checksum context.
+        reader.add_non_trailing_checksum(checksum, true)?;
+        reader.set_trailer(trailer);
+
         Ok(reader)
     }
 }
@@ -445,10 +457,73 @@ impl WritePlan {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use http::{HeaderMap, HeaderValue};
+    use rustfs_rio::{Checksum, ChecksumType};
     use rustfs_utils::CompressionAlgorithm;
     use std::io::Cursor;
     use tokio::io::AsyncReadExt;
 
+    async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
+        let plaintext = b"checksum-context-through-write-plan".repeat(256);
+        let actual_size = plaintext.len() as i64;
+        let checksum = Checksum::new_from_data(ChecksumType::CRC32, &plaintext).expect("create CRC32 checksum");
+        let mut reader = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+        reader
+            .add_non_trailing_checksum(Some(checksum.clone()), false)
+            .expect("attach plaintext checksum");
+
+        let mut transformed = plan.apply(reader, actual_size).expect("apply write plan");
+        assert_eq!(transformed.content_crc_type(), Some(ChecksumType::CRC32));
+
+        let mut transformed_bytes = Vec::new();
+        transformed
+            .read_to_end(&mut transformed_bytes)
+            .await
+            .expect("stream transformed data without rehashing ciphertext");
+
+        assert!(!transformed_bytes.is_empty());
+        assert_eq!(transformed.content_crc().get("CRC32"), Some(&checksum.encoded));
+    }
+
+    #[tokio::test]
+    async fn write_plan_preserves_non_trailing_checksum_context_across_transforms() {
+        assert_non_trailing_checksum_survives(WritePlan::new().with_compression(CompressionAlgorithm::default())).await;
+        assert_non_trailing_checksum_survives(
+            WritePlan::new().with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12])),
+        )
+        .await;
+        assert_non_trailing_checksum_survives(
+            WritePlan::new()
+                .with_compression(CompressionAlgorithm::default())
+                .with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12])),
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn write_plan_preserves_trailing_checksum_type_across_transforms() {
+        let plaintext = b"trailing-checksum-context".to_vec();
+        let actual_size = plaintext.len() as i64;
+        let mut reader = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32"));
+        reader
+            .add_checksum_from_s3s(&headers, None, false)
+            .expect("attach trailing checksum metadata");
+
+        let transformed = WritePlan::new()
+            .with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12]))
+            .apply(reader, actual_size)
+            .expect("apply encryption plan");
+
+        assert_eq!(
+            transformed.content_crc_type(),
+            Some(ChecksumType(ChecksumType::CRC32.0 | ChecksumType::TRAILING.0))
+        );
+    }
+
     #[cfg(feature = "rio-v2")]
     fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
         let mut chunk_types = Vec::new();

crates/ecstore/src/runtime/sources.rs

@@ -246,6 +246,22 @@ pub fn deployment_id() -> Option<String> {
     get_global_deployment_id()
 }
 
+/// Test-only inverse of [`deployment_upload_id`]: returns the raw
+/// `<uuid>x<timestamp>` suffix without the deployment-id prefix. Under plain
+/// `cargo test` (thread-parallel, shared process globals) a concurrently
+/// running test that re-initializes a store can swap the global deployment id
+/// between create time and list time, so assertions must compare only this
+/// suffix, never the full encoded upload id.
+#[cfg(test)]
+pub(crate) fn upload_uuid_suffix(upload_id: &str) -> String {
+    base64_simd::URL_SAFE_NO_PAD
+        .decode_to_vec(upload_id.as_bytes())
+        .ok()
+        .and_then(|decoded| String::from_utf8(decoded).ok())
+        .and_then(|decoded| decoded.split_once('.').map(|(_, suffix)| suffix.to_owned()))
+        .unwrap_or_else(|| upload_id.to_owned())
+}
+
 pub(crate) fn replication_pool() -> Option<Arc<DynReplicationPool>> {
     crate::runtime::global::current_ctx().replication_pool()
 }
@@ -549,8 +565,13 @@ pub(crate) async fn initialize_local_disk_maps(
     endpoint_pools: EndpointServerPools,
     opt: &DiskOption,
 ) -> Result<()> {
+    // Every caller passes the FULL topology, so (re)initialization must replace
+    // any previous registration wholesale: appending would leave the pool/set
+    // vectors sized for a stale topology and panic on wider disk indices (seen
+    // as cross-test contamination under single-process `cargo test`).
     let set_drives = instance_ctx.local_disk_set_drives();
     let mut global_set_drives = set_drives.write().await;
+    global_set_drives.clear();
     for pool_eps in endpoint_pools.as_ref().iter() {
         let mut set_count_drives = Vec::with_capacity(pool_eps.set_count);
         for _ in 0..pool_eps.set_count {
@@ -562,6 +583,7 @@ pub(crate) async fn initialize_local_disk_maps(
 
     let map = instance_ctx.local_disk_map();
     let mut global_local_disk_map = map.write().await;
+    global_local_disk_map.clear();
 
     for pool_eps in endpoint_pools.as_ref().iter() {
         for ep in pool_eps.endpoints.as_ref().iter() {
@@ -727,4 +749,69 @@ mod tests {
         process_ctx.local_disk_id_map().write().await.remove(&process_sentinel);
         bootstrap_ctx.local_disk_id_map().write().await.remove(&bootstrap_sentinel);
     }
+
+    /// Re-initializing the same context with a WIDER topology must replace the
+    /// previous registration, not append to it: the stale pool-0 drive vector
+    /// (sized for the narrow topology) made `global_set_drives[0][0][disk_idx]`
+    /// panic for the wider set's higher disk indices. CI's nextest
+    /// process-per-test isolation never exercises re-init, so this pins it.
+    #[tokio::test]
+    async fn reinitializing_local_disk_maps_replaces_previous_topology() {
+        use crate::disk::DiskOption;
+        use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+
+        let temp_dir = tempfile::tempdir().expect("reinit test directory should be created");
+        let build_pools = |label: &str, disk_count: usize| {
+            let mut endpoints = Vec::new();
+            for disk_idx in 0..disk_count {
+                let disk_path = temp_dir.path().join(format!("{label}-disk{disk_idx}"));
+                std::fs::create_dir_all(&disk_path).expect("reinit test disk should be created");
+                let mut endpoint =
+                    Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
+                endpoint.set_pool_index(0);
+                endpoint.set_set_index(0);
+                endpoint.set_disk_index(disk_idx);
+                endpoints.push(endpoint);
+            }
+            EndpointServerPools(vec![PoolEndpoints {
+                legacy: false,
+                set_count: 1,
+                drives_per_set: disk_count,
+                endpoints: Endpoints::from(endpoints),
+                cmd_line: format!("reinit-test-{label}"),
+                platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+            }])
+        };
+        let opt = DiskOption {
+            cleanup: false,
+            health_check: false,
+        };
+
+        let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        super::initialize_local_disk_maps(&instance_ctx, build_pools("narrow", 2), &opt)
+            .await
+            .expect("narrow topology should initialize");
+        super::initialize_local_disk_maps(&instance_ctx, build_pools("wide", 4), &opt)
+            .await
+            .expect("re-initializing with a wider topology must not panic or fail");
+
+        let set_drives = instance_ctx.local_disk_set_drives();
+        let set_drives = set_drives.read().await;
+        assert_eq!(set_drives.len(), 1, "stale pools must not accumulate across re-inits");
+        assert_eq!(set_drives[0][0].len(), 4, "pool 0 set 0 must be sized for the new topology");
+        assert!(
+            set_drives[0][0].iter().all(Option::is_some),
+            "every wide-topology drive slot must be registered"
+        );
+        drop(set_drives);
+
+        let disk_map = instance_ctx.local_disk_map();
+        let disk_map = disk_map.read().await;
+        assert_eq!(disk_map.len(), 4, "stale narrow-topology disk entries must be dropped");
+        assert!(
+            disk_map.keys().all(|path| path.contains("wide-disk")),
+            "only the new topology's disks may remain registered: {:?}",
+            disk_map.keys().collect::<Vec<_>>()
+        );
+    }
 }

crates/ecstore/src/services/metrics_realtime.rs

@@ -15,7 +15,9 @@
 use crate::diagnostics::admin_server_info::get_local_server_property;
 use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::admin::StorageAdminApi;
+#[cfg(test)]
 use chrono::Utc;
+use jiff::Timestamp;
 use rustfs_common::{heal_channel::DriveState, metrics::global_metrics};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
 use rustfs_madmin::metrics::{
@@ -67,6 +69,18 @@ impl MetricType {
     }
 }
 
+fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp {
+    let millis = match i64::try_from(millis) {
+        Ok(millis) => millis,
+        Err(_) => return fallback,
+    };
+
+    match Timestamp::from_millisecond(millis) {
+        Ok(timestamp) => timestamp,
+        Err(_) => fallback,
+    }
+}
+
 fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
     MadminScannerMetrics {
         collected_at: metrics.collected_at,
@@ -386,7 +400,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
     if types.contains(&MetricType::DISK) {
         debug!("start get disk metrics");
         let mut aggr = DiskMetric {
-            collected_at: Utc::now(),
+            collected_at: Timestamp::now(),
             ..Default::default()
         };
         for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
@@ -412,7 +426,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
     if types.contains(&MetricType::NET) {
         let snapshot = global_internode_metrics().snapshot();
         real_time_metrics.aggregated.net = Some(NetMetrics {
-            collected_at: Utc::now(),
+            collected_at: Timestamp::now(),
             interface_name: "internode".to_string(),
             net_stats: NetDevLine {
                 name: "internode".to_string(),
@@ -428,10 +442,9 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
     // if types.contains(&MetricType::CPU) {}
 
     if types.contains(&MetricType::RPC) {
-        let collected_at = Utc::now();
+        let collected_at = Timestamp::now();
         let snapshot = global_internode_metrics().snapshot();
-        let last_connect_time =
-            chrono::DateTime::<Utc>::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at);
+        let last_connect_time = unix_millis_to_jiff_timestamp(snapshot.last_dial_unix_millis, collected_at);
 
         real_time_metrics.aggregated.rpc = Some(RPCMetrics {
             collected_at,
@@ -543,6 +556,10 @@ mod test {
     use serial_test::serial;
     use std::time::Duration;
 
+    fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
+        jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
+    }
+
     #[test]
     fn tes_types() {
         let t = MetricType::ALL;
@@ -591,7 +608,7 @@ mod test {
         let current_started = Utc::now() - chrono::Duration::seconds(5);
         let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
             current_cycle_active: true,
-            current_started,
+            current_started: chrono_to_jiff_timestamp(current_started),
             last_cycle_partial_source: "usage".to_string(),
             last_cycle_partial_source_code: 1,
             partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
@@ -602,7 +619,7 @@ mod test {
         });
 
         assert_eq!(scanner.current_cycle_active, Some(true));
-        assert_eq!(scanner.current_started, current_started);
+        assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
         assert_eq!(scanner.last_cycle_partial_source, "usage");
         assert_eq!(scanner.last_cycle_partial_source_code, 1);
         let usage = scanner
@@ -643,7 +660,7 @@ mod test {
         aggregated.merge(decoded);
         let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
         assert_eq!(scanner.current_cycle_active, Some(true));
-        assert_eq!(scanner.current_started, cycle_started);
+        assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
     }
 
     #[test]

crates/ecstore/src/services/rebalance/entry.rs

@@ -18,8 +18,8 @@ use super::meta::{
 };
 use super::migration::migrate_entry_version;
 use super::worker::{
-    RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error,
-    resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
+    RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
+    resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
     resolve_rebalance_migrate_result_error, resolve_rebalance_stats_update_result, resolve_rebalance_worker_result,
     run_rebalance_listing_with_retry, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
     should_defer_rebalance_entry_failure, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks,
@@ -27,7 +27,7 @@ use super::worker::{
 };
 use super::{
     EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_ENTRY, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
-    REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
+    ObjectInfo, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
 };
 use crate::core::pools::ListCallback;
 use crate::data_movement;
@@ -37,13 +37,55 @@ use crate::object_api::{GetObjectReader, ObjectOptions};
 use crate::set_disk::SetDisks;
 use crate::storage_api_contracts::object::ObjectOperations as _;
 use crate::store::ECStore;
-use rustfs_filemeta::MetaCacheEntry;
+use rustfs_filemeta::{FileInfo, MetaCacheEntry};
 use std::sync::Arc;
 use time::OffsetDateTime;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
 impl ECStore {
+    async fn finish_rebalance_entry_after_cleanup(
+        &self,
+        pool_index: usize,
+        bucket: &str,
+        object: &str,
+        stats_updates: &[&FileInfo],
+        cleanup: impl std::future::Future<Output = std::result::Result<ObjectInfo, data_movement::SourceCleanupError>>,
+    ) -> Result<RebalanceEntryCleanupResult> {
+        // Persisted stats can complete a pool on restart, so source cleanup must resolve first.
+        let cleanup_result = resolve_rebalance_entry_cleanup_delete_result(cleanup.await, bucket, object);
+        let RebalanceEntryCleanupResult::Completed { warning } = cleanup_result else {
+            return Ok(cleanup_result);
+        };
+        if let Some(message) = warning.as_ref()
+            && let Err(err) = self
+                .record_rebalance_cleanup_warning(pool_index, bucket, object, message.clone())
+                .await
+        {
+            error!(
+                event = EVENT_REBALANCE_ENTRY,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                pool_index,
+                bucket,
+                object,
+                stage = "cleanup_source",
+                error = ?err,
+                "Failed to record rebalance source cleanup warning"
+            );
+        }
+
+        resolve_rebalance_stats_update_result(
+            self.update_pool_stats_batch(pool_index, bucket.to_string(), stats_updates)
+                .await,
+            pool_index,
+            bucket,
+            object,
+        )?;
+
+        Ok(RebalanceEntryCleanupResult::Completed { warning })
+    }
+
     #[allow(unused_assignments)]
     #[tracing::instrument(skip(self, set))]
     async fn rebalance_entry(
@@ -216,7 +258,7 @@ impl ECStore {
                 );
                 if should_defer_rebalance_entry_failure(&err) {
                     let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
-                    warn!(
+                    debug!(
                         event = EVENT_REBALANCE_ENTRY,
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_REBALANCE,
@@ -260,46 +302,40 @@ impl ECStore {
             }
         }
 
-        resolve_rebalance_stats_update_result(
-            self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
-                .await,
-            pool_index,
-            bucket.as_str(),
-            entry.name.as_str(),
-        )?;
-
         if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
-            let cleanup_warning = resolve_rebalance_entry_cleanup_delete_result(
-                data_movement::cleanup_source_entry_if_unchanged(
-                    set.clone(),
+            let cleanup_result = self
+                .finish_rebalance_entry_after_cleanup(
+                    pool_index,
                     bucket.as_str(),
                     entry.name.as_str(),
-                    &fivs,
-                    &cleanup_preflight_allowed_missing,
-                    "rebalance",
+                    stats_updates.as_slice(),
+                    data_movement::cleanup_source_entry_if_unchanged(
+                        set.clone(),
+                        bucket.as_str(),
+                        entry.name.as_str(),
+                        &fivs,
+                        &cleanup_preflight_allowed_missing,
+                        "rebalance",
+                    ),
                 )
-                .await,
-                bucket.as_str(),
-                entry.name.as_str(),
-            )?;
-            if let Some(message) = cleanup_warning {
-                warn!(
-                    event = EVENT_REBALANCE_ENTRY,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REBALANCE,
-                    pool_index,
-                    bucket = %bucket,
-                    object = %entry.name,
-                    stage = "cleanup_source",
-                    cleanup_status = "failed_ignored",
-                    error = %message,
-                    "Ignored rebalance source cleanup failure"
-                );
-                if let Err(err) = self
-                    .record_rebalance_cleanup_warning(pool_index, bucket.as_str(), entry.name.as_str(), message)
-                    .await
-                {
-                    error!(
+                .await?;
+            match cleanup_result {
+                RebalanceEntryCleanupResult::Deferred { last_error } => {
+                    debug!(
+                        event = EVENT_REBALANCE_ENTRY,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        bucket = %bucket,
+                        object = %entry.name,
+                        state = "deferred",
+                        error = %last_error,
+                        "Deferred rebalance entry after source cleanup conflict"
+                    );
+                    return Ok(RebalanceEntryOutcome::Deferred { last_error });
+                }
+                RebalanceEntryCleanupResult::Completed { warning: Some(message) } => {
+                    warn!(
                         event = EVENT_REBALANCE_ENTRY,
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_REBALANCE,
@@ -307,21 +343,23 @@ impl ECStore {
                         bucket = %bucket,
                         object = %entry.name,
                         stage = "cleanup_source",
-                        error = ?err,
-                        "Failed to record rebalance source cleanup warning"
+                        cleanup_status = "failed_ignored",
+                        error = %message,
+                        "Ignored rebalance source cleanup failure"
+                    );
+                }
+                RebalanceEntryCleanupResult::Completed { warning: None } => {
+                    debug!(
+                        event = EVENT_REBALANCE_ENTRY,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        pool_index,
+                        bucket = %bucket,
+                        object = %entry.name,
+                        state = "source_deleted",
+                        "Deleted rebalance source entry"
                     );
                 }
-            } else {
-                debug!(
-                    event = EVENT_REBALANCE_ENTRY,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REBALANCE,
-                    pool_index,
-                    bucket = %bucket,
-                    object = %entry.name,
-                    state = "source_deleted",
-                    "Deleted rebalance source entry"
-                );
             }
         } else if rebalanced != fivs.versions.len() || expired > 0 {
             warn!(
@@ -337,6 +375,14 @@ impl ECStore {
                 state = "source_retained",
                 "Rebalance source object retained"
             );
+
+            resolve_rebalance_stats_update_result(
+                self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
+                    .await,
+                pool_index,
+                bucket.as_str(),
+                entry.name.as_str(),
+            )?;
         }
 
         Ok(RebalanceEntryOutcome::Completed)
@@ -493,9 +539,23 @@ impl ECStore {
             let entry_tasks = entry_tasks.clone();
 
             let job = tokio::spawn(async move {
-                let list_result =
-                    run_rebalance_listing_with_retry(set, rx, bucket.clone(), rebalance_entry, set_idx, rebalance_max_attempts())
-                        .await;
+                let list_rx = rx.clone();
+                let list_bucket = bucket.clone();
+                let list_result = run_rebalance_listing_with_retry(
+                    rx,
+                    bucket,
+                    rebalance_entry,
+                    set_idx,
+                    rebalance_max_attempts(),
+                    entry_tasks.clone(),
+                    move |cb| {
+                        let set = set.clone();
+                        let rx = list_rx.clone();
+                        let bucket = list_bucket.clone();
+                        async move { set.list_objects_to_rebalance(rx, bucket, cb).await }
+                    },
+                )
+                .await;
                 let entry_result = wait_rebalance_entry_tasks(set_idx, entry_tasks).await;
                 let result = list_result.and(entry_result);
                 if let Err(err) = &result {
@@ -548,3 +608,124 @@ impl ECStore {
         Ok(RebalanceBucketOutcome::Completed)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
+    use rustfs_filemeta::FileInfo;
+    use time::OffsetDateTime;
+
+    #[tokio::test]
+    async fn rebalance_stats_wait_for_source_cleanup_result() {
+        let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
+        let store = Arc::new(ECStore {
+            id: uuid::Uuid::new_v4(),
+            disk_map: std::collections::HashMap::new(),
+            pools: Vec::new(),
+            peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
+            pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
+            rebalance_meta: tokio::sync::RwLock::new(Some(RebalanceMeta {
+                pool_stats: vec![RebalanceStats {
+                    participating: true,
+                    info: RebalanceInfo {
+                        start_time: Some(OffsetDateTime::now_utc()),
+                        status: RebalStatus::Started,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                }],
+                ..Default::default()
+            })),
+            decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(()),
+            ctx: crate::runtime::instance::bootstrap_ctx(),
+            bucket_fence_registry: std::sync::Arc::default(),
+        });
+        let mut version = FileInfo::new("object.bin", 4, 2);
+        version.name = "object.bin".to_string();
+        version.size = 128;
+        version.is_latest = true;
+        let warning_version = version.clone();
+        let (release_cleanup, cleanup_released) = tokio::sync::oneshot::channel();
+
+        let finish_store = Arc::clone(&store);
+        let finish = tokio::spawn(async move {
+            finish_store
+                .finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&version], async move {
+                    cleanup_released.await.expect("cleanup release sender should remain alive");
+                    Ok(ObjectInfo::default())
+                })
+                .await
+        });
+
+        tokio::task::yield_now().await;
+        assert_eq!(
+            store
+                .rebalance_meta
+                .read()
+                .await
+                .as_ref()
+                .expect("rebalance metadata should exist")
+                .pool_stats[0]
+                .bytes,
+            0,
+            "stats must not become visible before source cleanup resolves"
+        );
+
+        release_cleanup.send(()).expect("cleanup waiter should remain alive");
+        assert_eq!(
+            finish
+                .await
+                .expect("finish task should not panic")
+                .expect("finish should succeed"),
+            RebalanceEntryCleanupResult::Completed { warning: None }
+        );
+        assert!(
+            store
+                .rebalance_meta
+                .read()
+                .await
+                .as_ref()
+                .expect("rebalance metadata should exist")
+                .pool_stats[0]
+                .bytes
+                > 0,
+            "stats should become visible after source cleanup resolves"
+        );
+
+        {
+            let mut meta = store.rebalance_meta.write().await;
+            meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
+        }
+        let warning_result = store
+            .finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
+                Err(Error::SlowDown.into())
+            })
+            .await
+            .expect("cleanup warnings should not fail the completed migration");
+        assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
+        let meta = store.rebalance_meta.read().await;
+        let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+        assert_eq!(pool_stats.cleanup_warnings.count, 1, "cleanup warning must block pool completion");
+        assert!(pool_stats.bytes > 0, "completed migration bytes should still be recorded");
+        drop(meta);
+
+        {
+            let mut meta = store.rebalance_meta.write().await;
+            meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
+        }
+        let deferred = store
+            .finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
+                Err(data_movement::SourceCleanupError::SourceChanged)
+            })
+            .await
+            .expect("source changes should defer cleanup without failing the worker");
+        assert!(matches!(deferred, RebalanceEntryCleanupResult::Deferred { .. }));
+        let meta = store.rebalance_meta.read().await;
+        let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+        assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
+        assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
+    }
+}

crates/ecstore/src/services/rebalance/mod.rs

@@ -27,12 +27,14 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
 const REBAL_META_VER: u16 = 1; // Replace with actual version value
 pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
 const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
+pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
 const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
 const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
 const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
 const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
 const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
 const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
+pub(crate) const REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX: &str = "deferred rebalance source cleanup conflict:";
 const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
 
 mod control;

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX;
 use super::control::validate_rebalance_disk_stats_coverage;
 use super::meta::{
     RebalanceMetaMergeOutcome, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
@@ -33,13 +32,15 @@ use super::migration::{
     MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
     rebalance_delete_marker_opts,
 };
+use super::runtime::{should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt};
 use super::worker::{
-    ensure_rebalance_listing_disks_available, is_transient_rebalance_error, parse_rebalance_max_attempts,
-    rebalance_listing_retry_delay, rebalance_migration_retry_delay, resolve_load_rebalance_stats_update_result,
-    resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result,
-    resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
-    resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
-    resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
+    RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
+    parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
+    resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
+    resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
+    resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
+    resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
+    resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
     send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
     should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
     wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
@@ -48,15 +49,17 @@ use super::{
     DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
     RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
 };
+use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
 use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
 use crate::data_movement;
+use crate::data_movement::SourceCleanupError;
 use crate::data_usage::DATA_USAGE_CACHE_NAME;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::disk::error::DiskError;
 use crate::error::{Error, Result};
 use crate::storage_api_contracts::range::HTTPRangeSpec;
-use rustfs_filemeta::FileInfo;
 use rustfs_filemeta::TRANSITION_COMPLETE;
+use rustfs_filemeta::{FileInfo, MetaCacheEntry};
 use rustfs_rio::Index;
 use s3s::dto::ReplicationConfiguration;
 use serde::Serialize;
@@ -1665,26 +1668,63 @@ fn test_resolve_rebalance_meta_load_result_wraps_error_context() {
 #[test]
 fn test_resolve_rebalance_entry_cleanup_delete_result_passthrough() {
     let result = resolve_rebalance_entry_cleanup_delete_result(Ok(ObjectInfo::default()), "bucket-a", "obj.txt");
-    assert_eq!(result.expect("successful cleanup should pass through"), None);
+    assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
 }
 
 #[test]
 fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
     let result = resolve_rebalance_entry_cleanup_delete_result(
-        Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string())),
+        Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()).into()),
         "bucket-a",
         "obj.txt",
     );
-    assert_eq!(result.expect("missing cleanup source should be ignored"), None);
+    assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
 }
 
 #[test]
 fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
-    let warning = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
-        .expect("cleanup delete failures should be downgraded to warnings")
-        .expect("cleanup delete failure should return warning");
-    let message = warning.as_str();
-    assert!(message.contains("rebalance cleanup delete failed for bucket-a/obj.txt"));
+    let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
+    assert!(matches!(
+        result,
+        RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
+            if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
+    ));
+}
+
+#[test]
+fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
+    let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
+    assert!(matches!(
+        result,
+        RebalanceEntryCleanupResult::Deferred { ref last_error }
+            if last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX)
+                && last_error.contains("source changed during cleanup preflight for bucket-a/obj.txt")
+    ));
+}
+
+#[test]
+fn test_resolve_rebalance_entry_cleanup_delete_result_does_not_defer_other_precondition_failure() {
+    let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::PreconditionFailed.into()), "bucket-a", "obj.txt");
+    assert!(matches!(
+        result,
+        RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
+            if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
+    ));
+}
+
+#[test]
+fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
+    let mut deferred_buckets = std::collections::HashSet::new();
+
+    assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
+    assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
+    assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
+    assert!(should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
+
+    let mut source_attempts = std::collections::HashMap::new();
+    assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
+    assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
+    assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
 }
 
 #[test]
@@ -1809,6 +1849,109 @@ fn test_should_retry_rebalance_listing_respects_attempt_limit_and_error_type() {
     assert!(!should_retry_rebalance_listing(&Error::FileAccessDenied, 0, 3));
 }
 
+#[tokio::test(start_paused = true)]
+async fn test_rebalance_listing_retry_waits_for_scheduled_entries() {
+    let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let task_registered = Arc::new(tokio::sync::Notify::new());
+    let release_task = Arc::new(tokio::sync::Notify::new());
+    let task_finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let callback: crate::core::pools::ListCallback = Arc::new({
+        let entry_tasks = entry_tasks.clone();
+        let task_registered = task_registered.clone();
+        let release_task = release_task.clone();
+        let task_finished = task_finished.clone();
+        move |_| {
+            let entry_tasks = entry_tasks.clone();
+            let task_registered = task_registered.clone();
+            let release_task = release_task.clone();
+            let task_finished = task_finished.clone();
+            Box::pin(async move {
+                let task = tokio::spawn(async move {
+                    release_task.notified().await;
+                    task_finished.store(true, Ordering::SeqCst);
+                    Ok(RebalanceEntryOutcome::Completed)
+                });
+                entry_tasks.lock().await.push(task);
+                task_registered.notify_one();
+            })
+        }
+    });
+    let attempts = Arc::new(AtomicUsize::new(0));
+    let runner = tokio::spawn(run_rebalance_listing_with_retry(
+        CancellationToken::new(),
+        "bucket-a".to_string(),
+        callback,
+        0,
+        3,
+        entry_tasks,
+        {
+            let attempts = attempts.clone();
+            let task_finished = task_finished.clone();
+            move |cb| {
+                let attempt = attempts.fetch_add(1, Ordering::SeqCst);
+                let task_finished = task_finished.clone();
+                async move {
+                    if attempt == 0 {
+                        cb(MetaCacheEntry::default()).await;
+                        return Err(Error::SlowDown);
+                    }
+                    assert!(task_finished.load(Ordering::SeqCst), "retry must wait for scheduled entries");
+                    Ok(())
+                }
+            }
+        },
+    ));
+
+    task_registered.notified().await;
+    tokio::time::advance(Duration::from_secs(1)).await;
+    tokio::task::yield_now().await;
+    assert_eq!(attempts.load(Ordering::SeqCst), 1, "retry must not overlap the scheduled entry task");
+    release_task.notify_one();
+
+    runner
+        .await
+        .expect("listing retry task should join")
+        .expect("listing retry should complete after the scheduled entry");
+    assert_eq!(attempts.load(Ordering::SeqCst), 2);
+}
+
+#[tokio::test(start_paused = true)]
+async fn test_rebalance_listing_retry_propagates_scheduled_entry_failure() {
+    let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+    let callback: crate::core::pools::ListCallback = Arc::new({
+        let entry_tasks = entry_tasks.clone();
+        move |_| {
+            let entry_tasks = entry_tasks.clone();
+            Box::pin(async move {
+                entry_tasks
+                    .lock()
+                    .await
+                    .push(tokio::spawn(async { Err(Error::other("scheduled entry failed")) }));
+            })
+        }
+    });
+    let attempts = Arc::new(AtomicUsize::new(0));
+
+    let err = run_rebalance_listing_with_retry(CancellationToken::new(), "bucket-a".to_string(), callback, 0, 3, entry_tasks, {
+        let attempts = attempts.clone();
+        move |cb| {
+            let attempt = attempts.fetch_add(1, Ordering::SeqCst);
+            async move {
+                if attempt == 0 {
+                    cb(MetaCacheEntry::default()).await;
+                    return Err(Error::SlowDown);
+                }
+                panic!("entry failure must stop listing retries")
+            }
+        }
+    })
+    .await
+    .expect_err("scheduled entry failure must be returned before retrying the listing");
+
+    assert!(err.to_string().contains("scheduled entry failed"));
+    assert_eq!(attempts.load(Ordering::SeqCst), 1);
+}
+
 #[test]
 fn test_parse_rebalance_max_attempts_uses_positive_override_or_default() {
     assert_eq!(parse_rebalance_max_attempts(Some("5")), 5);
@@ -2454,6 +2597,7 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
         start_gate: tokio::sync::Mutex::new(()),
         pool_meta_save_gate: tokio::sync::Mutex::new(()),
         ctx: crate::runtime::instance::bootstrap_ctx(),
+        bucket_fence_registry: std::sync::Arc::default(),
     });
 
     let err = store

crates/ecstore/src/services/rebalance/runtime.rs

@@ -10,12 +10,14 @@ use super::worker::{
     resolve_rebalance_terminal_error, send_rebalance_done_signal,
 };
 use super::{
-    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, RebalSaveOpt, RebalStatus,
+    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
+    REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
     RebalanceBucketOutcome,
 };
 use crate::error::{Error, Result};
 use crate::runtime::sources as runtime_sources;
 use crate::store::ECStore;
+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::sync::Arc;
 use time::OffsetDateTime;
@@ -23,6 +25,20 @@ use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 
+pub(super) fn should_fail_repeated_rebalance_bucket_defer(
+    deferred_buckets: &mut HashSet<String>,
+    bucket: &str,
+    source_cleanup_deferred: bool,
+) -> bool {
+    !source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
+}
+
+pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
+    let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
+    *attempts = attempts.saturating_add(1);
+    *attempts
+}
+
 impl ECStore {
     #[tracing::instrument(skip_all)]
     pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
@@ -298,6 +314,7 @@ impl ECStore {
         );
         let mut final_result: Result<()> = Ok(());
         let mut deferred_buckets = HashSet::new();
+        let mut source_cleanup_deferred_attempts = HashMap::new();
 
         loop {
             if rx.is_cancelled() {
@@ -375,7 +392,8 @@ impl ECStore {
                 };
 
                 if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
-                    if !deferred_buckets.insert(bucket.clone()) {
+                    let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
+                    if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
                         let err = Error::other(format!(
                             "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
                         ));
@@ -396,6 +414,11 @@ impl ECStore {
                         break;
                     }
 
+                    let source_cleanup_attempt = if source_cleanup_deferred {
+                        source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
+                    } else {
+                        0
+                    };
                     warn!(
                         event = EVENT_REBALANCE_BUCKET,
                         component = LOG_COMPONENT_ECSTORE,
@@ -406,7 +429,10 @@ impl ECStore {
                         error = %last_error,
                         "Deferred rebalance bucket after transient object failures"
                     );
-                    if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
+                    if let Err(err) = self
+                        .defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone())
+                        .await
+                    {
                         error!(
                             event = EVENT_REBALANCE_BUCKET,
                             component = LOG_COMPONENT_ECSTORE,
@@ -423,6 +449,38 @@ impl ECStore {
                         ));
                         break;
                     }
+                    if source_cleanup_deferred {
+                        if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
+                            let err = Error::other(format!(
+                                "rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
+                                super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
+                            ));
+                            warn!(
+                                event = EVENT_REBALANCE_BUCKET,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                pool_index,
+                                bucket = %bucket,
+                                state = "source_cleanup_defer_limit",
+                                error = ?err,
+                                "Rebalance bucket failed after repeated source cleanup conflicts"
+                            );
+                            final_result = Err(resolve_rebalance_terminal_error(
+                                err.clone(),
+                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                            ));
+                            break;
+                        }
+                        if let Err(err) =
+                            super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
+                        {
+                            final_result = Err(resolve_rebalance_terminal_error(
+                                err.clone(),
+                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
+                            ));
+                            break;
+                        }
+                    }
                     continue;
                 }
 
@@ -435,6 +493,7 @@ impl ECStore {
                     state = "completed",
                     "Completed rebalance bucket"
                 );
+                source_cleanup_deferred_attempts.remove(&bucket);
                 if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
                     error!(
                         event = EVENT_REBALANCE_BUCKET,

crates/ecstore/src/services/rebalance/worker.rs

@@ -2,10 +2,12 @@ use super::migration::MigrationVersionResult;
 use super::{
     DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
     REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
-    REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
+    REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
+    RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
 };
 use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
 use crate::core::pools::ListCallback;
+use crate::data_movement::SourceCleanupError;
 use crate::disk::error::DiskError;
 use crate::error::{
     Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down,
@@ -36,6 +38,12 @@ pub(super) fn resolve_rebalance_worker_result<T>(
 
 pub(super) type RebalanceEntryTask = tokio::task::JoinHandle<Result<RebalanceEntryOutcome>>;
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum RebalanceEntryCleanupResult {
+    Completed { warning: Option<String> },
+    Deferred { last_error: String },
+}
+
 pub(super) async fn wait_rebalance_entry_tasks(
     set_idx: usize,
     tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
@@ -145,14 +153,23 @@ where
 }
 
 pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
-    result: Result<crate::object_api::ObjectInfo>,
+    result: std::result::Result<crate::object_api::ObjectInfo, SourceCleanupError>,
     bucket: &str,
     object_name: &str,
-) -> Result<Option<String>> {
+) -> RebalanceEntryCleanupResult {
     match result {
-        Ok(_) => Ok(None),
-        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
-        Err(err) => Ok(Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))),
+        Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
+        Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+            RebalanceEntryCleanupResult::Completed { warning: None }
+        }
+        Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
+            last_error: format!(
+                "{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
+            ),
+        },
+        Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
+            warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
+        },
     }
 }
 
@@ -399,19 +416,24 @@ pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) -
     })
 }
 
-pub(super) async fn run_rebalance_listing_with_retry(
-    set: Arc<SetDisks>,
+pub(super) async fn run_rebalance_listing_with_retry<List, ListFuture>(
     rx: CancellationToken,
     bucket: String,
     cb: ListCallback,
     set_idx: usize,
     max_attempts: usize,
-) -> Result<()> {
+    entry_tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
+    mut list: List,
+) -> Result<()>
+where
+    List: FnMut(ListCallback) -> ListFuture,
+    ListFuture: std::future::Future<Output = Result<()>>,
+{
     let max_attempts = max_attempts.max(1);
     let mut last_error = None;
 
     for attempt in 0..max_attempts {
-        match set.list_objects_to_rebalance(rx.clone(), bucket.clone(), cb.clone()).await {
+        match list(cb.clone()).await {
             Ok(()) => return Ok(()),
             Err(err) if should_retry_rebalance_listing(&err, attempt, max_attempts) => {
                 let next_attempt = attempt + 2;
@@ -426,6 +448,8 @@ pub(super) async fn run_rebalance_listing_with_retry(
                     delay
                 );
                 last_error = Some(err);
+                // The full retry re-evaluates deferred entries; only task failures block the next attempt.
+                let _ = wait_rebalance_entry_tasks(set_idx, entry_tasks.clone()).await?;
                 wait_rebalance_listing_retry(&rx, delay).await?;
                 info!(
                     "rebalance listing retrying bucket {} set {} attempt {}/{}",

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -49,7 +49,7 @@ use crate::diagnostics::get::{
 use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
 use crate::disk::{
     DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
-    PartTransactionAction, part_transaction_path,
+    PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
 };
 use crate::erasure::coding::BitrotReader;
 use crate::io_support::bitrot::ShardReader;
@@ -2950,6 +2950,68 @@ impl SetDisks {
             return Err(ret_err);
         }
 
+        // The write is authoritatively committed, so the per-disk rollback
+        // backup (`object/<rollback_dir>/xl.meta.bkp`) is dead weight now.
+        // When the rollback dir doubles as the real dereferenced data dir it
+        // is reclaimed wholesale by `commit_rename_data_dir`; a rollback dir
+        // reported separately (an overwrite of an inline version, whose dir is
+        // synthetic) is excluded from that recursive reclamation for safety
+        // (#5703) and must be reclaimed here instead — otherwise every inline
+        // overwrite strands a backup file that keeps the object dir non-empty
+        // and makes a later DeleteBucket fail with BucketNotEmpty forever.
+        // Delete exactly the backup file, never the directory tree: the
+        // synthetic UUID is a fixed, publicly-known constant for unversioned
+        // objects, so `object/<rollback_dir>` can simultaneously be a
+        // legitimate child key's directory — recursively deleting it would
+        // reopen the authorization bypass #5703 closed. The non-recursive
+        // delete removes the directory only when the backup was its sole
+        // content. Best-effort space reclamation — like
+        // `commit_rename_data_dir`, this must never negate the already-durable
+        // ACK.
+        let mut backup_reclaims = Vec::new();
+        for (idx, disk) in disks.iter().enumerate() {
+            if errs[idx].is_some() {
+                continue;
+            }
+            let Some(rollback_dir) = data_dirs[idx] else {
+                continue;
+            };
+            if cleanup_data_dirs[idx] == Some(rollback_dir) {
+                continue;
+            }
+            let Some(disk) = disk.clone() else {
+                continue;
+            };
+            let dst_bucket = dst_bucket.clone();
+            let dst_object = dst_object.clone();
+            backup_reclaims.push(tokio::spawn(async move {
+                let backup_path = format!("{dst_object}/{rollback_dir}/{STORAGE_FORMAT_FILE_BACKUP}");
+                disk.delete(&dst_bucket, &backup_path, DeleteOptions::default()).await
+            }));
+        }
+        for result in join_all(backup_reclaims).await {
+            match result {
+                Ok(Ok(())) => {}
+                Ok(Err(DiskError::FileNotFound | DiskError::VolumeNotFound)) => {}
+                Ok(Err(err)) => {
+                    warn!(
+                        dst_bucket = %dst_bucket,
+                        dst_object = %dst_object,
+                        error = %err,
+                        "rollback backup reclamation failed after committed rename"
+                    );
+                }
+                Err(join_err) => {
+                    warn!(
+                        dst_bucket = %dst_bucket,
+                        dst_object = %dst_object,
+                        error = %join_err,
+                        "rollback backup reclamation task failed after committed rename"
+                    );
+                }
+            }
+        }
+
         let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
         let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
         let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
@@ -5203,6 +5265,84 @@ mod tests {
         assert!(stored.is_canonical_delete_marker());
     }
 
+    /// Overwriting an inline version with a non-inline one stages the old
+    /// xl.meta as `<object>/<synthetic-rollback-dir>/xl.meta.bkp` for the
+    /// quorum-failure undo. After a successful quorum commit that dir must be
+    /// reclaimed — leftover residue keeps DeleteBucket failing with
+    /// BucketNotEmpty long after the object itself is deleted.
+    #[tokio::test]
+    async fn rename_data_reclaims_synthetic_inline_rollback_dir_after_commit() {
+        let bucket = "rename-inline-rollback-bucket";
+        let object = "object";
+        let (dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
+        let online_disk = online_disks.pop().expect("one test disk slot should be present");
+        let disk = online_disk.as_ref().expect("test disk should be online");
+        match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
+            Ok(()) | Err(DiskError::VolumeExists) => {}
+            Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
+        }
+
+        let disk_root = dirs[0].path();
+
+        // Commit an inline version (data carried in xl.meta, no data dir).
+        let mut inline_fi = metadata_test_fileinfo(object);
+        inline_fi.data = Some(Bytes::from_static(b"inline-body"));
+        inline_fi.mod_time = Some(OffsetDateTime::now_utc());
+        std::fs::create_dir_all(disk_root.join(RUSTFS_META_TMP_BUCKET).join("tmp-inline"))
+            .expect("inline staging dir should be created");
+        SetDisks::rename_data(
+            std::slice::from_ref(&online_disk),
+            RUSTFS_META_TMP_BUCKET,
+            "tmp-inline",
+            std::slice::from_ref(&inline_fi),
+            bucket,
+            object,
+            1,
+        )
+        .await
+        .expect("inline version should commit");
+
+        // Overwrite the same (nil) version with a non-inline one.
+        let new_data_dir = Uuid::new_v4();
+        let mut streaming_fi = metadata_test_fileinfo(object);
+        streaming_fi.data_dir = Some(new_data_dir);
+        streaming_fi.mod_time = Some(OffsetDateTime::now_utc());
+        let staged_data_dir = disk_root
+            .join(RUSTFS_META_TMP_BUCKET)
+            .join("tmp-streaming")
+            .join(new_data_dir.to_string());
+        std::fs::create_dir_all(&staged_data_dir).expect("streaming staging dir should be created");
+        std::fs::write(staged_data_dir.join("part.1"), b"streamed-body").expect("staged part should be written");
+        SetDisks::rename_data(
+            std::slice::from_ref(&online_disk),
+            RUSTFS_META_TMP_BUCKET,
+            "tmp-streaming",
+            std::slice::from_ref(&streaming_fi),
+            bucket,
+            object,
+            1,
+        )
+        .await
+        .expect("non-inline overwrite should commit");
+
+        let mut leftovers: Vec<String> = std::fs::read_dir(disk_root.join(bucket).join(object))
+            .expect("committed object dir should be readable")
+            .map(|entry| {
+                entry
+                    .expect("object dir entry should be readable")
+                    .file_name()
+                    .to_string_lossy()
+                    .into_owned()
+            })
+            .collect();
+        leftovers.sort();
+        assert_eq!(
+            leftovers,
+            vec![new_data_dir.to_string(), STORAGE_FORMAT_FILE.to_string()],
+            "only the committed data dir and xl.meta may remain — synthetic rollback residue breaks DeleteBucket"
+        );
+    }
+
     #[tokio::test]
     async fn rename_delete_marker_quorum_failure_restores_existing_metadata() {
         let bucket = "rename-marker-quorum-bucket";

crates/ecstore/src/set_disk/metadata.rs

@@ -250,14 +250,26 @@ impl SetDisks {
                 continue;
             }
 
+            // A parity count outside [0, total_shards] cannot describe a real
+            // layout on this set: it comes from corrupt or foreign metadata
+            // (e.g. stray leftovers, rustfs#5801). Treat the entry as invalid
+            // instead of clamping to i32::MAX, which would poison
+            // `common_parity`'s occurrence counting.
+            let erasure_parity = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(-1);
+            let erasure_parity = if (0..=total_shards_i32).contains(&erasure_parity) {
+                erasure_parity
+            } else {
+                -1
+            };
             if metadata.is_canonical_delete_marker() || metadata.size == 0 {
                 parities[index] = half;
+            } else if erasure_parity < 0 {
+                parities[index] = -1;
             } else if metadata.transition_status == TRANSITION_COMPLETE {
                 let majority_metadata_parity = total_shards_i32 - (half + 1);
-                let erasure_parity = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
                 parities[index] = majority_metadata_parity.max(erasure_parity);
             } else {
-                parities[index] = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
+                parities[index] = erasure_parity;
             }
         }
         parities
@@ -294,6 +306,19 @@ impl SetDisks {
         let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
 
         if parity_blocks < 0 {
+            // No parity value reached read quorum. Distinguish two cases:
+            // enough disks answered with valid-looking metadata that simply
+            // cannot be reconciled (corrupt/foreign entries — retrying cannot
+            // help, and heal should see Corrupt, rustfs#5801) versus too few
+            // healthy answers (a genuine quorum condition where retry may
+            // succeed once disks recover).
+            let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
+            if healthy_replies >= expected_rquorum {
+                error!(
+                    "object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
+                );
+                return Err(DiskError::FileCorrupt);
+            }
             error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs);
             return Err(DiskError::ErasureReadQuorum);
         }
@@ -1136,7 +1161,9 @@ mod tests {
         let invalid = vec![FileInfo::default(); 4];
         let err = SetDisks::object_quorum_from_meta(&invalid, &vec![None; 4], 2)
             .expect_err("invalid metadata without a common parity must fail closed");
-        assert_eq!(err, DiskError::ErasureReadQuorum);
+        // A full set of healthy replies whose metadata cannot be reconciled is
+        // corrupt (heal-actionable), not a retryable quorum outage (rustfs#5801).
+        assert_eq!(err, DiskError::FileCorrupt);
     }
 
     #[test]
@@ -1468,4 +1495,55 @@ mod tests {
             "compatible prefixes carrying the same mapping must share one identity"
         );
     }
+
+    /// rustfs#5801: parity counts outside [0, total_shards] come from corrupt
+    /// or foreign metadata and must be treated as invalid entries instead of
+    /// clamped values that poison `common_parity`'s occurrence counting.
+    #[test]
+    fn out_of_range_parity_is_treated_as_invalid_entry() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
+        for fi in &mut metas {
+            fi.erasure.parity_blocks = usize::MAX;
+        }
+        let errs: Vec<Option<DiskError>> = vec![None; 4];
+
+        let parities = SetDisks::list_object_parities(&metas, &errs);
+        assert_eq!(parities, vec![-1; 4], "garbage parity must not survive as a candidate");
+    }
+
+    /// rustfs#5801: when a read quorum of healthy disks answers but their
+    /// parity values are irreconcilable, the object metadata is corrupt —
+    /// return `FileCorrupt` (heal-actionable, non-retryable) instead of the
+    /// retryable-looking `ErasureReadQuorum`.
+    #[test]
+    fn irreconcilable_parity_with_healthy_quorum_is_file_corrupt() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
+        for fi in &mut metas {
+            fi.erasure.parity_blocks = usize::MAX;
+        }
+        let errs: Vec<Option<DiskError>> = vec![None; 4];
+
+        let err = SetDisks::object_quorum_from_meta(&metas, &errs, 2).expect_err("garbage parity cannot form a quorum");
+        assert_eq!(err, DiskError::FileCorrupt);
+    }
+
+    /// Too few healthy replies remains a genuine quorum condition where a
+    /// retry may succeed once disks recover.
+    #[test]
+    fn insufficient_healthy_replies_stays_erasure_read_quorum() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
+        metas[0].erasure.parity_blocks = usize::MAX;
+        let errs: Vec<Option<DiskError>> = vec![
+            None,
+            Some(DiskError::DiskNotFound),
+            Some(DiskError::DiskNotFound),
+            Some(DiskError::DiskNotFound),
+        ];
+
+        let err = SetDisks::object_quorum_from_meta(&metas, &errs, 2).expect_err("one healthy reply is below quorum");
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
 }

crates/ecstore/src/set_disk/mod.rs

@@ -3964,9 +3964,15 @@ async fn disks_with_all_parts(
         };
 
         if corrupted {
-            info!(
-                "disks_with_all_partsv2: metadata is corrupted, object_name={}, index: {index}",
-                object_name
+            debug!(
+                event = EVENT_SET_DISK_HEAL,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                bucket,
+                object = %object_name,
+                disk_index = index,
+                state = "metadata_corrupt",
+                "Set disk object metadata is corrupt"
             );
             meta_errs[index] = Some(DiskError::FileCorrupt);
             parts_metadata[index] = FileInfo::default();
@@ -3977,9 +3983,15 @@ async fn disks_with_all_parts(
 
         if erasure_distribution_reliable {
             if !file_info_is_valid_for_metadata(meta) {
-                info!(
-                    "disks_with_all_partsv2: metadata is not valid, object_name={}, index: {index}",
-                    object_name
+                debug!(
+                    event = EVENT_SET_DISK_HEAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object = %object_name,
+                    disk_index = index,
+                    state = "metadata_invalid",
+                    "Set disk object metadata is invalid"
                 );
                 parts_metadata[index] = FileInfo::default();
                 meta_errs[index] = Some(DiskError::FileCorrupt);
@@ -3991,9 +4003,15 @@ async fn disks_with_all_parts(
                 // Erasure distribution is not the same as onlineDisks
                 // attempt a fix if possible, assuming other entries
                 // might have the right erasure distribution.
-                info!(
-                    "disks_with_all_partsv2: erasure distribution is not the same as onlineDisks, object_name={}, index: {index}",
-                    object_name
+                debug!(
+                    event = EVENT_SET_DISK_HEAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object = %object_name,
+                    disk_index = index,
+                    state = "erasure_distribution_mismatch",
+                    "Set disk erasure distribution mismatched online disks"
                 );
                 parts_metadata[index] = FileInfo::default();
                 meta_errs[index] = Some(DiskError::FileCorrupt);
@@ -4066,6 +4084,7 @@ async fn disks_with_all_parts(
                         event = EVENT_SET_DISK_HEAL,
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_SET_DISK,
+                        bucket,
                         object = %object_name,
                         disk_index = index,
                         state = "verify_failed",
@@ -4085,6 +4104,7 @@ async fn disks_with_all_parts(
                         event = EVENT_SET_DISK_HEAL,
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_SET_DISK,
+                        bucket,
                         object = %object_name,
                         disk_index = index,
                         state = "check_parts_failed",
@@ -4153,9 +4173,13 @@ pub fn should_heal_object_on_disk(
     }
 
     if !meta.equals(latest_meta) {
-        warn!(
-            "should_heal_object_on_disk: metadata is outdated, object_name={}, meta: {:?}, latest_meta: {:?}",
-            meta.name, meta, latest_meta
+        debug!(
+            event = EVENT_SET_DISK_HEAL,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_SET_DISK,
+            object = %meta.name,
+            state = "metadata_outdated",
+            "Set disk object metadata is outdated"
         );
         return (true, true, Some(DiskError::OutdatedXLMeta));
     }
@@ -6285,6 +6309,136 @@ mod tests {
         assert_eq!(read_back.size, 9, "HEAD must observe the new version, not stale metadata");
     }
 
+    // Regression for the inline-overwrite rollback backup leak: #5703 stopped
+    // reporting the synthetic rollback dir for recursive post-commit cleanup,
+    // which stranded `object/<synthetic>/xl.meta.bkp` after every inline
+    // overwrite. The object dir then never emptied, so the s3-tests teardown
+    // sequence (delete object, delete bucket) failed with BucketNotEmpty
+    // forever. After a committed overwrite the backup must be reclaimed and a
+    // subsequent delete must leave nothing behind.
+    #[tokio::test]
+    async fn inline_overwrite_reclaims_synthetic_rollback_backup() {
+        let set_disks = make_local_bucket_test_set_disks().await;
+        let bucket = "bucket-inline-rollback-leak";
+        let object = "obj";
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        for body in [b"hello".to_vec(), b"goodbye".to_vec()] {
+            let mut reader = PutObjReader::from_vec(body);
+            set_disks
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..ObjectOptions::default()
+                    },
+                )
+                .await
+                .expect("inline write should succeed");
+        }
+
+        // The committed overwrite must leave only xl.meta in the object dir on
+        // every disk; a stranded rollback dir keeps the bucket undeletable.
+        for endpoint in &set_disks.set_endpoints {
+            let object_dir = std::path::PathBuf::from(endpoint.get_file_path()).join(bucket).join(object);
+            let mut entries: Vec<String> = std::fs::read_dir(&object_dir)
+                .expect("object dir should exist")
+                .map(|entry| entry.expect("entry should read").file_name().to_string_lossy().into_owned())
+                .collect();
+            entries.sort();
+            assert_eq!(
+                entries,
+                vec![STORAGE_FORMAT_FILE.to_string()],
+                "only xl.meta may remain after an inline overwrite in {object_dir:?}"
+            );
+        }
+
+        // With only xl.meta left, the s3-tests teardown (delete object, delete
+        // bucket) empties the dir; the delete paths themselves are covered by
+        // their own tests. This harness has no bucket metadata sys, so the
+        // full delete_object flow cannot run here.
+    }
+
+    // #5703's security property must survive the backup reclamation: the
+    // synthetic rollback dir of key K maps to the directory `K/<uuid>`, which
+    // can simultaneously be a legitimate child key. Reclaiming the backup must
+    // remove exactly the backup file — never the child key's metadata.
+    #[tokio::test]
+    async fn inline_overwrite_backup_reclaim_spares_child_key_dir() {
+        let set_disks = make_local_bucket_test_set_disks().await;
+        let bucket = "bucket-inline-rollback-child";
+        let object = "obj";
+        let synthetic = crate::disk::local::inline_metadata_rollback_dir(Uuid::nil(), &FileMeta::new());
+        let child_object = format!("{object}/{synthetic}");
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let mut reader = PutObjReader::from_vec(b"child".to_vec());
+        set_disks
+            .put_object(
+                bucket,
+                &child_object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..ObjectOptions::default()
+                },
+            )
+            .await
+            .expect("child write should succeed");
+
+        // Create then overwrite the parent key: the overwrite writes its
+        // rollback backup into the child's directory and must afterwards
+        // reclaim only that file.
+        for body in [b"first".to_vec(), b"second".to_vec()] {
+            let mut reader = PutObjReader::from_vec(body);
+            set_disks
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..ObjectOptions::default()
+                    },
+                )
+                .await
+                .expect("parent write should succeed");
+        }
+
+        let child_info = set_disks
+            .get_object_info(bucket, &child_object, &ObjectOptions::default())
+            .await
+            .expect("child key must survive the parent's rollback backup reclamation");
+        assert_eq!(child_info.size, 5, "child key content must be untouched");
+
+        for endpoint in &set_disks.set_endpoints {
+            let child_dir = std::path::PathBuf::from(endpoint.get_file_path())
+                .join(bucket)
+                .join(object)
+                .join(synthetic.to_string());
+            let mut entries: Vec<String> = std::fs::read_dir(&child_dir)
+                .expect("child object dir should exist")
+                .map(|entry| entry.expect("entry should read").file_name().to_string_lossy().into_owned())
+                .collect();
+            entries.sort();
+            assert_eq!(
+                entries,
+                vec![STORAGE_FORMAT_FILE.to_string()],
+                "the child dir must keep its xl.meta and lose only the stray backup in {child_dir:?}"
+            );
+        }
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn test_acquire_dist_delete_object_locks_batch_succeeds_with_two_healthy_lockers() {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -15,6 +15,7 @@
 use super::super::*;
 use crate::io_support::bitrot::object_mmap_read_enabled;
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
+use tracing::trace;
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_HEAL: &str = "heal";
@@ -71,7 +72,9 @@ fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> boo
         .expect("heal rename failure registry should not poison");
     if let Some(position) = failures
         .iter()
-        .position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
+        .position(|(registered_bucket, registered_object, registered_index)| {
+            registered_bucket == bucket && registered_object == object && *registered_index == disk_index
+        })
     {
         failures.swap_remove(position);
         true
@@ -85,6 +88,67 @@ fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) ->
     false
 }
 
+#[cfg(test)]
+static HEAL_WRITER_FAILURES: std::sync::Mutex<Vec<(String, String, usize, DiskError)>> = std::sync::Mutex::new(Vec::new());
+
+#[cfg(test)]
+struct HealWriterFailureScope {
+    bucket: String,
+    object: String,
+}
+
+#[cfg(test)]
+impl HealWriterFailureScope {
+    fn install(bucket: &str, object: &str, disk_indexes: &[usize], error: DiskError) -> Self {
+        let mut failures = HEAL_WRITER_FAILURES
+            .lock()
+            .expect("heal writer failure registry should not poison");
+        assert!(
+            !failures.iter().any(|(registered_bucket, registered_object, _, _)| {
+                registered_bucket == bucket && registered_object == object
+            }),
+            "heal writer failures must be installed once per object"
+        );
+        failures.extend(
+            disk_indexes
+                .iter()
+                .map(|index| (bucket.to_string(), object.to_string(), *index, error.clone())),
+        );
+        Self {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+        }
+    }
+}
+
+#[cfg(test)]
+impl Drop for HealWriterFailureScope {
+    fn drop(&mut self) {
+        HEAL_WRITER_FAILURES
+            .lock()
+            .expect("heal writer failure registry should not poison")
+            .retain(|(bucket, object, _, _)| bucket != &self.bucket || object != &self.object);
+    }
+}
+
+#[cfg(test)]
+fn injected_heal_writer_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
+    let mut failures = HEAL_WRITER_FAILURES
+        .lock()
+        .expect("heal writer failure registry should not poison");
+    failures
+        .iter()
+        .position(|(registered_bucket, registered_object, registered_index, _)| {
+            registered_bucket == bucket && registered_object == object && *registered_index == disk_index
+        })
+        .map(|position| failures.swap_remove(position).3)
+}
+
+#[cfg(not(test))]
+fn injected_heal_writer_error(_bucket: &str, _object: &str, _disk_index: usize) -> Option<DiskError> {
+    None
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 struct PartFailureSummary {
     part_number: usize,
@@ -233,8 +297,41 @@ fn first_unhealthy_part_summary(
         .map(|(_, summary)| summary)
 }
 
+fn heal_writer_error_summary(error: &DiskError) -> String {
+    match error {
+        DiskError::Io(io_error) => format!("io::{:?}", io_error.kind()),
+        _ => error.to_string(),
+    }
+}
+
+fn warn_heal_writer_failures(
+    bucket: &str,
+    object: &str,
+    version_id: &str,
+    writer_failure_count: usize,
+    result: &'static str,
+    first_failure: &(usize, usize, String),
+) {
+    let (first_part_number, first_disk_index, first_error) = first_failure;
+    warn!(
+        event = EVENT_SET_DISK_HEAL,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_SET_DISK,
+        bucket,
+        object,
+        version_id,
+        writer_failure_count,
+        first_part_number,
+        first_disk_index,
+        error = %first_error,
+        result,
+        state = "writer_unavailable",
+        "Set disk object heal writer failures"
+    );
+}
+
 impl SetDisks {
-    #[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
+    #[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
     pub(in crate::set_disk) async fn heal_object(
         &self,
         bucket: &str,
@@ -254,7 +351,16 @@ impl SetDisks {
         opts: &HealOpts,
         allow_explicit_version_regen: bool,
     ) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
-        info!(?opts, "Starting heal_object");
+        trace!(
+            event = EVENT_SET_DISK_HEAL,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_SET_DISK,
+            scan_mode = %opts.scan_mode.as_str(),
+            dry_run = opts.dry_run,
+            remove = opts.remove,
+            state = "started",
+            "Set disk object heal started"
+        );
 
         let disks = self.get_disks_internal().await;
 
@@ -290,16 +396,26 @@ impl SetDisks {
         let (mut parts_metadata, errs) =
             Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
 
-        info!(
+        trace!(
+            event = EVENT_SET_DISK_HEAL,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_SET_DISK,
             parts_count = parts_metadata.len(),
-            bucket = bucket,
-            object = object,
-            version_id = version_id,
-            ?errs,
-            "File info read complete"
+            error_count = errs.iter().flatten().count(),
+            state = "metadata_read",
+            "Set disk object metadata read"
         );
         if DiskError::is_all_not_found(&errs) {
-            debug!(bucket, object, version_id, "heal_object skipped missing object");
+            debug!(
+                event = EVENT_SET_DISK_HEAL,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                bucket,
+                object,
+                version_id,
+                state = "missing_object_skipped",
+                "Set disk heal skipped missing object"
+            );
             let err = if !version_id.is_empty() {
                 DiskError::FileVersionNotFound
             } else {
@@ -313,7 +429,14 @@ impl SetDisks {
             ));
         }
 
-        info!(parts_count = parts_metadata.len(), "heal_object Initiating quorum check");
+        trace!(
+            event = EVENT_SET_DISK_HEAL,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_SET_DISK,
+            parts_count = parts_metadata.len(),
+            state = "quorum_check",
+            "Set disk object quorum check started"
+        );
         match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) {
             Ok((read_quorum, _)) => {
                 result.parity_blocks = result.disk_count - read_quorum as usize;
@@ -325,14 +448,35 @@ impl SetDisks {
                     (Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len)
                 };
 
-                info!(?parts_metadata, ?errs, ?read_quorum, ?disk_len, "heal_object List disks metadata");
-
-                info!(?online_disks, ?quorum_mod_time, ?quorum_etag, "heal_object List online disks");
+                trace!(
+                    event = EVENT_SET_DISK_HEAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    metadata_count = parts_metadata.len(),
+                    error_count = errs.iter().flatten().count(),
+                    read_quorum,
+                    disk_count = disk_len,
+                    online_disk_count = online_disks.iter().flatten().count(),
+                    state = "disk_metadata_resolved",
+                    "Set disk object metadata resolved"
+                );
 
                 let filter_by_etag = quorum_etag.is_some();
                 match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
                     Ok(latest_meta) => {
-                        info!("heal_object latest_meta: {:?}", latest_meta);
+                        trace!(
+                            event = EVENT_SET_DISK_HEAL,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_SET_DISK,
+                            deleted = latest_meta.deleted,
+                            remote = latest_meta.is_remote(),
+                            inline = latest_meta.inline_data(),
+                            part_count = latest_meta.parts.len(),
+                            data_shards = latest_meta.erasure.data_blocks,
+                            parity_shards = latest_meta.erasure.parity_blocks,
+                            state = "canonical_metadata_selected",
+                            "Set disk canonical object metadata selected"
+                        );
 
                         let (data_errs_by_disk, data_errs_by_part) = disks_with_all_parts(
                             &mut online_disks,
@@ -346,10 +490,14 @@ impl SetDisks {
                         )
                         .await?;
 
-                        info!(
-                            "disks_with_all_parts heal_object results: available_disks count={}, total_disks={}",
-                            online_disks.iter().filter(|d| d.is_some()).count(),
-                            online_disks.len()
+                        trace!(
+                            event = EVENT_SET_DISK_HEAL,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_SET_DISK,
+                            available_disk_count = online_disks.iter().flatten().count(),
+                            disk_count = online_disks.len(),
+                            state = "parts_checked",
+                            "Set disk object parts checked"
                         );
 
                         let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
@@ -388,7 +536,18 @@ impl SetDisks {
                                 if is_meta {
                                     meta_to_heal_count += 1;
                                 }
-                                debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]);
+                                debug!(
+                                    event = EVENT_SET_DISK_HEAL,
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                                    bucket,
+                                    object,
+                                    version_id,
+                                    disk_index = index,
+                                    endpoint = %self.set_endpoints[index],
+                                    state = "disk_marked_for_healing",
+                                    "Set disk marked for healing"
+                                );
                             }
 
                             let drive_state = match reason {
@@ -472,6 +631,8 @@ impl SetDisks {
                                     latest_meta.erasure.data_blocks
                                 );
                                 error!(
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_HEAL,
                                     bucket,
                                     object,
                                     version_id,
@@ -490,6 +651,8 @@ impl SetDisks {
                                     latest_meta.erasure.parity_blocks
                                 );
                                 error!(
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_HEAL,
                                     bucket,
                                     object,
                                     version_id,
@@ -545,6 +708,8 @@ impl SetDisks {
                                 }
                                 Err(err) => {
                                     error!(
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_HEAL,
                                         bucket,
                                         object,
                                         version_id,
@@ -558,11 +723,23 @@ impl SetDisks {
                         }
 
                         if !latest_meta.deleted && latest_meta.erasure.distribution.len() != online_disks.len() {
+                            let distribution_len = latest_meta.erasure.distribution.len();
+                            let disk_slot_count = online_disks.len();
                             let err_str = format!(
-                                "unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
-                                latest_meta.erasure.distribution, online_disks, bucket, object, version_id
+                                "unexpected file distribution length {distribution_len} for {disk_slot_count} disk slots; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
+                            );
+                            warn!(
+                                event = EVENT_SET_DISK_HEAL,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                                bucket,
+                                object,
+                                version_id,
+                                distribution_len,
+                                disk_slot_count,
+                                state = "invalid_distribution",
+                                "Set disk object heal refused due to invalid erasure distribution"
                             );
-                            warn!(err_str);
                             let err = DiskError::other(err_str);
                             return Ok((
                                 self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
@@ -572,11 +749,23 @@ impl SetDisks {
 
                         let latest_disks = Self::shuffle_disks(&online_disks, &latest_meta.erasure.distribution);
                         if !latest_meta.deleted && latest_meta.erasure.distribution.len() != out_dated_disks.len() {
+                            let distribution_len = latest_meta.erasure.distribution.len();
+                            let disk_slot_count = out_dated_disks.len();
                             let err_str = format!(
-                                "unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
-                                latest_meta.erasure.distribution, out_dated_disks, bucket, object, version_id
+                                "unexpected file distribution length {distribution_len} for {disk_slot_count} disk slots; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
+                            );
+                            warn!(
+                                event = EVENT_SET_DISK_HEAL,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                                bucket,
+                                object,
+                                version_id,
+                                distribution_len,
+                                disk_slot_count,
+                                state = "invalid_distribution",
+                                "Set disk object heal refused due to invalid erasure distribution"
                             );
-                            warn!(err_str);
                             let err = DiskError::other(err_str);
                             return Ok((
                                 self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
@@ -585,15 +774,23 @@ impl SetDisks {
                         }
 
                         if !latest_meta.deleted && latest_meta.erasure.distribution.len() != parts_metadata.len() {
+                            let distribution_len = latest_meta.erasure.distribution.len();
+                            let metadata_count = parts_metadata.len();
                             let err_str = format!(
-                                "unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
-                                latest_meta.erasure.distribution,
-                                parts_metadata.len(),
+                                "unexpected file distribution length {distribution_len} for {metadata_count} metadata entries; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
+                            );
+                            warn!(
+                                event = EVENT_SET_DISK_HEAL,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_SET_DISK,
                                 bucket,
                                 object,
-                                version_id
+                                version_id,
+                                distribution_len,
+                                metadata_count,
+                                state = "invalid_distribution",
+                                "Set disk object heal refused due to invalid erasure distribution"
                             );
-                            warn!(err_str);
                             let err = DiskError::other(err_str);
                             return Ok((
                                 self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
@@ -639,8 +836,12 @@ impl SetDisks {
                             None => {
                                 if !latest_meta.deleted && !latest_meta.is_remote() {
                                     error!(
-                                        "heal: latest metadata for {}/{} has no data_dir, cannot heal object data",
-                                        bucket, object
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_HEAL,
+                                        bucket,
+                                        object,
+                                        version_id,
+                                        "Heal object latest metadata has no data_dir, cannot heal object data"
                                     );
                                     return Err(DiskError::FileCorrupt);
                                 }
@@ -652,6 +853,9 @@ impl SetDisks {
 
                         if !latest_meta.deleted && !latest_meta.is_remote() {
                             let erasure_info = latest_meta.erasure.clone();
+                            let mut writer_failure_count = 0usize;
+                            let mut first_writer_failure = None;
+                            let mut writer_failure_warned = false;
 
                             for (part_index, part) in latest_meta.parts.iter().enumerate() {
                                 let till_offset = erasure.shard_file_offset(0, part.size, part.size);
@@ -666,9 +870,15 @@ impl SetDisks {
                                     let this_part_errs =
                                         Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution);
                                     if this_part_errs[index] != CHECK_PART_SUCCESS {
-                                        info!(
-                                            "reading part {}: index={}, part_errs={:?}, skipping",
-                                            part.number, index, this_part_errs[index]
+                                        trace!(
+                                            event = EVENT_SET_DISK_HEAL,
+                                            component = LOG_COMPONENT_ECSTORE,
+                                            subsystem = LOG_SUBSYSTEM_SET_DISK,
+                                            part_number = part.number,
+                                            disk_index = index,
+                                            part_status = this_part_errs[index],
+                                            state = "source_shard_skipped",
+                                            "Set disk source shard skipped"
                                         );
                                         readers.push(None);
                                         continue;
@@ -726,28 +936,33 @@ impl SetDisks {
                                 // create writers for all disk positions, but only for outdated disks
                                 for (index, disk_op) in out_dated_disks.iter().enumerate() {
                                     if let Some(outdated_disk) = disk_op {
-                                        let writer = match create_bitrot_writer(
-                                            is_inline_buffer,
-                                            Some(outdated_disk),
-                                            RUSTFS_META_TMP_BUCKET,
-                                            &path_join_buf(&[
-                                                &tmp_id.to_string(),
-                                                &dst_data_dir.to_string(),
-                                                &format!("part.{}", part.number),
-                                            ]),
-                                            erasure.shard_file_size(part.size as i64),
-                                            erasure.shard_size(),
-                                            HashAlgorithm::HighwayHash256S,
-                                        )
-                                        .await
+                                        let writer_result = if let Some(error) = injected_heal_writer_error(bucket, object, index)
                                         {
+                                            Err(error)
+                                        } else {
+                                            create_bitrot_writer(
+                                                is_inline_buffer,
+                                                Some(outdated_disk),
+                                                RUSTFS_META_TMP_BUCKET,
+                                                &path_join_buf(&[
+                                                    &tmp_id.to_string(),
+                                                    &dst_data_dir.to_string(),
+                                                    &format!("part.{}", part.number),
+                                                ]),
+                                                erasure.shard_file_size(part.size as i64),
+                                                erasure.shard_size(),
+                                                HashAlgorithm::HighwayHash256S,
+                                            )
+                                            .await
+                                        };
+                                        let writer = match writer_result {
                                             Ok(writer) => writer,
                                             Err(err) => {
-                                                info!(
-                                                    "create_bitrot_writer  disk {}, err {:?}, skipping operation",
-                                                    outdated_disk.to_string(),
-                                                    err
-                                                );
+                                                writer_failure_count += 1;
+                                                if first_writer_failure.is_none() {
+                                                    first_writer_failure =
+                                                        Some((part.number, index, heal_writer_error_summary(&err)));
+                                                }
                                                 writers.push(None);
                                                 continue;
                                             }
@@ -761,6 +976,20 @@ impl SetDisks {
                                 // Heal each part. erasure.Heal() will write the healed
                                 // part to .rustfs/tmp/uuid/ which needs to be renamed
                                 // later to the final location.
+                                if writer_failure_count > 0
+                                    && writers.iter().all(Option::is_none)
+                                    && let Some(first_failure) = first_writer_failure.as_ref()
+                                {
+                                    warn_heal_writer_failures(
+                                        bucket,
+                                        object,
+                                        version_id,
+                                        writer_failure_count,
+                                        "all_targets_unavailable",
+                                        first_failure,
+                                    );
+                                    writer_failure_warned = true;
+                                }
                                 if let Err(e) = erasure.heal(&mut writers, readers, part.size, &prefer).await {
                                     // Don't leak the partially-written healed shards in
                                     // .rustfs/tmp when heal fails midway (backlog#799 B20).
@@ -805,6 +1034,16 @@ impl SetDisks {
                                 }
 
                                 if disks_to_heal_count == 0 {
+                                    if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() {
+                                        warn_heal_writer_failures(
+                                            bucket,
+                                            object,
+                                            version_id,
+                                            writer_failure_count,
+                                            "all_targets_unavailable",
+                                            first_failure,
+                                        );
+                                    }
                                     // Clean up healed shards written to .rustfs/tmp before bailing (B20).
                                     let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
                                     return Ok((
@@ -815,6 +1054,17 @@ impl SetDisks {
                                     ));
                                 }
                             }
+
+                            if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() {
+                                warn_heal_writer_failures(
+                                    bucket,
+                                    object,
+                                    version_id,
+                                    writer_failure_count,
+                                    "partial_targets_unavailable",
+                                    first_failure,
+                                );
+                            }
                         }
                         // Rename from tmp location to the actual location.
                         // MinIO stops on the first RenameData error. RustFS intentionally
@@ -1075,6 +1325,8 @@ impl SetDisks {
                 Ok(()) => wrote += 1,
                 Err(error) => {
                     warn!(
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_HEAL,
                         bucket,
                         object,
                         disk_index = index,
@@ -1095,11 +1347,29 @@ impl SetDisks {
     async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
         match self.reclaim_orphan_data_dirs(bucket, object).await {
             Ok(removed) if removed > 0 => {
-                info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
+                debug!(
+                    event = EVENT_SET_DISK_HEAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object,
+                    removed,
+                    state = "orphan_data_reclaimed",
+                    "Set disk orphaned data reclaimed"
+                );
             }
             Ok(_) => {}
             Err(e) => {
-                warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
+                warn!(
+                    event = EVENT_SET_DISK_HEAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object,
+                    error = %e,
+                    state = "orphan_data_reclaim_failed",
+                    "Set disk orphan data-dir reclaim failed"
+                );
             }
         }
     }
@@ -1338,7 +1608,7 @@ impl SetDisks {
         Ok((result, None))
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
     pub(in crate::set_disk) async fn heal_object_dir(
         &self,
         bucket: &str,
@@ -1503,7 +1773,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
         };
 
         if count_errs(&errs, &DiskError::UnformattedDisk) == 0 {
-            info!("set disk formats success, NoHealRequired, errs: {:?}", errs);
+            debug!(
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_HEAL,
+                error_count = errs.iter().flatten().count(),
+                result = "no_heal_required",
+                "set disk formats success"
+            );
             return Ok((result, Some(StorageError::NoHealRequired)));
         }
 
@@ -1532,7 +1808,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
         Ok(result)
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -1636,8 +1912,8 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
 
 #[cfg(test)]
 mod heal_result_report_tests {
-    use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks};
-    use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
+    use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks, heal_writer_error_summary};
+    use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope, HealWriterFailureScope};
     use crate::disk::endpoint::Endpoint;
     use crate::disk::error::DiskError;
     use crate::disk::format::FormatV3;
@@ -1654,12 +1930,166 @@ mod heal_result_report_tests {
     };
     use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
     use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
-    use std::sync::Arc;
+    use std::sync::{Arc, Mutex};
     use tempfile::TempDir;
     use time::OffsetDateTime;
     use tokio::sync::RwLock;
+    use tracing_subscriber::fmt::MakeWriter;
     use uuid::Uuid;
 
+    #[derive(Clone, Default)]
+    struct CapturedLogs {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    struct CapturedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl CapturedLogs {
+        fn contents(&self) -> String {
+            let buffer = self
+                .buffer
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .clone();
+            String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
+        }
+    }
+
+    impl std::io::Write for CapturedLogWriter {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            self.buffer
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for CapturedLogs {
+        type Writer = CapturedLogWriter;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            CapturedLogWriter {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }
+
+    #[test]
+    fn heal_writer_error_summary_redacts_io_message() {
+        let error = DiskError::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "/sensitive/storage/path"));
+
+        let summary = heal_writer_error_summary(&error);
+
+        assert_eq!(summary, "io::PermissionDenied");
+        assert!(!summary.contains("sensitive"));
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    #[serial_test::serial]
+    async fn heal_writer_failures_emit_one_aggregate_warning_per_object() {
+        for (case, failed_target_count, expected_result, expect_error) in [
+            ("partial", 1usize, "partial_targets_unavailable", false),
+            ("all", 2usize, "all_targets_unavailable", true),
+        ] {
+            let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+            let bucket = format!("heal-writer-{case}");
+            let object = "object.bin";
+            for disk in &disks {
+                disk.make_volume(&bucket).await.expect("bucket volume should be created");
+            }
+
+            let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
+            set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
+                .await
+                .expect("source object should be written");
+            let source = disks[2]
+                .read_version("", &bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("source metadata should be readable");
+            let data_dir = source.data_dir.expect("non-inline source should have a data directory");
+            let mut target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
+            target_slots.sort_unstable();
+
+            for index in [0, 1] {
+                tokio::fs::remove_file(
+                    temp_dirs[index]
+                        .path()
+                        .join(&bucket)
+                        .join(object)
+                        .join(data_dir.to_string())
+                        .join("part.1"),
+                )
+                .await
+                .expect("target shard should be removed before heal");
+            }
+
+            let failed_slots = &target_slots[..failed_target_count];
+            let logs = CapturedLogs::default();
+            let subscriber = tracing_subscriber::fmt()
+                .with_max_level(tracing::Level::WARN)
+                .with_writer(logs.clone())
+                .with_ansi(false)
+                .without_time()
+                .finish();
+            let subscriber_guard = tracing::subscriber::set_default(subscriber);
+            let failure_scope = HealWriterFailureScope::install(&bucket, object, failed_slots, DiskError::DiskFull);
+
+            let heal_outcome = set
+                .heal_object(
+                    &bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        no_lock: true,
+                        scan_mode: HealScanMode::Deep,
+                        ..Default::default()
+                    },
+                )
+                .await;
+            drop(failure_scope);
+            drop(subscriber_guard);
+
+            assert_eq!(
+                heal_outcome.is_err(),
+                expect_error,
+                "{case}: aggregate heal result should match writer outcomes"
+            );
+            let output = logs.contents();
+            assert_eq!(
+                output.matches("Set disk object heal writer failures").count(),
+                1,
+                "{case}: writer failures must emit one aggregate warning per object: {output}"
+            );
+            assert!(
+                output.contains(&format!("writer_failure_count={failed_target_count}")),
+                "{case}: warning must report the aggregate failure count: {output}"
+            );
+            assert!(
+                output.contains(&format!("first_disk_index={}", failed_slots[0])),
+                "{case}: warning must report the first failed target: {output}"
+            );
+            assert!(
+                output.contains("first_part_number=1"),
+                "{case}: warning must report the first failed part: {output}"
+            );
+            assert!(
+                output.contains(&format!("result=\"{expected_result}\"")),
+                "{case}: warning must distinguish partial from all-target failure: {output}"
+            );
+            assert!(
+                output.contains("error=drive path full"),
+                "{case}: warning must preserve a redacted failure reason: {output}"
+            );
+        }
+    }
+
     async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
         let dir = tempfile::tempdir().expect("tempdir should be created");
         let endpoint =

crates/ecstore/src/set_disk/ops/list.rs

@@ -134,7 +134,7 @@ impl crate::storage_api_contracts::list::ListOperations for SetDisks {
     type WalkCancellation = CancellationToken;
     type WalkResultSender = Sender<ObjectInfoOrErr>;
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         bucket: &str,

crates/ecstore/src/set_disk/ops/locking.rs

@@ -28,7 +28,7 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
     type Error = Error;
     type NamespaceLock = NamespaceLockWrapper;
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
         // Resolved from this set's own instance context (backlog#1052), not the
         // ambient facade: the facade tracks whichever context is currently

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -357,6 +357,20 @@ fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec
     (page, is_truncated, next_upload_id_marker)
 }
 
+/// Deterministic per-upload damage during a multipart listing: the metadata
+/// exists but is torn, undecodable, or not identifiable as an upload. The same
+/// corrupt family as `classify_metadata_response_error`'s corrupt group. This
+/// deliberately excludes transient fault shapes (`DiskNotFound`, `Timeout`,
+/// `Io`, `ErasureReadQuorum` from offline-disk quorum loss, ...): degrading on
+/// those would silently drop healthy uploads from a 200 listing while disks
+/// are merely unreachable (issue #5716 review).
+fn is_corrupt_upload_metadata_error(err: &DiskError) -> bool {
+    matches!(
+        err,
+        DiskError::FileCorrupt | DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::OutdatedXLMeta
+    )
+}
+
 async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result<Vec<String>> {
     if !disk.is_online().await {
         return Err(DiskError::DiskNotFound);
@@ -637,25 +651,78 @@ impl SetDisks {
                         }
                         return Ok(None);
                     }
-                    let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
-                    let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
-                    if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
-                        return Err(err);
-                    }
-                    let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
-                    let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?;
-                    if expected_incarnation_id
-                        .is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected))
-                    {
+                    // Affirmative per-upload damage (issue #5716): the staging
+                    // namespace is one flat set of sha256(bucket/object)
+                    // directories shared by every bucket, so a directory whose
+                    // metadata is torn or undecodable must degrade to that
+                    // upload alone. Erroring out instead turns one damaged
+                    // directory into a permanent InternalError for every
+                    // ListMultipartUploads of the bucket. Only the corrupt
+                    // error family counts as damage — transient faults (offline
+                    // disks, timeouts, IO churn) keep failing the listing below
+                    // so clients retry instead of silently losing entries.
+                    let corrupt_metadata = errs
+                        .iter()
+                        .filter(|err| err.as_ref().is_some_and(is_corrupt_upload_metadata_error))
+                        .count();
+                    if corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
+                        debug!(
+                            bucket,
+                            upload_path = %upload_path,
+                            missing_metadata,
+                            corrupt_metadata,
+                            "skipping multipart upload directory with corrupt metadata during listing"
+                        );
                         return Ok(None);
                     }
+                    let decoded: disk::error::Result<Option<(FileInfo, String)>> = (|| {
+                        let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
+                        let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
+                        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
+                            return Err(err);
+                        }
+                        let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
+                        let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?;
+                        if expected_incarnation_id
+                            .is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected))
+                        {
+                            return Ok(None);
+                        }
 
-                    let object = match (
-                        file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
-                        file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
-                    ) {
-                        (Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => object.clone(),
-                        _ => return Err(DiskError::CorruptedFormat),
+                        let object = match (
+                            file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
+                            file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
+                        ) {
+                            (Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => {
+                                object.clone()
+                            }
+                            // A healthy upload that belongs to another bucket:
+                            // not ours to list, and not corruption.
+                            (Some(stored_bucket), Some(_)) if stored_bucket != bucket => return Ok(None),
+                            _ => return Err(DiskError::CorruptedFormat),
+                        };
+                        Ok(Some((file_info, object)))
+                    })();
+                    let (file_info, object) = match decoded {
+                        Ok(Some(decoded)) => decoded,
+                        Ok(None) => return Ok(None),
+                        // Deterministic damage (undecodable metadata that still
+                        // reached quorum, or metadata without its owner keys):
+                        // skip this upload only.
+                        Err(err) if is_corrupt_upload_metadata_error(&err) => {
+                            debug!(
+                                bucket,
+                                upload_path = %upload_path,
+                                error = %err,
+                                "skipping multipart upload directory with unidentifiable metadata during listing"
+                            );
+                            return Ok(None);
+                        }
+                        // Everything else — quorum loss from offline disks,
+                        // timeouts, transport errors — can hide uploads that are
+                        // actually fine; keep failing the listing so clients
+                        // retry instead of silently losing entries.
+                        Err(err) => return Err(err),
                     };
                     if !object.starts_with(prefix) {
                         return Ok(None);
@@ -1988,6 +2055,26 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         )
         .await?;
 
+        // Detach admission before any post-commit await: client cancellation
+        // must not couple durable convergence repair to cleanup work.
+        if convergence.needs_heal() {
+            let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
+                bucket.to_string(),
+                Some(object.to_string()),
+                false,
+                Some(HealChannelPriority::Normal),
+                Some(self.pool_index),
+                Some(self.set_index),
+            );
+            request.object_version_id = fi
+                .version_id
+                .or_else(|| opts.version_suspended.then(Uuid::nil))
+                .map(|version_id| version_id.to_string());
+            tokio::spawn(async move {
+                let _ = rustfs_common::heal_channel::send_heal_request(request).await;
+            });
+        }
+
         // Crash-consistency injection: hard power loss after the authoritative
         // rename_data commit succeeded but before the stale part.N.meta cleanup.
         // The new version is durably committed and visible, so a crash here must
@@ -2061,48 +2148,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         }
         drop(object_lock_guard); // drop object lock guard to release the lock
 
-        // backlog#1321: enqueue heal only when the committed replicas actually
-        // need to converge — a partial commit (some disk failed/offline) or a
-        // signature divergence between committed replicas. A fully healthy MPU
-        // (identical signatures on every disk) is `AllSuccessIdentical` and
-        // submits nothing, which is the fix: the old `Option::is_some()` gate
-        // treated the mere existence of a version signature as "needs heal", so
-        // every healthy <=10-version completion self-enqueued.
-        //
-        // The submit is detached (`tokio::spawn`) so it stays off the ACK
-        // critical path AND survives cancellation of the completion future: the
-        // write is already durable and ACK-worthy, so the heal admission must
-        // not ride the client's request lifetime. The admission itself is
-        // bounded / deduplicated / observable (`send_heal_request` ->
-        // `HealAdmissionResult`), so this emits at most one submit per
-        // completion and coalesces with any in-flight heal for the same object.
-        //
-        // Scanner backstop (backlog#1321 patch): a `PartialCommit` whose
-        // completion is cancelled in the narrow window after the durable commit
-        // but before this spawn runs is not lost — the divergence it would have
-        // healed is exactly what the background scanner reconciles. `Unknown`
-        // (>10 versions, no signature produced) likewise relies on the scanner
-        // rather than self-enqueuing.
-        if convergence.needs_heal() {
-            let bucket = bucket.to_string();
-            let object = object.to_string();
-            let pool_index = self.pool_index;
-            let set_index = self.set_index;
-            tokio::spawn(async move {
-                let _ = rustfs_common::heal_channel::send_heal_request(
-                    rustfs_common::heal_channel::create_heal_request_with_options(
-                        bucket,
-                        Some(object),
-                        false,
-                        Some(HealChannelPriority::Normal),
-                        Some(pool_index),
-                        Some(set_index),
-                    ),
-                )
-                .await;
-            });
-        }
-
         for (i, op_disk) in online_disks.iter().enumerate() {
             if let Some(disk) = op_disk
                 && disk.is_online().await
@@ -3976,6 +4021,9 @@ mod tests {
         }
 
         // Start more in-progress uploads on the same object than a single page holds.
+        // Track only the decoded `<uuid>x<timestamp>` suffixes: the full upload id
+        // embeds the process-global deployment id, which a concurrently running
+        // test can swap between create and list time.
         let total = 5usize;
         let mut created = HashSet::new();
         for _ in 0..total {
@@ -3983,7 +4031,10 @@ mod tests {
                 .new_multipart_upload(bucket, object, &ObjectOptions::default())
                 .await
                 .expect("multipart upload should be created");
-            assert!(created.insert(res.upload_id), "each upload id must be unique");
+            assert!(
+                created.insert(runtime_sources::upload_uuid_suffix(&res.upload_id)),
+                "each upload id must be unique"
+            );
         }
 
         // A single page must never return more than max_uploads entries.
@@ -4038,7 +4089,7 @@ mod tests {
             assert!(page.uploads.len() <= 1, "max_uploads=1 must never return more than one upload");
             for upload in &page.uploads {
                 assert!(
-                    seen.insert(upload.upload_id.clone()),
+                    seen.insert(runtime_sources::upload_uuid_suffix(&upload.upload_id)),
                     "upload {} was returned more than once across pages",
                     upload.upload_id
                 );
@@ -4067,13 +4118,16 @@ mod tests {
             disk.make_volume(bucket).await.expect("bucket volume should be created");
         }
 
+        // Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
+        // upload id embeds the process-global deployment id, which a
+        // concurrently running test can swap between create and list time.
         let mut expected = Vec::new();
         for object in ["logs/a.bin", "logs/a.bin", "logs/b.bin", "other/c.bin"] {
             let upload = set_disks
                 .new_multipart_upload(bucket, object, &ObjectOptions::default())
                 .await
                 .expect("multipart upload should be created");
-            expected.push((object.to_string(), upload.upload_id));
+            expected.push((object.to_string(), runtime_sources::upload_uuid_suffix(&upload.upload_id)));
         }
         expected.sort();
 
@@ -4081,11 +4135,12 @@ mod tests {
             .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
             .await
             .expect("bucket-wide multipart listing should succeed");
-        let listed = all
+        let mut listed = all
             .uploads
             .iter()
-            .map(|upload| (upload.object.clone(), upload.upload_id.clone()))
+            .map(|upload| (upload.object.clone(), runtime_sources::upload_uuid_suffix(&upload.upload_id)))
             .collect::<Vec<_>>();
+        listed.sort();
         assert_eq!(listed, expected);
         assert!(!all.is_truncated);
 
@@ -4149,7 +4204,18 @@ mod tests {
             assert!(upload_id_marker.is_some());
         }
 
-        assert_eq!(listed, expected);
+        // Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
+        // upload id embeds the process-global deployment id, which a
+        // concurrently running test can swap between create and list time.
+        let normalize = |uploads: &[(String, String)]| {
+            let mut normalized = uploads
+                .iter()
+                .map(|(object, upload_id)| (object.clone(), runtime_sources::upload_uuid_suffix(upload_id)))
+                .collect::<Vec<_>>();
+            normalized.sort();
+            normalized
+        };
+        assert_eq!(normalize(&listed), normalize(&expected));
 
         let key_only = set_disks
             .list_multipart_uploads_for_incarnation(bucket, "logs/", Some("logs/a.bin".to_string()), None, None, 1000, None)
@@ -4268,7 +4334,203 @@ mod tests {
             .await
             .expect("incarnation-scoped multipart listing should succeed");
         assert_eq!(scoped.uploads.len(), 1);
-        assert_eq!(scoped.uploads[0].upload_id, current.upload_id);
+        // Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
+        // upload id embeds the process-global deployment id, which a
+        // concurrently running test can swap between create and list time.
+        assert_eq!(
+            runtime_sources::upload_uuid_suffix(&scoped.uploads[0].upload_id),
+            runtime_sources::upload_uuid_suffix(&current.upload_id)
+        );
+    }
+
+    /// The `<deployment-id>.` prefix inside an upload id is read from a
+    /// process-global that concurrently-running tests reinitialize, so id
+    /// assertions compare only the stable `<uuid>x<timestamp>` suffix.
+    fn upload_uuid_suffix(upload_id: &str) -> String {
+        let decoded = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(upload_id.as_bytes())
+            .expect("upload id should be url-safe base64");
+        let decoded = String::from_utf8(decoded).expect("upload id should decode to utf8");
+        decoded
+            .split_once('.')
+            .map(|(_, suffix)| suffix.to_owned())
+            .unwrap_or(decoded)
+    }
+
+    /// Regression (issue #5716): the multipart staging namespace is flat —
+    /// `sha256(bucket/object)` directories from every bucket share one volume —
+    /// so a bucket-scoped listing reads metadata belonging to other buckets'
+    /// in-flight uploads. Those entries must be filtered out, not treated as
+    /// corruption: with the `_ => CorruptedFormat` arm, any concurrent upload
+    /// in another bucket turned every ListMultipartUploads for this bucket
+    /// into an InternalError.
+    #[tokio::test]
+    async fn list_multipart_uploads_ignores_other_buckets_uploads() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cross-bucket-a";
+        let other_bucket = "multipart-cross-bucket-b";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+            disk.make_volume(other_bucket)
+                .await
+                .expect("other bucket volume should be created");
+        }
+
+        let mine = set_disks
+            .new_multipart_upload(bucket, "blobs/data/layer.bin", &ObjectOptions::default())
+            .await
+            .expect("multipart upload should be created");
+        set_disks
+            .new_multipart_upload(other_bucket, "cache/other-layer.bin", &ObjectOptions::default())
+            .await
+            .expect("other bucket multipart upload should be created");
+
+        let listed = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "blobs/", None, None, None, 1000, None)
+            .await
+            .expect("a concurrent upload in another bucket must not poison this bucket's listing");
+        assert_eq!(listed.uploads.len(), 1);
+        assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
+
+        let bucket_wide = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
+            .await
+            .expect("bucket-wide listing must also skip other buckets' uploads");
+        assert_eq!(bucket_wide.uploads.len(), 1);
+        assert_eq!(bucket_wide.uploads[0].object, "blobs/data/layer.bin");
+    }
+
+    /// Regression (issue #5716): a single upload directory whose `xl.meta` was
+    /// destroyed (crash mid-write, torn disk state) must degrade to that upload
+    /// alone. Failing the whole ListMultipartUploads turns one piece of stale
+    /// debris into a permanent outage for every multipart client of the bucket.
+    #[tokio::test]
+    async fn list_multipart_uploads_skips_undecodable_upload_dirs() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-corrupt-dir-bucket";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let healthy = set_disks
+            .new_multipart_upload(bucket, "healthy/object.bin", &ObjectOptions::default())
+            .await
+            .expect("healthy multipart upload should be created");
+        set_disks
+            .new_multipart_upload(bucket, "debris/object.bin", &ObjectOptions::default())
+            .await
+            .expect("debris multipart upload should be created");
+
+        // Destroy the debris upload's metadata on every disk, as an unclean
+        // shutdown mid-create can. The directory stays listable while its
+        // xl.meta no longer decodes.
+        let debris_sha = SetDisks::get_multipart_sha_dir(bucket, "debris/object.bin");
+        let mut corrupted = 0usize;
+        for temp_dir in &temp_dirs {
+            let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&debris_sha);
+            for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
+                if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
+                    tokio::fs::write(&meta, b"not an xl.meta")
+                        .await
+                        .expect("corrupting xl.meta should succeed");
+                    corrupted += 1;
+                }
+            }
+        }
+        assert!(corrupted > 0, "the debris upload must exist on disk before corruption");
+
+        let listed = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "healthy/", None, None, None, 1000, None)
+            .await
+            .expect("one undecodable upload dir must not fail the whole listing");
+        assert_eq!(listed.uploads.len(), 1);
+        assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&healthy.upload_id));
+
+        let bucket_wide = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
+            .await
+            .expect("bucket-wide listing must skip the undecodable upload dir");
+        assert_eq!(bucket_wide.uploads.len(), 1);
+        assert_eq!(
+            upload_uuid_suffix(&bucket_wide.uploads[0].upload_id),
+            upload_uuid_suffix(&healthy.upload_id)
+        );
+    }
+
+    /// Companion boundary to `list_multipart_uploads_skips_undecodable_upload_dirs`:
+    /// corruption BELOW the discovery quorum must not hide the upload — the
+    /// surviving disks still identify it, so it stays listed.
+    #[tokio::test]
+    async fn list_multipart_uploads_survives_sub_quorum_corruption() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-partial-corrupt-bucket";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let upload = set_disks
+            .new_multipart_upload(bucket, "partial/object.bin", &ObjectOptions::default())
+            .await
+            .expect("multipart upload should be created");
+
+        // Corrupt the upload's metadata on exactly one disk (discovery quorum
+        // on this 4-disk set is 2): the other replicas keep it identifiable.
+        let sha = SetDisks::get_multipart_sha_dir(bucket, "partial/object.bin");
+        let mut corrupted = 0usize;
+        for temp_dir in &temp_dirs {
+            let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&sha);
+            for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
+                if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
+                    tokio::fs::write(&meta, b"not an xl.meta")
+                        .await
+                        .expect("corrupting xl.meta should succeed");
+                    corrupted += 1;
+                }
+            }
+            if corrupted > 0 {
+                break;
+            }
+        }
+        assert_eq!(corrupted, 1, "exactly one disk's metadata should be corrupted");
+
+        let listed = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
+            .await
+            .expect("sub-quorum corruption must not fail the listing");
+        assert_eq!(listed.uploads.len(), 1);
+        assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
+    }
+
+    /// Pins the degrade-vs-propagate classification (issue #5716 review): only
+    /// affirmative corruption may skip an upload during listing; every
+    /// transient fault shape must keep failing the listing so clients retry
+    /// instead of silently losing entries.
+    #[test]
+    fn corrupt_upload_metadata_classification_excludes_transient_faults() {
+        for corrupt in [
+            DiskError::FileCorrupt,
+            DiskError::CorruptedFormat,
+            DiskError::CorruptedBackend,
+            DiskError::OutdatedXLMeta,
+        ] {
+            assert!(is_corrupt_upload_metadata_error(&corrupt), "{corrupt:?} is deterministic damage");
+        }
+        for transient in [
+            DiskError::DiskNotFound,
+            DiskError::Timeout,
+            DiskError::FaultyDisk,
+            DiskError::FaultyRemoteDisk,
+            DiskError::DiskAccessDenied,
+            DiskError::VolumeAccessDenied,
+            DiskError::ErasureReadQuorum,
+            DiskError::FileNotFound,
+            DiskError::Io(std::io::Error::other("connection refused")),
+        ] {
+            assert!(
+                !is_corrupt_upload_metadata_error(&transient),
+                "{transient:?} must keep failing the listing"
+            );
+        }
     }
 
     /// Recursively collect every file named `file_name` under the multipart
@@ -4755,7 +5017,13 @@ mod tests {
                 .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None)
                 .await
                 .expect("listing multipart uploads should succeed");
-            page.uploads.iter().any(|u| u.upload_id == upload_id)
+            // Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
+            // upload id embeds the process-global deployment id, which a
+            // concurrently running test can swap between create and list time.
+            let expected_suffix = runtime_sources::upload_uuid_suffix(upload_id);
+            page.uploads
+                .iter()
+                .any(|u| runtime_sources::upload_uuid_suffix(&u.upload_id) == expected_suffix)
         }
 
         #[tokio::test]

crates/ecstore/src/set_disk/ops/object.rs

@@ -972,8 +972,9 @@ impl SetDisks {
 
         let mut object_lock_guard = None;
         let mut bucket_lifecycle_guard = None;
+        let deferred_data_movement_precondition = opts.data_movement && opts.http_preconditions.is_some();
 
-        if opts.http_preconditions.is_some() {
+        if opts.http_preconditions.is_some() && !deferred_data_movement_precondition {
             if !opts.no_lock {
                 if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
                     && opts.bucket_lifecycle_lock_fence.is_none()
@@ -981,10 +982,9 @@ impl SetDisks {
                     bucket_lifecycle_guard = Some(
                         metadata_sys::object_store_in(&self.ctx)
                             .await?
-                            .acquire_bucket_lifecycle_read_lock(bucket)
+                            .acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
                             .await?,
                     );
-                    self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
                 }
                 object_lock_guard = Some(
                     self.acquire_write_lock_diag("put_object_precondition", bucket, object)
@@ -1320,16 +1320,19 @@ impl SetDisks {
                     bucket_lifecycle_guard = Some(
                         metadata_sys::object_store_in(&self.ctx)
                             .await?
-                            .acquire_bucket_lifecycle_read_lock(bucket)
+                            .acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
                             .await?,
                     );
-                    self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
                 }
                 object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?);
             }
             #[cfg(test)]
             pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterNamespace).await;
 
+            if deferred_data_movement_precondition && let Some(err) = self.check_write_precondition(bucket, object, opts).await {
+                return Err(err);
+            }
+
             // Generate ordinary PUT timestamps under the commit lock so version
             // ordering follows durable commit ordering when writers queued on
             // the same object. Internal callers with an explicit timestamp keep
@@ -1445,7 +1448,7 @@ impl SetDisks {
             }
 
             let rename_stage_start = Instant::now();
-            let (online_disks, _, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data(
+            let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data(
                 &shuffle_disks,
                 RUSTFS_META_TMP_BUCKET,
                 tmp_dir.as_str(),
@@ -1455,6 +1458,23 @@ impl SetDisks {
                 write_quorum,
             )
             .await?;
+            // Do this before any post-commit await so request cancellation cannot
+            // bypass best-effort admission. A process crash before admission
+            // remains subject to the existing scanner reconciliation path.
+            if convergence.needs_heal() {
+                let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
+                    bucket.to_string(),
+                    Some(object.to_string()),
+                    false,
+                    Some(HealChannelPriority::Normal),
+                    Some(self.pool_index),
+                    Some(self.set_index),
+                );
+                request.object_version_id = fi.version_id.map(|version_id| version_id.to_string());
+                tokio::spawn(async move {
+                    let _ = rustfs_common::heal_channel::send_heal_request(request).await;
+                });
+            }
             let rename_stage_ms = rename_stage_start.elapsed().as_millis() as u64;
             rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", rename_stage_ms as f64);
             if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
@@ -4343,7 +4363,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         Ok(obj_info)
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "trace", skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         crate::hp_guard!("SetDisks::get_object_info");
         // Acquire a shared read-lock to protect consistency during info fetch
@@ -4368,17 +4388,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
     #[tracing::instrument(skip(self))]
     async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
-        if let Err(e) =
-            rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
-                bucket.to_string(),
-                Some(object.to_string()),
-                false,
-                Some(HealChannelPriority::Normal),
-                Some(self.pool_index),
-                Some(self.set_index),
-            ))
-            .await
-        {
+        let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
+            bucket.to_string(),
+            Some(object.to_string()),
+            false,
+            Some(HealChannelPriority::Normal),
+            Some(self.pool_index),
+            Some(self.set_index),
+        );
+        request.object_version_id = (!version_id.is_empty()).then(|| version_id.to_string());
+        if let Err(e) = rustfs_common::heal_channel::send_heal_request(request).await {
             warn!(
                 bucket,
                 object,
@@ -5764,7 +5783,7 @@ mod transition_commit_failure_tests {
     use http::HeaderMap;
     use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
     use s3s::dto::RestoreRequest;
-    use tokio::io::AsyncReadExt;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
     fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
         let mut metadata = HashMap::new();
@@ -7562,6 +7581,58 @@ mod transition_upload_integrity_tests {
         assert_local_source_intact(&set_disks, bucket, object, &payload).await;
     }
 
+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial]
+    async fn data_movement_cleanup_aborts_after_outer_lock_loss() {
+        let refresh_calls = Arc::new(AtomicUsize::new(0));
+        let lockers: Vec<Arc<dyn LockClient>> = (0..4)
+            .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
+            .collect();
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+        let bucket = "data-movement-cleanup-lock-lost";
+        let object = "object.bin";
+        let payload = b"lost data movement cleanup lock must preserve the source".repeat(1024);
+        write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        let expected = set_disks
+            .load_file_info_versions_exact(bucket, object)
+            .await
+            .expect("source versions should be readable")
+            .expect("source versions should exist");
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+        let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object);
+
+        let cleanup_set = Arc::clone(&set_disks);
+        let cleanup = tokio::spawn(async move {
+            crate::data_movement::cleanup_source_entry_if_unchanged(
+                cleanup_set,
+                bucket,
+                object,
+                &expected,
+                &[],
+                "test_data_movement",
+            )
+            .await
+        });
+        barrier.wait_until_paused().await;
+        tokio::time::advance(Duration::from_secs(11)).await;
+        tokio::task::yield_now().await;
+        assert!(
+            refresh_calls.load(Ordering::SeqCst) > 0,
+            "test must drive the real distributed-lock heartbeat before cleanup commit"
+        );
+        barrier.release();
+
+        let error = cleanup
+            .await
+            .expect("cleanup task should not panic")
+            .expect_err("cleanup must fail after its outer namespace lock loses refresh quorum");
+        assert!(matches!(
+            error,
+            crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. })
+        ));
+        assert_local_source_intact(&set_disks, bucket, object, &payload).await;
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
@@ -8656,6 +8727,87 @@ mod put_object_tmp_cleanup_tests {
         );
     }
 
+    #[tokio::test]
+    async fn data_movement_precondition_is_rechecked_at_commit() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "data-movement-commit-precondition";
+        let object = "object";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let migration_body = vec![b'm'; 64 * 1024];
+        let split = migration_body.len() / 2;
+        let (mut source, stream) = tokio::io::duplex(64);
+        let hash_reader = HashReader::from_stream(
+            stream,
+            i64::try_from(migration_body.len()).expect("migration body length should fit i64"),
+            i64::try_from(migration_body.len()).expect("migration body length should fit i64"),
+            None,
+            None,
+            false,
+        )
+        .expect("migration hash reader should be created");
+        let migration_store = Arc::clone(&set_disks);
+        let migration = tokio::spawn(async move {
+            let mut reader = PutObjReader::new(hash_reader);
+            migration_store
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        data_movement: true,
+                        http_preconditions: Some(HTTPPreconditions {
+                            if_none_match: Some("*".to_string()),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        source
+            .write_all(&migration_body[..split])
+            .await
+            .expect("migration should consume the first half before commit");
+        let mut client_reader = PutObjReader::from_vec(b"new client body".to_vec());
+        tokio::time::timeout(
+            Duration::from_secs(5),
+            set_disks.put_object(bucket, object, &mut client_reader, &ObjectOptions::default()),
+        )
+        .await
+        .expect("client write must not wait for the migration body")
+        .expect("client write should commit while migration waits for the remaining source body");
+        let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
+        source
+            .write_all(&migration_body[split..])
+            .await
+            .expect("migration should consume the remaining source body");
+        drop(source);
+        barrier.wait_until_paused().await;
+        barrier.release();
+
+        let err = migration
+            .await
+            .expect("migration task should join")
+            .expect_err("migration must recheck the target after acquiring its commit lock");
+        assert_eq!(err, StorageError::PreconditionFailed);
+
+        let mut reader = set_disks
+            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("client object should remain readable");
+        let mut body = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut body)
+            .await
+            .expect("client object should drain");
+        assert_eq!(body, b"new client body");
+    }
+
     #[tokio::test]
     async fn metadata_copy_no_lock_aborts_after_outer_namespace_lock_loss() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/store/bucket.rs

@@ -176,6 +176,52 @@ impl ECStore {
         })
     }
 
+    /// Acquire the bucket lifecycle read lock and validate the bucket
+    /// incarnation against `expected`, memoizing the validation while this
+    /// node keeps continuous read-lock coverage (see [`super::bucket_fence`]).
+    ///
+    /// Semantics are identical to the pre-existing per-PUT
+    /// `acquire_bucket_lifecycle_read_lock` + from-disk
+    /// `validate_bucket_incarnation` pair: the first PUT in a coverage window
+    /// performs exactly that authoritative disk validation; overlapping PUTs
+    /// reuse its result, which is sound because bucket deletion/recreation
+    /// requires the lifecycle WRITE lock and therefore cannot have run while
+    /// any read guard was continuously held.
+    pub(crate) async fn acquire_bucket_incarnation_fence(
+        &self,
+        bucket: &str,
+        expected: uuid::Uuid,
+    ) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
+        let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
+        let pieces = super::bucket_fence::FencePieces {
+            registry: self.bucket_fence_registry.clone(),
+            inner,
+        };
+        let memoized = pieces.enter(bucket);
+        let current = match memoized {
+            Some(current) => current,
+            None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
+                Ok(current) => {
+                    // Never memoize under lost coverage: a granted lifecycle
+                    // write lock could already have changed the incarnation.
+                    if !pieces.lock_lost() {
+                        pieces.memoize(bucket, current);
+                    }
+                    current
+                }
+                Err(err) => {
+                    pieces.abandon(bucket);
+                    return Err(err);
+                }
+            },
+        };
+        if current != expected {
+            pieces.abandon(bucket);
+            return Err(StorageError::BucketNotFound(bucket.to_string()));
+        }
+        Ok(pieces.into_guard(bucket))
+    }
+
     pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
         let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
         lock.get_write_lock(get_lock_acquire_timeout())

crates/ecstore/src/store/bucket_fence.rs

@@ -0,0 +1,215 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Memoized bucket-incarnation validation under continuous lifecycle read-lock
+//! coverage.
+//!
+//! The PUT commit fence introduced by #5648 validated the bucket incarnation
+//! with an uncached read (`get_bucket_incarnation_id_from_disk`: a distributed
+//! metadata-transaction read lock plus an EC quorum read of the bucket
+//! metadata) on every PUT commit. Under small-object write load that is two
+//! extra quorum round-trips per PUT, and the resulting lock-manager pressure
+//! produced sustained `Lock acquisition timeout` errors (~1,000 client-visible
+//! failures per 5-minute 64-concurrency window in benchmarks).
+//!
+//! The memo exploits the fence's own locking protocol: bucket deletion and
+//! recreation take the bucket lifecycle WRITE lock, while every fenced PUT
+//! holds a lifecycle READ lock for the whole commit. Therefore, while at least
+//! one lifecycle read guard on this node has been held continuously, no
+//! lifecycle write lock can have been granted anywhere in the cluster, so the
+//! bucket incarnation cannot have changed. The first fenced PUT in such a
+//! coverage window pays the authoritative disk validation exactly as before;
+//! subsequent PUTs whose guards overlap that window compare against the
+//! memoized value. When the node's last guard drops — or any guard observes
+//! `is_lock_lost` — the memo is cleared and the next PUT revalidates from
+//! disk.
+//!
+//! The memo is deliberately per-node process state (not a cross-node cache):
+//! its validity is derived purely from locks this process itself holds, so
+//! best-effort peer cache invalidation (which is why the fence read from disk
+//! in the first place) is irrelevant to its correctness.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use rustfs_lock::NamespaceLockGuard;
+use uuid::Uuid;
+
+#[derive(Default)]
+struct FenceEntry {
+    guards: usize,
+    validated: Option<Uuid>,
+}
+
+/// Per-store registry tracking, per bucket, how many lifecycle read guards are
+/// live on this node and the incarnation id validated under that coverage.
+#[derive(Default)]
+pub(crate) struct BucketFenceRegistry {
+    entries: Mutex<HashMap<String, FenceEntry>>,
+}
+
+impl BucketFenceRegistry {
+    /// Register a new live guard for `bucket` and return the memoized
+    /// incarnation id if one is valid for the current coverage window.
+    fn enter(&self, bucket: &str) -> Option<Uuid> {
+        let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
+        let entry = entries.entry(bucket.to_string()).or_default();
+        entry.guards += 1;
+        entry.validated
+    }
+
+    /// Memoize `incarnation` for `bucket`. Only meaningful while the caller
+    /// still holds a registered guard (which it does by construction).
+    fn memoize(&self, bucket: &str, incarnation: Uuid) {
+        let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
+        if let Some(entry) = entries.get_mut(bucket)
+            && entry.guards > 0
+        {
+            entry.validated = Some(incarnation);
+        }
+    }
+
+    /// Deregister a guard. Clears the memo when the last guard leaves or when
+    /// the leaving guard lost its lock (lost coverage means a lifecycle write
+    /// lock may have been granted, so the memo can no longer be trusted).
+    fn exit(&self, bucket: &str, lock_lost: bool) {
+        let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
+        if let Some(entry) = entries.get_mut(bucket) {
+            entry.guards = entry.guards.saturating_sub(1);
+            if lock_lost {
+                entry.validated = None;
+            }
+            if entry.guards == 0 {
+                entries.remove(bucket);
+            }
+        }
+    }
+}
+
+/// A held bucket lifecycle read lock plus its registration in the fence
+/// registry. Dropping the guard deregisters it; the memo is cleared when the
+/// last guard for the bucket drops (or a lost lock is observed).
+pub(crate) struct BucketIncarnationFenceGuard {
+    inner: Option<NamespaceLockGuard>,
+    registry: Arc<BucketFenceRegistry>,
+    bucket: String,
+}
+
+impl BucketIncarnationFenceGuard {
+    pub(crate) fn is_lock_lost(&self) -> bool {
+        self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
+    }
+}
+
+impl Drop for BucketIncarnationFenceGuard {
+    fn drop(&mut self) {
+        let lost = self.is_lock_lost();
+        self.registry.exit(&self.bucket, lost);
+        self.inner.take();
+    }
+}
+
+pub(super) struct FencePieces {
+    pub(super) registry: Arc<BucketFenceRegistry>,
+    pub(super) inner: NamespaceLockGuard,
+}
+
+impl FencePieces {
+    /// Register the freshly acquired read lock and return the memoized
+    /// incarnation for the coverage window, if any.
+    pub(super) fn enter(&self, bucket: &str) -> Option<Uuid> {
+        self.registry.enter(bucket)
+    }
+
+    pub(super) fn memoize(&self, bucket: &str, incarnation: Uuid) {
+        self.registry.memoize(bucket, incarnation)
+    }
+
+    pub(super) fn lock_lost(&self) -> bool {
+        self.inner.is_lock_lost()
+    }
+
+    pub(super) fn into_guard(self, bucket: &str) -> BucketIncarnationFenceGuard {
+        BucketIncarnationFenceGuard {
+            inner: Some(self.inner),
+            registry: self.registry,
+            bucket: bucket.to_string(),
+        }
+    }
+
+    /// Abandon the acquisition (validation failed): deregister and release.
+    pub(super) fn abandon(self, bucket: &str) {
+        let lost = self.lock_lost();
+        self.registry.exit(bucket, lost);
+        drop(self.inner);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn uuid(n: u128) -> Uuid {
+        Uuid::from_u128(n)
+    }
+
+    #[test]
+    fn memo_valid_only_while_guards_overlap() {
+        let reg = BucketFenceRegistry::default();
+
+        assert_eq!(reg.enter("b"), None, "first guard sees no memo");
+        reg.memoize("b", uuid(1));
+        assert_eq!(reg.enter("b"), Some(uuid(1)), "overlapping guard reuses memo");
+        reg.exit("b", false);
+        reg.exit("b", false);
+
+        // Coverage gap: all guards gone, memo must be dropped.
+        assert_eq!(reg.enter("b"), None, "post-gap guard must revalidate");
+        reg.exit("b", false);
+    }
+
+    #[test]
+    fn lost_lock_clears_memo_but_keeps_other_guards_registered() {
+        let reg = BucketFenceRegistry::default();
+
+        assert_eq!(reg.enter("b"), None);
+        reg.memoize("b", uuid(7));
+        assert_eq!(reg.enter("b"), Some(uuid(7)));
+
+        // First guard exits reporting a lost lock: memo cleared even though
+        // a second guard is still live.
+        reg.exit("b", true);
+        assert_eq!(reg.enter("b"), None, "memo not trusted after a lost lock");
+        reg.exit("b", false);
+        reg.exit("b", false);
+    }
+
+    #[test]
+    fn buckets_are_isolated() {
+        let reg = BucketFenceRegistry::default();
+        assert_eq!(reg.enter("a"), None);
+        reg.memoize("a", uuid(1));
+        assert_eq!(reg.enter("b"), None, "memo does not leak across buckets");
+        reg.exit("b", false);
+        reg.exit("a", false);
+    }
+
+    #[test]
+    fn memoize_without_live_guard_is_ignored() {
+        let reg = BucketFenceRegistry::default();
+        reg.memoize("b", uuid(9));
+        assert_eq!(reg.enter("b"), None);
+        reg.exit("b", false);
+    }
+}

crates/ecstore/src/store/heal.rs

@@ -14,6 +14,7 @@
 
 use super::*;
 use crate::storage_api_contracts::heal::HealOperations as _;
+use tracing::trace;
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_HEAL: &str = "heal";
@@ -83,7 +84,7 @@ impl ECStore {
         Ok(res)
     }
 
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
     pub(super) async fn handle_heal_object(
         &self,
         bucket: &str,
@@ -91,7 +92,7 @@ impl ECStore {
         version_id: &str,
         opts: &HealOpts,
     ) -> Result<(HealResultItem, Option<Error>)> {
-        info!(
+        trace!(
             event = EVENT_HEAL_OBJECT_STARTED,
             component = LOG_COMPONENT_ECSTORE,
             subsystem = LOG_SUBSYSTEM_HEAL,
@@ -283,6 +284,7 @@ mod tests {
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(()),
             ctx: crate::runtime::instance::bootstrap_ctx(),
+            bucket_fence_registry: std::sync::Arc::default(),
         };
 
         let (result, err) = store

crates/ecstore/src/store/init.rs

@@ -419,6 +419,7 @@ impl ECStore {
             // legacy path) so startup writes (erasure type recorded before
             // this point) and later reads share one cell.
             ctx: instance_ctx.clone(),
+            bucket_fence_registry: std::sync::Arc::default(),
         });
 
         // Only set it when this instance's deployment ID is not yet configured
@@ -611,6 +612,7 @@ mod tests {
     };
     use http::HeaderMap;
     use rustfs_config::server_config::KVS;
+    use rustfs_filemeta::ObjectPartInfo;
     #[cfg(feature = "test-util")]
     use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
     use std::{
@@ -1153,6 +1155,145 @@ mod tests {
         (instance_ctx, store, shutdown)
     }
 
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn data_movement_conflicts_preserve_newer_target_and_abort_staging() {
+        let temp_dir = tempfile::tempdir().expect("create data movement store dir");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-conflict-convergence", &[4, 4]))
+                .await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create data movement bucket");
+        let source_mod_time = OffsetDateTime::UNIX_EPOCH;
+        let target_mod_time = source_mod_time + time::Duration::SECOND;
+
+        let object = "single-object";
+        let target_body = b"newer client body".to_vec();
+        let mut target_reader = PutObjReader::from_vec(target_body.clone());
+        store.pools[1]
+            .put_object(
+                &bucket,
+                object,
+                &mut target_reader,
+                &ObjectOptions {
+                    mod_time: Some(target_mod_time),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write newer single-part target");
+
+        let source_body = b"stale migration body".to_vec();
+        crate::data_movement::migrate_object(
+            store.clone(),
+            0,
+            bucket.clone(),
+            GetObjectReader {
+                stream: Box::new(Cursor::new(source_body.clone())),
+                object_info: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: object.to_string(),
+                    size: i64::try_from(source_body.len()).expect("single source size should fit i64"),
+                    actual_size: i64::try_from(source_body.len()).expect("single source size should fit i64"),
+                    etag: Some("0123456789abcdef0123456789abcdef".to_string()),
+                    mod_time: Some(source_mod_time),
+                    ..Default::default()
+                },
+                buffered_body: None,
+                body_source: Default::default(),
+            },
+            "test_data_movement",
+        )
+        .await
+        .expect("newer single-part target should converge migration");
+
+        let mut reader = store
+            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read converged single-part target");
+        let mut body = Vec::new();
+        reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
+        assert_eq!(body, target_body);
+
+        let multipart_object = "multipart-object";
+        let multipart_target_body = b"newer multipart client body".to_vec();
+        let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
+        store.pools[1]
+            .put_object(
+                &bucket,
+                multipart_object,
+                &mut multipart_target_reader,
+                &ObjectOptions {
+                    mod_time: Some(target_mod_time),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write newer multipart target");
+
+        let first_part_size = 5 * 1024 * 1024;
+        let mut multipart_source_body = vec![b'a'; first_part_size];
+        multipart_source_body.push(b'b');
+        let multipart_source_size = i64::try_from(multipart_source_body.len()).expect("multipart source size should fit i64");
+        crate::data_movement::migrate_object(
+            store.clone(),
+            0,
+            bucket.clone(),
+            GetObjectReader {
+                stream: Box::new(Cursor::new(multipart_source_body)),
+                object_info: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: multipart_object.to_string(),
+                    size: multipart_source_size,
+                    actual_size: multipart_source_size,
+                    etag: Some("source-multipart-etag-2".to_string()),
+                    mod_time: Some(source_mod_time),
+                    parts: Arc::new(vec![
+                        ObjectPartInfo {
+                            number: 1,
+                            size: first_part_size,
+                            actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"),
+                            etag: "source-part-1".to_string(),
+                            ..Default::default()
+                        },
+                        ObjectPartInfo {
+                            number: 2,
+                            size: 1,
+                            actual_size: 1,
+                            etag: "source-part-2".to_string(),
+                            ..Default::default()
+                        },
+                    ]),
+                    ..Default::default()
+                },
+                buffered_body: None,
+                body_source: Default::default(),
+            },
+            "test_data_movement",
+        )
+        .await
+        .expect("newer multipart target should converge migration");
+
+        let uploads = store.pools[1]
+            .list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
+            .await
+            .expect("list target pool multipart uploads");
+        assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
+
+        let mut reader = store
+            .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read converged multipart target");
+        let mut body = Vec::new();
+        reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
+        assert_eq!(body, multipart_target_body);
+    }
+
     #[cfg(feature = "test-util")]
     async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
         store

crates/ecstore/src/store/list.rs

@@ -15,7 +15,7 @@
 use super::*;
 
 impl ECStore {
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self))]
     #[allow(clippy::too_many_arguments)]
     pub(super) async fn handle_list_objects_v2(
         self: Arc<Self>,

crates/ecstore/src/store/mod.rs

@@ -141,6 +141,7 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
 const MAX_UPLOADS_LIST: usize = 10000;
 
 mod bucket;
+mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
 mod heal;
 mod heal_walk;
@@ -193,6 +194,9 @@ pub struct ECStore {
     /// startup writes and post-construction reads share one cell — single
     /// instance behavior is unchanged.
     pub(crate) ctx: Arc<InstanceContext>,
+    /// Memoizes bucket-incarnation validation under continuous lifecycle
+    /// read-lock coverage (see [`bucket_fence`]).
+    pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
 }
 
 impl std::fmt::Debug for ECStore {
@@ -582,7 +586,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore {
     // @start_after as marker when continuation_token empty
     // @delimiter default="/", empty when recursive
     // @max_keys limit
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         bucket: &str,
@@ -787,7 +791,7 @@ impl crate::storage_api_contracts::heal::HealOperations for ECStore {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.handle_heal_bucket(bucket, opts).await
     }
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -890,6 +894,7 @@ mod tests {
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(()),
             ctx,
+            bucket_fence_registry: Arc::default(),
         })
     }
 

crates/ecstore/src/store/multipart.rs

@@ -761,6 +761,7 @@ mod tests {
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(()),
             ctx: crate::runtime::instance::bootstrap_ctx(),
+            bucket_fence_registry: std::sync::Arc::default(),
         }
     }
 

crates/ecstore/src/store/object.rs

@@ -1253,7 +1253,7 @@ impl ECStore {
             .await
     }
 
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self))]
     pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         check_object_args(bucket, object)?;
 
@@ -3012,6 +3012,7 @@ mod tests {
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(()),
             ctx: crate::runtime::instance::bootstrap_ctx(),
+            bucket_fence_registry: std::sync::Arc::default(),
         }
     }
 
@@ -3052,6 +3053,7 @@ mod tests {
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(()),
             ctx: crate::runtime::instance::bootstrap_ctx(),
+            bucket_fence_registry: std::sync::Arc::default(),
         }
     }
 

crates/ecstore/src/store/rebalance.rs

@@ -661,7 +661,7 @@ impl ECStore {
         unique_disks.into_values().collect()
     }
 
-    #[instrument(skip(self))]
+    #[instrument(level = "trace", skip(self))]
     pub(super) async fn handle_new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
         self.pools[0].new_ns_lock(bucket, object).await
     }

crates/filemeta/src/fileinfo.rs

@@ -19,6 +19,7 @@ use rustfs_utils::HashAlgorithm;
 use rustfs_utils::http::{
     SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID,
     SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str,
+    is_encryption_metadata_key, starts_with_ignore_ascii_case,
 };
 use s3s::dto::{RestoreStatus, Timestamp};
 use s3s::header::X_AMZ_RESTORE;
@@ -231,7 +232,7 @@ pub enum TransitionVersionState {
     Exact,
 }
 
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(PartialEq, Clone, Default)]
 pub struct FileInfo {
     pub volume: String,
     pub name: String,
@@ -271,6 +272,117 @@ pub struct FileInfo {
     pub uses_legacy_checksum: bool,
 }
 
+/// Metadata keys whose values carry sealed encryption material (KEK-wrapped DEK,
+/// IV) under either the `x-rustfs-internal-` or `x-minio-internal-` prefix.
+/// Values of these keys must never reach logs at any level.
+fn is_sensitive_metadata_key(key: &str) -> bool {
+    // `is_encryption_metadata_key` covers the x-minio-internal- SSE prefix but not
+    // its x-rustfs-internal- twin, which the dual-key invariant writes alongside it.
+    is_encryption_metadata_key(key) || starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
+}
+
+struct RedactedMetadata<'a>(&'a HashMap<String, String>);
+
+impl std::fmt::Debug for RedactedMetadata<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut map = f.debug_map();
+        for (key, value) in self.0 {
+            if is_sensitive_metadata_key(key) {
+                map.entry(key, &format_args!("<redacted {} bytes>", value.len()));
+            } else {
+                map.entry(key, value);
+            }
+        }
+        map.finish()
+    }
+}
+
+struct ElidedBytes<'a>(&'a Option<Bytes>);
+
+impl std::fmt::Debug for ElidedBytes<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0 {
+            Some(bytes) => write!(f, "Some(<{} bytes elided>)", bytes.len()),
+            None => f.write_str("None"),
+        }
+    }
+}
+
+// Manual Debug: `data` holds full inline object bytes (plaintext user content for
+// non-SSE objects) and `metadata` holds sealed key material — both must stay out
+// of Debug output so whole-struct log dumps cannot leak them. `checksum` is elided
+// too: for non-SSE objects it fingerprints plaintext content, and its raw bytes
+// carry no diagnostic value. The exhaustive destructuring (no `..`) forces every
+// future field through an explicit show/redact decision here.
+impl std::fmt::Debug for FileInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self {
+            volume,
+            name,
+            version_id,
+            is_latest,
+            deleted,
+            transition_status,
+            transitioned_objname,
+            transition_tier,
+            transition_version_id,
+            transition_version,
+            transition_version_state,
+            expire_restored,
+            data_dir,
+            mod_time,
+            size,
+            mode,
+            written_by_version,
+            metadata,
+            parts,
+            erasure,
+            mark_deleted,
+            replication_state_internal,
+            data,
+            num_versions,
+            successor_mod_time,
+            fresh,
+            idx,
+            checksum,
+            versioned,
+            uses_legacy_checksum,
+        } = self;
+        f.debug_struct("FileInfo")
+            .field("volume", volume)
+            .field("name", name)
+            .field("version_id", version_id)
+            .field("is_latest", is_latest)
+            .field("deleted", deleted)
+            .field("transition_status", transition_status)
+            .field("transitioned_objname", transitioned_objname)
+            .field("transition_tier", transition_tier)
+            .field("transition_version_id", transition_version_id)
+            .field("transition_version", transition_version)
+            .field("transition_version_state", transition_version_state)
+            .field("expire_restored", expire_restored)
+            .field("data_dir", data_dir)
+            .field("mod_time", mod_time)
+            .field("size", size)
+            .field("mode", mode)
+            .field("written_by_version", written_by_version)
+            .field("metadata", &RedactedMetadata(metadata))
+            .field("parts", parts)
+            .field("erasure", erasure)
+            .field("mark_deleted", mark_deleted)
+            .field("replication_state_internal", replication_state_internal)
+            .field("data", &ElidedBytes(data))
+            .field("num_versions", num_versions)
+            .field("successor_mod_time", successor_mod_time)
+            .field("fresh", fresh)
+            .field("idx", idx)
+            .field("checksum", &ElidedBytes(checksum))
+            .field("versioned", versioned)
+            .field("uses_legacy_checksum", uses_legacy_checksum)
+            .finish()
+    }
+}
+
 #[derive(Deserialize)]
 #[serde(remote = "FileInfo")]
 struct FileInfoMapDef {
@@ -999,6 +1111,12 @@ impl FileInfo {
         insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string());
     }
 
+    /// Reader for the marker [`Self::set_healing`] writes: true when this
+    /// FileInfo is being committed by the heal path.
+    pub fn is_healing(&self) -> bool {
+        contains_key_str(&self.metadata, SUFFIX_HEALING)
+    }
+
     pub fn set_tier_free_version_id(&mut self, version_id: &str) {
         insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string());
     }
@@ -2410,4 +2528,54 @@ mod tests {
         };
         assert!(with_state.replication_info_equals(&with_state_clone));
     }
+
+    #[test]
+    fn debug_redacts_sealed_encryption_metadata_values() {
+        let sealed_key = "IAAfANqt7wIJfVSgFAG3f5S6HuC2eyM5DdJlx7RSJKw2ZakSb3d5";
+        let sealed_iv = "0Vr8QLGvQThk8gIWFCUnBOTUwZgs7TTBteRnAK9avD0=";
+        let mut fi = FileInfo::default();
+        for key in [
+            "X-Rustfs-Internal-Server-Side-Encryption-Sealed-Key",
+            "X-Minio-Internal-Server-Side-Encryption-Sealed-Key",
+        ] {
+            fi.metadata.insert(key.to_string(), sealed_key.to_string());
+        }
+        for key in [
+            "X-Rustfs-Internal-Server-Side-Encryption-Iv",
+            "X-Minio-Internal-Server-Side-Encryption-Iv",
+            "x-rustfs-encryption-iv",
+        ] {
+            fi.metadata.insert(key.to_string(), sealed_iv.to_string());
+        }
+        fi.metadata.insert("content-type".to_string(), "text/plain".to_string());
+
+        let dump = format!("{fi:?}");
+        assert!(!dump.contains(sealed_key), "sealed key leaked into Debug output: {dump}");
+        assert!(!dump.contains(sealed_iv), "sealed IV leaked into Debug output: {dump}");
+        // Keys stay visible so operators can still see which metadata is present.
+        assert!(dump.contains("X-Rustfs-Internal-Server-Side-Encryption-Sealed-Key"));
+        assert!(dump.contains(&format!("<redacted {} bytes>", sealed_key.len())));
+        // Non-sensitive metadata values keep their diagnostic value.
+        assert!(dump.contains("text/plain"));
+    }
+
+    #[test]
+    fn debug_elides_inline_data_bytes() {
+        let fi = FileInfo {
+            data: Some(Bytes::from_static(b"plaintext user object content")),
+            checksum: Some(Bytes::from_static(b"\x01\x02checksumblob")),
+            ..Default::default()
+        };
+
+        let dump = format!("{fi:?}");
+        assert!(
+            !dump.contains("plaintext user object content"),
+            "inline data leaked into Debug output: {dump}"
+        );
+        assert!(!dump.contains("checksumblob"), "checksum bytes leaked into Debug output: {dump}");
+        assert!(dump.contains("data: Some(<29 bytes elided>)"), "missing data length summary: {dump}");
+
+        let empty = FileInfo::default();
+        assert!(format!("{empty:?}").contains("data: None"));
+    }
 }

crates/filemeta/src/filemeta/codec.rs

@@ -44,12 +44,17 @@ impl FileMeta {
     }
 
     pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> {
+        // A file too short to hold the XL2 magic, or one that carries the
+        // wrong magic, is not merely unreadable — it is affirmative evidence
+        // of a torn or foreign write. Classify it as FileCorrupt so quorum
+        // and listing code can distinguish deterministic damage from
+        // transient IO faults (issue #5716).
         if buf.len() < 8 {
-            return Err(Error::other("xl file header not exists"));
+            return Err(Error::FileCorrupt);
         }
 
         if buf[0..4] != XL_FILE_HEADER {
-            return Err(Error::other("xl file header err"));
+            return Err(Error::FileCorrupt);
         }
 
         let major = byteorder::LittleEndian::read_u16(&buf[4..6]);

crates/filemeta/src/replication.rs

@@ -48,6 +48,14 @@ pub const REPLICATE_HEAL: &str = "replicate:heal";
 pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
 
 /// StatusType of Replication for x-amz-replication-status header
+///
+/// NOTE: `rustfs-replication` owns a sibling copy of this enum (plus
+/// `VersionPurgeStatusType` and `ReplicationState`) bound to the MRF/resync
+/// persistence format, while this copy is bound to the xl.meta disk format.
+/// When adding or renaming a variant here, reconcile the sibling and the
+/// conversion layer — the reconciliation tests in
+/// `crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs`
+/// fail to compile until both sides agree.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
 pub enum ReplicationStatusType {
     /// Pending - replication is pending.

crates/heal/src/heal/erasure_healer.rs

@@ -16,7 +16,7 @@ use crate::heal::{
     progress::HealProgress,
     resume::{CheckpointManager, ResumeManager, ResumeUtils, compose_key},
     storage::{HealStorageAPI, next_heal_listing_token},
-    task::is_missing_object_dir_heal_result,
+    task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample},
 };
 use crate::{Error, Result};
 use futures::{StreamExt, stream::FuturesUnordered};
@@ -612,6 +612,12 @@ impl ErasureSetHealer {
         let page_concurrency_limit =
             Self::effective_heal_page_object_concurrency_for_source(self.source, self.heal_opts.scan_mode);
         let in_flight = Arc::new(AtomicUsize::new(0));
+        // Per-bucket sample caps for per-object warn! lines: a flapping rebuild
+        // disk can fail/skip hundreds of thousands of versions in one sweep, so
+        // only the first few occurrences warn and the rest demote to debug!.
+        // The end-of-pass summary reports the full failed/skipped counts.
+        let mut transient_skip_samples_logged = 0_u64;
+        let mut failure_samples_logged = 0_u64;
 
         // backlog#920: select the per-erasure-set DISK-WALK union enumerator when
         // the scan is Deep OR the request came from AutoHeal — these are the paths
@@ -748,8 +754,7 @@ impl ErasureSetHealer {
                     Err(Error::TransientSkip { message }) => {
                         *skipped_objects += 1;
                         checkpoint_manager.add_skipped_object(key).await?;
-                        warn!(
-                            target: "rustfs::heal::erasure_healer",
+                        demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
                             component = LOG_COMPONENT_HEAL,
                             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
@@ -760,13 +765,12 @@ impl ErasureSetHealer {
                             state = "transient_skip",
                             error = %message,
                             "Erasure set object heal skipped due to transient error"
-                        );
+                        });
                     }
                     Err(err) => {
                         *failed_objects += 1;
                         checkpoint_manager.add_failed_object(key).await?;
-                        warn!(
-                            target: "rustfs::heal::erasure_healer",
+                        demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
                             component = LOG_COMPONENT_HEAL,
                             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
@@ -777,7 +781,7 @@ impl ErasureSetHealer {
                             state = "failed",
                             error = %err,
                             "Erasure set object heal failed"
-                        );
+                        });
                     }
                 }
 

crates/heal/src/heal/manager.rs

@@ -15,7 +15,7 @@
 use crate::heal::{
     progress::{HealProgress, HealStatistics},
     storage::HealStorageAPI,
-    task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
+    task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
 };
 use crate::{Error, Result};
 use metrics::{counter, gauge};
@@ -54,6 +54,13 @@ const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
 const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
 const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
 
+// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/
+// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
+// recovery loops submit those per object, so a full queue or a retry storm
+// would otherwise emit one warn! per object (rustfs/rustfs#5716). The
+// `rustfs_heal_admission_total` metric and the `heal_queue_state` backlog
+// event keep the aggregate signal at operator-visible levels.
+
 #[cfg(test)]
 struct RetryOwnershipTestHook {
     task_id: String,
@@ -976,6 +983,7 @@ impl HealManager {
         let queue_len = queue.len();
         publish_heal_queue_length(queue);
         let queue_capacity = config.queue_size;
+        let per_object_request = request.heal_type.is_per_object();
 
         if queue_len >= queue_capacity && !request.force_start {
             if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) {
@@ -985,8 +993,7 @@ impl HealManager {
                 if let Some(displaced) = queue.push_displacing_lower_priority(request) {
                     publish_heal_queue_length(queue);
                     Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
-                    warn!(
-                        target: "rustfs::heal::manager",
+                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                         event = EVENT_HEAL_QUEUE_ADMISSION,
                         component = LOG_COMPONENT_HEAL,
                         subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -1000,12 +1007,11 @@ impl HealManager {
                         queue_capacity,
                         result = "accepted_by_displacement",
                         "Heal queue request accepted by displacement"
-                    );
+                    });
                     return HealAdmissionResult::Accepted;
                 }
 
-                warn!(
-                    target: "rustfs::heal::manager",
+                demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                     event = EVENT_HEAL_QUEUE_ADMISSION,
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -1017,7 +1023,7 @@ impl HealManager {
                     queue_capacity,
                     result = "full_no_displacement_candidate",
                     "Heal queue request rejected without displacement"
-                );
+                });
                 Self::record_admission_metric(source, HealAdmissionResult::Full, context);
                 return HealAdmissionResult::Full;
             }
@@ -1026,8 +1032,7 @@ impl HealManager {
             Self::record_admission_metric(request.source, admission, context);
             match admission {
                 HealAdmissionResult::Dropped(reason) => {
-                    warn!(
-                        target: "rustfs::heal::manager",
+                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                         event = EVENT_HEAL_QUEUE_ADMISSION,
                         component = LOG_COMPONENT_HEAL,
                         subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -1040,11 +1045,10 @@ impl HealManager {
                         reason = reason.as_str(),
                         result = "dropped_full",
                         "Heal queue request dropped"
-                    );
+                    });
                 }
                 HealAdmissionResult::Full => {
-                    warn!(
-                        target: "rustfs::heal::manager",
+                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                         event = EVENT_HEAL_QUEUE_ADMISSION,
                         component = LOG_COMPONENT_HEAL,
                         subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -1056,7 +1060,7 @@ impl HealManager {
                         queue_capacity,
                         result = "rejected_full",
                         "Heal queue request rejected"
-                    );
+                    });
                 }
                 HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
             }
@@ -1481,6 +1485,7 @@ impl HealManager {
             drop(retrying_heals);
             drop(queue);
             drop(active_heals);
+            Self::record_admission_metric(request.source, admission, "duplicate");
 
             match admission {
                 HealAdmissionResult::Merged => {
@@ -1501,8 +1506,7 @@ impl HealManager {
                     );
                 }
                 HealAdmissionResult::Dropped(reason) => {
-                    warn!(
-                        target: "rustfs::heal::manager",
+                    demote_to_debug_when!(request.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
                         event = EVENT_HEAL_QUEUE_ADMISSION,
                         component = LOG_COMPONENT_HEAL,
                         subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -1512,7 +1516,7 @@ impl HealManager {
                         duplicate_state,
                         result = "dropped_duplicate",
                         "Heal queue admission decided"
-                    );
+                    });
                 }
                 HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
             }
@@ -2554,8 +2558,7 @@ impl HealManager {
                         Err(e) => {
                             let will_retry = retry_request.is_some();
                             if will_retry {
-                                warn!(
-                                    target: "rustfs::heal::manager",
+                                demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
                                     event = EVENT_HEAL_SCHEDULER_STATE,
                                     component = LOG_COMPONENT_HEAL,
                                     subsystem = LOG_SUBSYSTEM_MANAGER,
@@ -2566,7 +2569,7 @@ impl HealManager {
                                     retry_attempt = task.retry_attempts.saturating_add(1),
                                     error = %e,
                                     "Heal scheduler task retrying"
-                                );
+                                });
                             } else {
                                 error!(
                                     target: "rustfs::heal::manager",
@@ -2669,7 +2672,7 @@ impl HealManager {
                             loop {
                                 tokio::select! {
                                     _ = retry_cancel_token.cancelled() => {
-                                        info!(
+                                        debug!(
                                             target: "rustfs::heal::manager",
                                             event = EVENT_HEAL_QUEUE_ADMISSION,
                                             component = LOG_COMPONENT_HEAL,
@@ -2702,7 +2705,7 @@ impl HealManager {
                                 };
                                 if active_duplicate {
                                     retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
-                                    info!(
+                                    debug!(
                                         target: "rustfs::heal::manager",
                                         event = EVENT_HEAL_QUEUE_ADMISSION,
                                         component = LOG_COMPONENT_HEAL,
@@ -2730,7 +2733,7 @@ impl HealManager {
                                         retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                         drop(queue);
                                         retry_completed_heals.lock().await.remove(&retry_request_id);
-                                        info!(
+                                        debug!(
                                             target: "rustfs::heal::manager",
                                             event = EVENT_HEAL_QUEUE_ADMISSION,
                                             component = LOG_COMPONENT_HEAL,
@@ -2751,7 +2754,7 @@ impl HealManager {
                                     HealAdmissionResult::Merged => {
                                         retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                         drop(queue);
-                                        info!(
+                                        debug!(
                                             target: "rustfs::heal::manager",
                                             event = EVENT_HEAL_QUEUE_ADMISSION,
                                             component = LOG_COMPONENT_HEAL,
@@ -2765,7 +2768,11 @@ impl HealManager {
                                         return;
                                     }
                                     HealAdmissionResult::Full => {
-                                        warn!(
+                                        // admit_request_to_queue already logged the
+                                        // rejection (context = "retry"); this repeats
+                                        // every backoff cycle while the queue stays
+                                        // full, so keep it at debug!.
+                                        debug!(
                                             target: "rustfs::heal::manager",
                                             event = EVENT_HEAL_QUEUE_ADMISSION,
                                             component = LOG_COMPONENT_HEAL,
@@ -3760,6 +3767,23 @@ mod tests {
         assert!(retry_error.contains("Lock acquisition timeout"));
     }
 
+    #[test]
+    fn test_retry_request_for_incomplete_heal_rename() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
+        let result = Err(Error::TaskExecutionFailed {
+            message: "Failed to heal object bucket/object: heal rename incomplete: 1 of 2 targets committed".to_string(),
+        });
+
+        let (retry_request, retry_delay, retry_error) =
+            retry_request_for_result(&task, &result).expect("incomplete target rename should be retryable");
+
+        assert_eq!(retry_request.id, task.id);
+        assert_eq!(retry_request.retry_attempts, 1);
+        assert!(retry_delay > Duration::ZERO);
+        assert!(retry_error.contains("heal rename incomplete"));
+    }
+
     #[test]
     fn test_retry_request_for_typed_read_quorum_error() {
         let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/heal/src/heal/task.rs

@@ -47,6 +47,25 @@ const MAX_RETAINED_HEAL_RESULT_ITEMS: usize = 1024;
 const EVENT_HEAL_OBJECT_RESULT: &str = "heal_object_result";
 const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3;
 const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5;
+
+/// Emits at `$level`, demoted to `debug!` when `$demote` is true. Keeps
+/// per-object heal work — Object/Metadata/MRF/ECDecode tasks queued per
+/// object by MRF/autoheal/scanner loops, and per-object sweep failures past
+/// a sample cap — from amplifying into one info!/warn!/error! line per
+/// object during mass recovery (rustfs/rustfs#5716). Aggregate task kinds
+/// and foreground (admin/internal) requests keep operator-visible levels;
+/// metrics and end-of-sweep summaries carry the aggregate signal for the
+/// demoted paths.
+macro_rules! demote_to_debug_when {
+    ($demote:expr, $level:ident, target: $target:expr, { $($fields:tt)* }) => {
+        if $demote {
+            tracing::debug!(target: $target, $($fields)*);
+        } else {
+            tracing::$level!(target: $target, $($fields)*);
+        }
+    };
+}
+pub(crate) use demote_to_debug_when;
 const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage";
 const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result";
 const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage";
@@ -100,6 +119,18 @@ impl HealType {
             Self::ECDecode { .. } => "ec_decode",
         }
     }
+
+    /// Task kinds enqueued at per-object granularity (MRF, autoheal, scanner,
+    /// read-repair loops). Their lifecycle and admission logs stay at `debug!`
+    /// so a recovery loop queuing hundreds of thousands of object heal tasks
+    /// cannot amplify into per-object `info!`/`warn!` lines; aggregate kinds
+    /// (cluster/bucket/prefix/erasure-set) keep operator-visible levels.
+    pub(crate) fn is_per_object(&self) -> bool {
+        matches!(
+            self,
+            Self::Object { .. } | Self::Metadata { .. } | Self::MRF { .. } | Self::ECDecode { .. }
+        )
+    }
 }
 
 fn is_object_level_not_found_error(err: &Error) -> bool {
@@ -115,6 +146,20 @@ pub(crate) fn is_missing_object_dir_heal_result(object: &str, err: &Error) -> bo
     object.ends_with(SLASH_SEPARATOR) && is_object_level_not_found_error(err)
 }
 
+/// Sample cap for per-object failure logs during a sweep: returns true (and
+/// consumes a sample slot) for the first [`MAX_BUCKET_FAILURE_LOG_SAMPLES`]
+/// calls, false afterwards so callers demote the remaining occurrences to
+/// `debug!`. Aggregate failed/skipped counts still surface in end-of-sweep
+/// summaries.
+pub(crate) fn take_failure_log_sample(samples_logged: &mut u64) -> bool {
+    if *samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
+        *samples_logged = samples_logged.saturating_add(1);
+        true
+    } else {
+        false
+    }
+}
+
 /// Heal priority
 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub enum HealPriority {
@@ -618,8 +663,7 @@ impl HealTask {
         )
         .increment(1);
 
-        info!(
-            target: "rustfs::heal::task",
+        demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
             event = EVENT_HEAL_TASK_STATE,
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_TASK,
@@ -628,7 +672,7 @@ impl HealTask {
             state = "started",
             queue_delay = ?queue_delay,
             "Heal task started"
-        );
+        });
 
         let result = match &self.heal_type {
             HealType::Cluster => self.heal_cluster().await,
@@ -660,8 +704,7 @@ impl HealTask {
             Ok(_) => {
                 let mut status = self.status.write().await;
                 *status = HealTaskStatus::Completed;
-                info!(
-                    target: "rustfs::heal::task",
+                demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
                     event = EVENT_HEAL_TASK_STATE,
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
@@ -669,7 +712,7 @@ impl HealTask {
                     heal_type = self.heal_type.log_kind(),
                     state = "completed",
                     "Heal task completed"
-                );
+                });
             }
             Err(Error::TaskCancelled) => {
                 let mut status = self.status.write().await;
@@ -688,8 +731,7 @@ impl HealTask {
             Err(Error::TaskTimeout) => {
                 let mut status = self.status.write().await;
                 *status = HealTaskStatus::Timeout;
-                warn!(
-                    target: "rustfs::heal::task",
+                demote_to_debug_when!(self.heal_type.is_per_object(), warn, target: "rustfs::heal::task", {
                     event = EVENT_HEAL_TASK_STATE,
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
@@ -697,13 +739,16 @@ impl HealTask {
                     heal_type = self.heal_type.log_kind(),
                     state = "timed_out",
                     "Heal task timed out"
-                );
+                });
             }
             Err(e) => {
                 let mut status = self.status.write().await;
                 *status = HealTaskStatus::Failed { error: e.to_string() };
-                error!(
-                    target: "rustfs::heal::task",
+                // Per-object failures are already logged with full object
+                // context by the heal_* implementations and terminally by the
+                // scheduler's task_failed error!; this generic duplicate would
+                // multiply every failed object by the retry count.
+                demote_to_debug_when!(self.heal_type.is_per_object(), error, target: "rustfs::heal::task", {
                     event = EVENT_HEAL_TASK_STATE,
                     component = LOG_COMPONENT_HEAL,
                     subsystem = LOG_SUBSYSTEM_TASK,
@@ -712,7 +757,7 @@ impl HealTask {
                     state = "failed",
                     error = %e,
                     "Heal task failed"
-                );
+                });
             }
         }
 
@@ -830,17 +875,21 @@ impl HealTask {
         };
 
         if !object_exists {
-            warn!(
-                target: "rustfs::heal::task",
+            // Background loops (scanner/MRF/autoheal/read-repair) routinely
+            // race object deletion, so a missing target is per-object noise
+            // for them; only foreground admin/internal requests keep the warn.
+            let background_source = !matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal);
+            demote_to_debug_when!(background_source, warn, target: "rustfs::heal::task", {
                 event = EVENT_HEAL_OBJECT_MISSING,
                 component = LOG_COMPONENT_HEAL,
                 subsystem = LOG_SUBSYSTEM_OBJECT,
                 task_id = %self.id,
                 bucket,
                 object,
+                source = self.source.as_str(),
                 recreate_missing = self.options.recreate_missing,
                 "Heal target object is missing"
-            );
+            });
             if self.options.recreate_missing {
                 debug!(
                     target: "rustfs::heal::task",
@@ -1536,8 +1585,7 @@ impl HealTask {
                             }
                             first_failed_object.get_or_insert_with(|| object.to_string());
                             first_error.get_or_insert_with(|| err.to_string());
-                            if failure_samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
-                                failure_samples_logged = failure_samples_logged.saturating_add(1);
+                            if take_failure_log_sample(&mut failure_samples_logged) {
                                 warn!(
                                     target: "rustfs::heal::task",
                                     event = EVENT_HEAL_BUCKET_RESULT,
@@ -2396,6 +2444,66 @@ mod tests {
         resume_disk: Mutex<Option<DiskStore>>,
     }
 
+    #[test]
+    fn per_object_heal_types_are_classified_for_log_demotion() {
+        assert!(
+            HealType::Object {
+                bucket: "b".to_string(),
+                object: "o".to_string(),
+                version_id: None,
+            }
+            .is_per_object()
+        );
+        assert!(
+            HealType::Metadata {
+                bucket: "b".to_string(),
+                object: "o".to_string(),
+            }
+            .is_per_object()
+        );
+        assert!(
+            HealType::MRF {
+                meta_path: "p".to_string(),
+            }
+            .is_per_object()
+        );
+        assert!(
+            HealType::ECDecode {
+                bucket: "b".to_string(),
+                object: "o".to_string(),
+                version_id: None,
+            }
+            .is_per_object()
+        );
+        assert!(!HealType::Cluster.is_per_object());
+        assert!(!HealType::Bucket { bucket: "b".to_string() }.is_per_object());
+        assert!(
+            !HealType::Prefix {
+                bucket: "b".to_string(),
+                prefix: "p".to_string(),
+            }
+            .is_per_object()
+        );
+        assert!(
+            !HealType::ErasureSet {
+                buckets: Vec::new(),
+                set_disk_id: "s".to_string(),
+            }
+            .is_per_object()
+        );
+    }
+
+    #[test]
+    fn failure_log_sampling_caps_at_max_samples() {
+        let mut samples_logged = 0_u64;
+        for _ in 0..MAX_BUCKET_FAILURE_LOG_SAMPLES {
+            assert!(take_failure_log_sample(&mut samples_logged));
+        }
+        assert!(!take_failure_log_sample(&mut samples_logged));
+        assert!(!take_failure_log_sample(&mut samples_logged));
+        assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES);
+    }
+
     /// Build a latest, non-delete-marker heal list item with no version id.
     fn heal_item(name: &str) -> HealListItem {
         HealListItem {

crates/iam/src/cache.rs

@@ -16,7 +16,7 @@ use std::{
     collections::{HashMap, HashSet},
     ops::{Deref, DerefMut},
     ptr,
-    sync::{Arc, Mutex},
+    sync::{Arc, Mutex, Weak},
 };
 
 use arc_swap::{ArcSwap, Guard};
@@ -65,6 +65,29 @@ pub struct Cache {
     state: ArcSwap<CacheState>,
     write_lock: Mutex<()>,
     service_account_mutation_lock: AsyncMutex<()>,
+    sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
+}
+
+struct StsMutationLockRegistry {
+    locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
+}
+
+pub(crate) struct StsMutationLockState {
+    access_key: String,
+    registry: Weak<StsMutationLockRegistry>,
+    lock: Weak<AsyncMutex<StsMutationLockState>>,
+}
+
+impl Drop for StsMutationLockState {
+    fn drop(&mut self) {
+        let Some(registry) = self.registry.upgrade() else {
+            return;
+        };
+        let mut locks = registry.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        if locks.get(&self.access_key).is_some_and(|current| current.ptr_eq(&self.lock)) {
+            locks.remove(&self.access_key);
+        }
+    }
 }
 
 impl Default for Cache {
@@ -73,6 +96,9 @@ impl Default for Cache {
             state: ArcSwap::new(Arc::new(CacheState::default())),
             write_lock: Mutex::new(()),
             service_account_mutation_lock: AsyncMutex::new(()),
+            sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
+                locks: Mutex::new(HashMap::new()),
+            }),
         }
     }
 }
@@ -84,6 +110,29 @@ impl Cache {
         &self.service_account_mutation_lock
     }
 
+    pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
+        let mut locks = self
+            .sts_account_mutation_locks
+            .locks
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
+            return lock;
+        }
+
+        let registry = Arc::downgrade(&self.sts_account_mutation_locks);
+        let access_key_owned = access_key.to_string();
+        let lock = Arc::new_cyclic(|lock| {
+            AsyncMutex::new(StsMutationLockState {
+                access_key: access_key_owned,
+                registry,
+                lock: lock.clone(),
+            })
+        });
+        locks.insert(access_key.to_string(), Arc::downgrade(&lock));
+        lock
+    }
+
     pub(crate) fn snapshot(&self) -> CacheSnapshot {
         self.state.load()
     }
@@ -445,6 +494,30 @@ mod tests {
     use crate::cache::Cache;
     use crate::store::MappedPolicy;
 
+    #[test]
+    fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
+        let cache = Cache::default();
+        let first = cache.sts_account_mutation_lock("first");
+        let same = cache.sts_account_mutation_lock("first");
+        let different = cache.sts_account_mutation_lock("different");
+
+        assert!(Arc::ptr_eq(&first, &same));
+        assert!(!Arc::ptr_eq(&first, &different));
+        drop(first);
+        drop(same);
+        drop(different);
+
+        let _next = cache.sts_account_mutation_lock("next");
+        let locks = cache
+            .sts_account_mutation_locks
+            .locks
+            .lock()
+            .expect("STS mutation lock registry mutex poisoned");
+        assert!(!locks.contains_key("first"));
+        assert!(!locks.contains_key("different"));
+        assert!(locks.contains_key("next"));
+    }
+
     #[tokio::test]
     async fn test_cache_entity_add() {
         let owner = Arc::new(Cache::default());

crates/iam/src/lib.rs

@@ -102,7 +102,49 @@ pub(crate) async fn notify_iam_delete_user(access_key: &str) -> Vec<IamNotificat
     }
 }
 
+#[cfg(test)]
+pub(crate) struct LoadUserNotificationProbe {
+    pub(crate) observed: std::sync::Mutex<Option<(String, bool)>>,
+    pub(crate) remaining_failures: std::sync::atomic::AtomicUsize,
+    pub(crate) attempts: std::sync::atomic::AtomicUsize,
+    pub(crate) panic: bool,
+    pub(crate) started: tokio::sync::Notify,
+    pub(crate) release: Option<tokio::sync::Notify>,
+    pub(crate) completed: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    pub(crate) static LOAD_USER_NOTIFICATION_PROBE: std::sync::Arc<LoadUserNotificationProbe>;
+}
+
 pub(crate) async fn notify_iam_load_user(access_key: &str, temp: bool) -> Vec<IamNotificationPeerErr> {
+    #[cfg(test)]
+    if let Ok(probe) = LOAD_USER_NOTIFICATION_PROBE.try_with(std::sync::Arc::clone) {
+        probe.attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        *probe.observed.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some((access_key.to_string(), temp));
+        probe.started.notify_one();
+        if let Some(release) = &probe.release {
+            release.notified().await;
+        }
+        assert!(!probe.panic, "notification probe panic");
+        let should_fail = probe
+            .remaining_failures
+            .fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |remaining| {
+                remaining.checked_sub(1)
+            })
+            .is_ok();
+        let result = if should_fail {
+            vec![IamNotificationPeerErr {
+                err: Some(IamEcstoreError::other("peer notification failed")),
+            }]
+        } else {
+            Vec::new()
+        };
+        probe.completed.notify_one();
+        return result;
+    }
+
     match runtime_sources::notification_sys() {
         Some(notification_sys) => notification_sys
             .load_user(access_key, temp)

crates/iam/src/manager.rs

@@ -293,6 +293,8 @@ where
     }
 
     pub async fn load_user(&self, access_key: &str) -> Result<()> {
+        let sts_mutation_lock = self.cache.sts_account_mutation_lock(access_key);
+        let _sts_mutation_guard = sts_mutation_lock.lock().await;
         let mut users_map: HashMap<String, UserIdentity> = HashMap::new();
         let mut user_policy_map = HashMap::new();
         let mut sts_users_map = HashMap::new();
@@ -1207,6 +1209,9 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let mutation_lock = self.cache.sts_account_mutation_lock(access_key);
+        let _mutation_guard = mutation_lock.lock().await;
+
         let sts_policy_update = if let Some(policy) = policy_name {
             let mp = MappedPolicy::new(policy);
             let (_, combined_policy_stmt) = filter_policies(&self.cache, &mp.policies, "temp");
@@ -1433,6 +1438,11 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
+        let _sts_mutation_guard = match &sts_mutation_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let _service_account_guard = if utype == UserType::Svc {
             Some(self.cache.service_account_mutation_lock().lock().await)
         } else {
@@ -1493,9 +1503,13 @@ where
             });
         }
 
-        let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
+        if utype != UserType::Sts {
+            let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
+        }
 
-        self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
+        if utype != UserType::Sts {
+            self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
+        }
 
         if let Err(err) = self.api.delete_user_identity(access_key, utype).await
             && !is_err_no_such_user(&err)
@@ -1507,8 +1521,17 @@ where
         self.cache.with_write_lock(|cache| {
             if utype == UserType::Sts {
                 cache.delete_sts_account(access_key, deleted_at);
+                if cache
+                    .state()
+                    .users
+                    .get(access_key)
+                    .is_some_and(|identity| identity.credentials.is_temp())
+                {
+                    cache.delete_user(access_key, deleted_at);
+                }
+            } else {
+                cache.delete_user(access_key, deleted_at);
             }
-            cache.delete_user(access_key, deleted_at);
         });
 
         Ok(deleted_at)
@@ -2032,6 +2055,11 @@ where
         Ok(())
     }
     pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
+        let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
+        let _sts_mutation_guard = match &sts_mutation_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let _service_account_guard = if user_type == UserType::Svc {
             Some(self.cache.service_account_mutation_lock().lock().await)
         } else {
@@ -2077,7 +2105,9 @@ where
                     UserType::Reg | UserType::Svc => cache.delete_user(name, now),
                     UserType::None => {}
                 }
-                self.remove_user_from_cached_groups(cache, name, now);
+                if user_type != UserType::Sts {
+                    self.remove_user_from_cached_groups(cache, name, now);
+                }
                 if user_type == UserType::Reg {
                     for access_key in service_accounts_to_delete.iter() {
                         cache.delete_user(access_key, now);
@@ -2087,7 +2117,9 @@ where
                         cache.delete_user(access_key, now);
                     }
                 }
-                cache.delete_user_policy(name, now);
+                if user_type != UserType::Sts {
+                    cache.delete_user_policy(name, now);
+                }
             });
 
             return Ok(());
@@ -2446,12 +2478,12 @@ mod tests {
         saved_user: Arc<Mutex<Option<UserIdentity>>>,
         load_attempts: Arc<AtomicUsize>,
         visible_after_attempt: usize,
-        block_service_save: Arc<AtomicBool>,
-        service_save_started: Arc<Notify>,
-        release_service_save: Arc<Notify>,
-        block_service_load: Arc<AtomicBool>,
-        service_load_started: Arc<Notify>,
-        release_service_load: Arc<Notify>,
+        block_account_save: Arc<AtomicBool>,
+        account_save_started: Arc<Notify>,
+        release_account_save: Arc<Notify>,
+        block_account_load: Arc<AtomicBool>,
+        account_load_started: Arc<Notify>,
+        release_account_load: Arc<Notify>,
     }
 
     impl DelayedTempUserVisibilityStore {
@@ -2460,12 +2492,12 @@ mod tests {
                 saved_user: Arc::new(Mutex::new(None)),
                 load_attempts: Arc::new(AtomicUsize::new(0)),
                 visible_after_attempt,
-                block_service_save: Arc::new(AtomicBool::new(false)),
-                service_save_started: Arc::new(Notify::new()),
-                release_service_save: Arc::new(Notify::new()),
-                block_service_load: Arc::new(AtomicBool::new(false)),
-                service_load_started: Arc::new(Notify::new()),
-                release_service_load: Arc::new(Notify::new()),
+                block_account_save: Arc::new(AtomicBool::new(false)),
+                account_save_started: Arc::new(Notify::new()),
+                release_account_save: Arc::new(Notify::new()),
+                block_account_load: Arc::new(AtomicBool::new(false)),
+                account_load_started: Arc::new(Notify::new()),
+                release_account_load: Arc::new(Notify::new()),
             }
         }
     }
@@ -2495,9 +2527,9 @@ mod tests {
             item: UserIdentity,
             _ttl: Option<usize>,
         ) -> Result<()> {
-            if user_type == UserType::Svc && self.block_service_save.load(Ordering::SeqCst) {
-                self.service_save_started.notify_one();
-                self.release_service_save.notified().await;
+            if matches!(user_type, UserType::Svc | UserType::Sts) && self.block_account_save.load(Ordering::SeqCst) {
+                self.account_save_started.notify_one();
+                self.release_account_save.notified().await;
             }
             *self.saved_user.lock().expect("saved_user mutex poisoned") = Some(item);
             Ok(())
@@ -2528,9 +2560,18 @@ mod tests {
                 .expect("saved_user mutex poisoned")
                 .clone()
                 .ok_or_else(|| Error::NoSuchUser(name.to_string()))?;
-            if user_type == UserType::Svc && self.block_service_load.load(Ordering::SeqCst) {
-                self.service_load_started.notify_one();
-                self.release_service_load.notified().await;
+            let matches_user_type = match user_type {
+                UserType::Sts => loaded.credentials.is_temp(),
+                UserType::Svc => loaded.credentials.is_service_account(),
+                UserType::Reg => !loaded.credentials.is_temp() && !loaded.credentials.is_service_account(),
+                UserType::None => false,
+            };
+            if !matches_user_type {
+                return Err(Error::NoSuchUser(name.to_string()));
+            }
+            if self.block_account_load.load(Ordering::SeqCst) {
+                self.account_load_started.notify_one();
+                self.release_account_load.notified().await;
             }
             m.insert(name.to_string(), loaded);
             Ok(())
@@ -2746,12 +2787,12 @@ mod tests {
         };
         cache.add_service_account(credentials).await.expect("seed service account");
 
-        store.block_service_load.store(true, Ordering::SeqCst);
+        store.block_account_load.store(true, Ordering::SeqCst);
         let notification = {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
         };
-        store.service_load_started.notified().await;
+        store.account_load_started.notified().await;
 
         let update = {
             let cache = Arc::clone(&cache);
@@ -2776,7 +2817,7 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!update.is_finished(), "update must wait for the in-flight cache refresh");
 
-        store.release_service_load.notify_one();
+        store.release_account_load.notify_one();
         notification.await.expect("notification task").expect("notification refresh");
         update.await.expect("update task").expect("service account update");
 
@@ -2793,7 +2834,7 @@ mod tests {
     #[tokio::test]
     async fn concurrent_service_account_create_cannot_overwrite_first_writer() {
         let store = DelayedTempUserVisibilityStore::new(0);
-        store.block_service_save.store(true, Ordering::SeqCst);
+        store.block_account_save.store(true, Ordering::SeqCst);
         let cache = Arc::new(build_test_iam_cache(store.clone()));
         let access_key = "SERIALIZEDSERVICE00";
         let credentials = |secret_key: &str| Credentials {
@@ -2808,7 +2849,7 @@ mod tests {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.add_service_account(credentials("firstServiceSecret123")).await })
         };
-        store.service_save_started.notified().await;
+        store.account_save_started.notified().await;
 
         let second = {
             let cache = Arc::clone(&cache);
@@ -2817,8 +2858,8 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!second.is_finished(), "second create must wait for the first writer");
 
-        store.block_service_save.store(false, Ordering::SeqCst);
-        store.release_service_save.notify_waiters();
+        store.block_account_save.store(false, Ordering::SeqCst);
+        store.release_account_save.notify_waiters();
         first.await.expect("first create task").expect("first create");
         let err = second
             .await
@@ -2862,12 +2903,12 @@ mod tests {
         };
         cache.add_service_account(credentials).await.expect("seed service account");
 
-        store.block_service_load.store(true, Ordering::SeqCst);
+        store.block_account_load.store(true, Ordering::SeqCst);
         let notification = {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
         };
-        store.service_load_started.notified().await;
+        store.account_load_started.notified().await;
 
         let delete = {
             let cache = Arc::clone(&cache);
@@ -2876,7 +2917,7 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!delete.is_finished(), "delete must wait for the in-flight cache refresh");
 
-        store.release_service_load.notify_one();
+        store.release_account_load.notify_one();
         notification.await.expect("notification task").expect("notification refresh");
         delete.await.expect("delete task").expect("service account delete");
 
@@ -2884,6 +2925,130 @@ mod tests {
         assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
     }
 
+    #[tokio::test]
+    async fn sts_notification_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+        cache
+            .set_temp_user(&access_key, &credentials, None)
+            .await
+            .expect("seed temporary account");
+
+        store.block_account_load.store(true, Ordering::SeqCst);
+        let notification = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.user_notification_handler(&access_key, UserType::Sts).await })
+        };
+        store.account_load_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_notification = !delete.is_finished();
+
+        store.release_account_load.notify_one();
+        notification.await.expect("notification task").expect("notification refresh");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_notification, "delete must wait for the in-flight STS cache refresh");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
+    #[tokio::test]
+    async fn sts_auth_reload_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+        cache
+            .set_temp_user(&access_key, &credentials, None)
+            .await
+            .expect("seed temporary account");
+        cache.cache.delete_sts_account(&access_key, OffsetDateTime::now_utc());
+
+        store.block_account_load.store(true, Ordering::SeqCst);
+        let reload = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.load_user(&access_key).await })
+        };
+        store.account_load_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_reload = !delete.is_finished();
+
+        store.release_account_load.notify_one();
+        reload.await.expect("reload task").expect("authentication cache reload");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_reload, "delete must wait for the in-flight authentication reload");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
+    #[tokio::test]
+    async fn sts_create_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        store.block_account_save.store(true, Ordering::SeqCst);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+
+        let create = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.set_temp_user(&access_key, &credentials, None).await })
+        };
+        store.account_save_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_create = !delete.is_finished();
+
+        store.block_account_save.store(false, Ordering::SeqCst);
+        store.release_account_save.notify_one();
+        create.await.expect("create task").expect("temporary account create");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_create, "delete must wait for the in-flight STS create");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
     #[tokio::test]
     async fn test_init_keeps_error_state_when_initial_load_fails() {
         let (sender, receiver) = mpsc::channel::<i64>(1);

crates/iam/src/store.rs

@@ -139,6 +139,59 @@ impl UserType {
     }
 }
 
+/// Encode a [`UserType`] as the site-replication wire value for
+/// `SRPolicyMapping.userType` / `SRCredInfo.iamUserType`.
+///
+/// The wire uses MinIO's `IAMUserType` table (cmd/iam.go):
+///
+/// | wire | MinIO meaning |
+/// |------|---------------|
+/// | -1   | unknown       |
+/// | 0    | regUser       |
+/// | 1    | stsUser       |
+/// | 2    | svcUser       |
+///
+/// This is deliberately distinct from the internal encoding
+/// [`UserType::to_u64`]/[`UserType::from_u64`] (None=0, Svc=1, Sts=2, Reg=3),
+/// which is used by intra-cluster node RPC and must never change (a rolling
+/// restart mixes old and new nodes on that RPC). Do not "unify" the two
+/// tables: internal values on the SR wire mislabel users on MinIO peers.
+///
+/// Group mappings always encode as 0: MinIO routes group mappings by the
+/// `isGroup` flag (userType is effectively ignored), and pre-fix RustFS peers
+/// sent 0 for groups, so 0 is the one value every peer generation accepts.
+pub fn sr_wire_user_type(user_type: UserType, is_group: bool) -> i64 {
+    if is_group {
+        return 0;
+    }
+    match user_type {
+        UserType::Reg | UserType::None => 0,
+        UserType::Sts => 1,
+        UserType::Svc => 2,
+    }
+}
+
+/// Decode a site-replication wire `userType` value (see [`sr_wire_user_type`]
+/// for the table) into a [`UserType`].
+///
+/// - `-1` (MinIO unknown, sent for group mappings) maps to [`UserType::None`];
+///   `policy_db_set` routes group items by `is_group`, and for non-group items
+///   `None` shares the users prefix with `Reg`.
+/// - `3` is a permanent alias for [`UserType::Reg`]: pre-fix RustFS peers sent
+///   the internal encoding (`Reg.to_u64() == 3`) on the wire. Keep it forever
+///   for mixed-version site replication; do not remove.
+/// - Anything else is unknown and rejected (`None`), so callers fail closed.
+pub fn user_type_from_sr_wire(v: i64) -> Option<UserType> {
+    match v {
+        -1 => Some(UserType::None),
+        0 => Some(UserType::Reg),
+        1 => Some(UserType::Sts),
+        2 => Some(UserType::Svc),
+        3 => Some(UserType::Reg),
+        _ => None,
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone)]
 pub struct MappedPolicy {
     pub version: i64,
@@ -214,7 +267,72 @@ impl GroupInfo {
 
 #[cfg(test)]
 mod tests {
-    use super::{GroupInfo, MappedPolicy};
+    use super::{GroupInfo, MappedPolicy, UserType, sr_wire_user_type, user_type_from_sr_wire};
+
+    /// Site-replication inbound decode of `SRPolicyMapping.userType` must
+    /// follow MinIO IAMUserType wire semantics (cmd/iam.go): stsUser = 1.
+    /// The internal `UserType::from_u64` table maps 1 to Svc — reusing it at
+    /// the SR boundary lands federated STS mappings under the wrong prefix
+    /// and silently drops their effect.
+    #[test]
+    fn sr_inbound_decodes_minio_sts_wire_value_as_sts() {
+        assert_eq!(user_type_from_sr_wire(1), Some(UserType::Sts));
+    }
+
+    /// Wire-constant contract: literal MinIO IAMUserType values (cmd/iam.go).
+    /// WARNING: these literals are the cross-vendor wire format. Never "tidy"
+    /// them to match `UserType::to_u64`/`from_u64` — that internal table
+    /// (None=0, Svc=1, Sts=2, Reg=3) belongs to intra-cluster node RPC only.
+    #[test]
+    fn sr_wire_decode_matches_minio_iam_user_type_table() {
+        assert_eq!(user_type_from_sr_wire(-1), Some(UserType::None)); // MinIO unknown (group mappings)
+        assert_eq!(user_type_from_sr_wire(0), Some(UserType::Reg)); // MinIO regUser
+        assert_eq!(user_type_from_sr_wire(1), Some(UserType::Sts)); // MinIO stsUser
+        assert_eq!(user_type_from_sr_wire(2), Some(UserType::Svc)); // MinIO svcUser
+        // Permanent alias: pre-fix RustFS peers sent internal Reg=3 on the wire.
+        assert_eq!(user_type_from_sr_wire(3), Some(UserType::Reg));
+        // Unknown values fail closed.
+        assert_eq!(user_type_from_sr_wire(4), None);
+        assert_eq!(user_type_from_sr_wire(-2), None);
+    }
+
+    /// Wire-constant contract for the outbound direction.
+    #[test]
+    fn sr_wire_encode_matches_minio_iam_user_type_table() {
+        assert_eq!(sr_wire_user_type(UserType::Reg, false), 0); // MinIO regUser
+        assert_eq!(sr_wire_user_type(UserType::Sts, false), 1); // MinIO stsUser
+        assert_eq!(sr_wire_user_type(UserType::Svc, false), 2); // MinIO svcUser
+        assert_eq!(sr_wire_user_type(UserType::None, false), 0);
+        // Group mappings always go out as 0 — the value both MinIO (routes by
+        // isGroup) and pre-fix RustFS peers accept.
+        for ut in [UserType::Reg, UserType::Sts, UserType::Svc, UserType::None] {
+            assert_eq!(sr_wire_user_type(ut, true), 0);
+        }
+    }
+
+    /// Mixed-version matrix: every value a peer generation can emit decodes to
+    /// a `UserType` the receiver stores correctly.
+    #[test]
+    fn sr_wire_round_trip_covers_old_rustfs_and_minio_peers() {
+        // Old RustFS outbound: user mappings as internal Reg=3, groups as 0.
+        assert_eq!(user_type_from_sr_wire(3), Some(UserType::Reg));
+        assert_eq!(user_type_from_sr_wire(0), Some(UserType::Reg));
+        // New RustFS outbound decodes on its own kind (self round-trip).
+        for (ut, is_group) in [
+            (UserType::Reg, false),
+            (UserType::Sts, false),
+            (UserType::Svc, false),
+            (UserType::None, true),
+        ] {
+            assert!(user_type_from_sr_wire(sr_wire_user_type(ut, is_group)).is_some());
+        }
+        // Internal RPC encoding is untouched (rolling-restart contract).
+        assert_eq!(UserType::None.to_u64(), 0);
+        assert_eq!(UserType::Svc.to_u64(), 1);
+        assert_eq!(UserType::Sts.to_u64(), 2);
+        assert_eq!(UserType::Reg.to_u64(), 3);
+        assert_eq!(UserType::from_u64(1), Some(UserType::Svc));
+    }
 
     /// uses RFC3339 for updatedAt. MappedPolicy must serialize as RFC3339.
     #[test]

crates/iam/src/store/object.rs

@@ -28,7 +28,10 @@ use crate::{
 use futures::future::join_all;
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
-use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
+use rustfs_utils::{
+    MaskedAccessKey,
+    path::{SLASH_SEPARATOR, path_join_buf},
+};
 use serde::{Serialize, de::DeserializeOwned};
 use std::sync::{LazyLock, Mutex};
 use std::time::{Duration, Instant};
@@ -600,10 +603,10 @@ impl ObjectStore {
             .await
             .map_err(|err| {
                 if is_err_config_not_found(&err) {
-                    warn!(name, user_type = ?user_type, "IAM user identity missing");
+                    debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
                     Error::NoSuchUser(name.to_owned())
                 } else {
-                    warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
+                    warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM user identity load failed");
                     err
                 }
             })?;
@@ -611,7 +614,7 @@ impl ObjectStore {
         if u.credentials.is_expired() {
             let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
             let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
-            warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
+            warn!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity expired and was removed");
             return Err(Error::NoSuchUser(name.to_owned()));
         }
 
@@ -635,7 +638,7 @@ impl ObjectStore {
                         let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
                         let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
                     }
-                    warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
+                    warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
                     return Err(Error::NoSuchUser(name.to_owned()));
                 }
             }
@@ -873,13 +876,7 @@ impl Store for ObjectStore {
     async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
         self.delete_iam_config(get_user_identity_path(name, user_type))
             .await
-            .map_err(|err| {
-                if is_err_config_not_found(&err) {
-                    Error::NoSuchPolicy
-                } else {
-                    err
-                }
-            })?;
+            .map_err(|err| map_delete_user_identity_error(name, err))?;
         Ok(())
     }
     async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity> {
@@ -1327,9 +1324,18 @@ impl Store for ObjectStore {
     }
 }
 
+fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
+    if is_err_config_not_found(&err) {
+        Error::NoSuchUser(name.to_owned())
+    } else {
+        err
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{DecryptSource, LoadMode, ObjectStore};
+    use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
+    use crate::error::Error;
     use crate::keyring;
     use rustfs_credentials::{Credentials, init_global_action_credentials};
     use serial_test::serial;
@@ -1352,6 +1358,12 @@ mod tests {
         assert!(!LoadMode::Locked.read_opts().no_lock);
     }
 
+    #[test]
+    fn missing_user_identity_delete_maps_to_no_such_user() {
+        let err = map_delete_user_identity_error("missing-sts", Error::ConfigNotFound);
+        assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
+    }
+
     fn test_cred() -> Credentials {
         if let Some(cred) = crate::root_credentials::credentials() {
             return cred;

crates/iam/src/sys.rs

@@ -48,6 +48,16 @@ use time::OffsetDateTime;
 use tokio::sync::RwLock;
 use tracing::{error, info, warn};
 
+#[cfg(not(test))]
+const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
+#[cfg(test)]
+const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_millis(1);
+const STS_INVALIDATION_MAX_ATTEMPTS: usize = 3;
+#[cfg(not(test))]
+const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+#[cfg(test)]
+const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(20);
+
 pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;
 pub const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
 
@@ -69,6 +79,42 @@ enum PolicyPluginState {
     Failed,
 }
 
+impl PolicyPluginState {
+    fn prepared_iam_auth(&self) -> Option<PreparedIamAuth> {
+        match self {
+            Self::Ready(_) => Some(PreparedIamAuth {
+                needs_existing_object_tag: true,
+                mode: PreparedIamMode::Opa,
+            }),
+            Self::Initializing | Self::Failed => Some(PreparedIamAuth {
+                needs_existing_object_tag: false,
+                mode: PreparedIamMode::Deny,
+            }),
+            Self::Disabled => None,
+        }
+    }
+}
+
+async fn resolve_policy_plugin_state() -> PolicyPluginState {
+    match opa::lookup_config().await {
+        Ok(conf) if conf.enable() => {
+            info!("OPA plugin enabled");
+            PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
+        }
+        Ok(_) => PolicyPluginState::Failed,
+        Err(e) => {
+            error!(
+                component = "iam",
+                subsystem = "policy_plugin",
+                result = "configuration_load_failed",
+                error_kind = e.kind(),
+                "OPA plugin configuration load failed"
+            );
+            PolicyPluginState::Failed
+        }
+    }
+}
+
 static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
 
 fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
@@ -83,23 +129,7 @@ fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
             if configured {
                 let state = Arc::clone(&state);
                 tokio::spawn(async move {
-                    let next_state = match opa::lookup_config().await {
-                        Ok(conf) if conf.enable() => {
-                            info!("OPA plugin enabled");
-                            PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
-                        }
-                        Ok(_) => PolicyPluginState::Failed,
-                        Err(e) => {
-                            error!(
-                                component = "iam",
-                                subsystem = "policy_plugin",
-                                result = "configuration_load_failed",
-                                error_kind = e.kind(),
-                                "OPA plugin configuration load failed"
-                            );
-                            PolicyPluginState::Failed
-                        }
-                    };
+                    let next_state = resolve_policy_plugin_state().await;
                     *state.write().await = next_state;
                 });
             }
@@ -393,17 +423,74 @@ impl<T: Store> IamSys<T> {
     /// associated session token. This is the primitive used by the admin
     /// `revoke-tokens` endpoint to revoke STS credentials for a parent user.
     pub async fn delete_temp_account(&self, access_key: &str, notify: bool) -> Result<()> {
-        self.store.delete_user(access_key, UserType::Sts).await?;
-
-        if notify && !self.has_watcher() {
-            for r in notify_iam_delete_user(access_key).await {
-                if let Some(err) = r.err {
-                    warn!("notify delete_temp_account failed: {}", err);
-                }
-            }
+        if !notify || self.has_watcher() {
+            return self.store.delete_user(access_key, UserType::Sts).await;
         }
 
-        Ok(())
+        let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
+        #[cfg(test)]
+        let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
+        #[cfg(test)]
+        let notification_available = notification_probe.is_some() || crate::runtime_sources::notification_sys().is_some();
+        #[cfg(not(test))]
+        let notification_available = crate::runtime_sources::notification_sys().is_some();
+        if !notification_available {
+            return Err(Error::other("IAM peer notification system is unavailable"));
+        }
+
+        let store = Arc::clone(&self.store);
+        let access_key = access_key.to_string();
+
+        let operation = async move {
+            store.delete_user(&access_key, UserType::Sts).await?;
+
+            let mut delay = STS_INVALIDATION_RETRY_INITIAL_DELAY;
+            for attempt in 1..=STS_INVALIDATION_MAX_ATTEMPTS {
+                let attempt_error =
+                    match tokio::time::timeout(STS_INVALIDATION_ATTEMPT_TIMEOUT, notify_iam_load_user(&access_key, true)).await {
+                        Ok(results) => results.into_iter().find_map(|result| result.err).map(Error::other),
+                        Err(_) => Some(Error::other("peer STS invalidation timed out")),
+                    };
+                let Some(err) = attempt_error else {
+                    return Ok(());
+                };
+                if attempt == STS_INVALIDATION_MAX_ATTEMPTS {
+                    return Err(Error::other(err));
+                }
+                tokio::time::sleep(delay).await;
+                delay = delay.saturating_mul(2);
+            }
+            unreachable!("STS invalidation retry loop always returns")
+        };
+
+        #[cfg(test)]
+        let task = runtime.spawn(async move {
+            if let Some(probe) = notification_probe {
+                return crate::LOAD_USER_NOTIFICATION_PROBE.scope(probe, operation).await;
+            }
+            operation.await
+        });
+        #[cfg(not(test))]
+        let task = runtime.spawn(operation);
+
+        task.await.map_err(Error::other)?
+    }
+
+    #[cfg(test)]
+    fn load_user_notification_probe(
+        failures_before_success: usize,
+        block: bool,
+        panic: bool,
+    ) -> Arc<crate::LoadUserNotificationProbe> {
+        Arc::new(crate::LoadUserNotificationProbe {
+            observed: std::sync::Mutex::new(None),
+            remaining_failures: std::sync::atomic::AtomicUsize::new(failures_before_success),
+            attempts: std::sync::atomic::AtomicUsize::new(0),
+            panic,
+            started: tokio::sync::Notify::new(),
+            release: block.then(tokio::sync::Notify::new),
+            completed: tokio::sync::Notify::new(),
+        })
     }
 
     async fn notify_for_user(&self, name: &str, is_temp: bool) {
@@ -1097,20 +1184,8 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        match Self::policy_plugin_state().await {
-            PolicyPluginState::Ready(_) => {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: true,
-                    mode: PreparedIamMode::Opa,
-                };
-            }
-            PolicyPluginState::Initializing | PolicyPluginState::Failed => {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: false,
-                    mode: PreparedIamMode::Deny,
-                };
-            }
-            PolicyPluginState::Disabled => {}
+        if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
+            return prepared;
         }
 
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
@@ -1773,6 +1848,8 @@ mod tests {
         sync::{Arc, Mutex},
     };
     use time::OffsetDateTime;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
 
     #[test]
     fn test_combined_policy_for_view_returns_regular_policy() {
@@ -1835,6 +1912,74 @@ mod tests {
         assert!(needs_secondary_tags, "OPA mode must request existing object tags for secondary actions");
     }
 
+    #[tokio::test]
+    async fn test_prepare_auth_denies_while_policy_plugin_is_unavailable() {
+        let store = StsTestMockStore::new(false);
+        let iam_sys = IamSys::new(IamCache::new(store).await.expect("initialize IAM cache"));
+        let claims = HashMap::new();
+        let groups = None;
+        let conditions = HashMap::new();
+        let args = Args {
+            account: "opa-unavailable-test-user",
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListAllMyBucketsAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        let mut outcomes = Vec::new();
+        for state in [PolicyPluginState::Initializing, PolicyPluginState::Failed] {
+            let prepared = state
+                .prepared_iam_auth()
+                .expect("unavailable policy plugin must prepare fail-closed IAM auth");
+            outcomes.push((
+                matches!(&prepared.mode, PreparedIamMode::Deny),
+                iam_sys.eval_prepared(&prepared, &args).await,
+            ));
+        }
+
+        assert_eq!(outcomes, [(true, false), (true, false)]);
+        assert!(PolicyPluginState::Disabled.prepared_iam_auth().is_none());
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_policy_plugin_state_fails_after_opa_validation_returns_503() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("bind OPA validation test listener");
+        let url = format!(
+            "http://{}/v1/data/rustfs/authz/allow",
+            listener.local_addr().expect("read listener address")
+        );
+        let server = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept OPA validation connection");
+            let mut request = [0_u8; 1024];
+            let bytes = stream.read(&mut request).await.expect("read OPA validation request");
+            assert!(bytes > 0, "OPA validation should send an HTTP request");
+            stream
+                .write_all(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .await
+                .expect("write OPA unavailable response");
+        });
+
+        let state = temp_env::async_with_vars(
+            [
+                ("RUSTFS_POLICY_PLUGIN_URL", Some(url.as_str())),
+                ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", None),
+            ],
+            resolve_policy_plugin_state(),
+        )
+        .await;
+        server.await.expect("join OPA validation test server");
+
+        assert!(matches!(state, PolicyPluginState::Failed));
+    }
+
     const CUSTOM_STS_CLAIM_POLICY: &str = "custom-sts-claim-getobject";
     const CUSTOM_STS_CLAIM_BUCKET: &str = "claim-bucket";
     const CUSTOM_STS_CLAIM_POLICY_JSON: &str = r#"{
@@ -1855,6 +2000,11 @@ mod tests {
         empty_policies: bool,
         saved_sts_users: Arc<Mutex<HashMap<String, UserIdentity>>>,
         saved_service_account_count: Arc<Mutex<usize>>,
+        fail_delete: Arc<std::sync::atomic::AtomicBool>,
+        deleted_mapped_policies: Arc<Mutex<Vec<(String, UserType)>>>,
+        block_delete: Arc<std::sync::atomic::AtomicBool>,
+        delete_started: Arc<tokio::sync::Notify>,
+        release_delete: Arc<tokio::sync::Notify>,
     }
 
     impl StsTestMockStore {
@@ -1863,6 +2013,11 @@ mod tests {
                 empty_policies,
                 saved_sts_users: Arc::new(Mutex::new(HashMap::new())),
                 saved_service_account_count: Arc::new(Mutex::new(0)),
+                fail_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                deleted_mapped_policies: Arc::new(Mutex::new(Vec::new())),
+                block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                delete_started: Arc::new(tokio::sync::Notify::new()),
+                release_delete: Arc::new(tokio::sync::Notify::new()),
             }
         }
 
@@ -1913,6 +2068,13 @@ mod tests {
         }
 
         async fn delete_user_identity(&self, name: &str, _user_type: UserType) -> Result<()> {
+            if self.block_delete.load(std::sync::atomic::Ordering::SeqCst) {
+                self.delete_started.notify_one();
+                self.release_delete.notified().await;
+            }
+            if self.fail_delete.load(std::sync::atomic::Ordering::SeqCst) {
+                return Err(Error::Io(std::io::Error::other("delete temporary account failed")));
+            }
             self.saved_sts_users
                 .lock()
                 .expect("saved_sts_users mutex poisoned")
@@ -1930,7 +2092,7 @@ mod tests {
         }
 
         async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
-            if name == "deleted-notify-user" {
+            if matches!(name, "deleted-notify-user" | "deleted-notify-sts") {
                 return Err(Error::NoSuchUser(name.to_string()));
             }
 
@@ -2008,7 +2170,11 @@ mod tests {
             Err(Error::InvalidArgument)
         }
 
-        async fn delete_mapped_policy(&self, _name: &str, _user_type: UserType, _is_group: bool) -> Result<()> {
+        async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
+            self.deleted_mapped_policies
+                .lock()
+                .expect("deleted_mapped_policies mutex poisoned")
+                .push((name.to_string(), user_type));
             Err(Error::InvalidArgument)
         }
 
@@ -3867,6 +4033,363 @@ mod tests {
         assert!(iam_sys.store.cache.snapshot().sts_policies.contains_key("notify-sts-parent"));
     }
 
+    #[tokio::test]
+    async fn delete_temp_account_notifies_peers_as_sts_user() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect("delete temporary account");
+            })
+            .await;
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "peer notification must retain the STS access key and user type"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_peer_invalidation_failure() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(STS_INVALIDATION_MAX_ATTEMPTS, false, false);
+
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                let result = iam_sys.delete_temp_account("deleted-notify-sts", true).await;
+                let err = result.expect_err("failed peer invalidation must fail STS revocation");
+                assert!(err.to_string().contains("peer notification failed"));
+            })
+            .await;
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "failed notification must retain the STS access key and user type"
+        );
+        assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
+    }
+
+    #[tokio::test]
+    async fn transient_peer_invalidation_retries_after_local_delete() {
+        const ACCESS_KEY: &str = "retryable-revoked-sts";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(2, false, false);
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), iam_sys.delete_temp_account(ACCESS_KEY, true))
+            .await
+            .expect("transient peer invalidation should converge within the retry budget");
+
+        assert_eq!(
+            probe.attempts.load(std::sync::atomic::Ordering::SeqCst),
+            STS_INVALIDATION_MAX_ATTEMPTS,
+            "peer invalidation must retry until it succeeds"
+        );
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some((ACCESS_KEY, true))
+        );
+    }
+
+    #[tokio::test]
+    async fn stalled_peer_invalidation_is_bounded_by_attempt_timeouts() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("stalled-peer-sts", true)
+                    .await
+                    .expect_err("stalled peer invalidation must fail after bounded attempts")
+            })
+            .await;
+
+        assert!(err.to_string().contains("peer STS invalidation timed out"));
+        assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_local_deletion_failure_without_notifying() {
+        let store = StsTestMockStore::new(false);
+        store.fail_delete.store(true, std::sync::atomic::Ordering::SeqCst);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect_err("local deletion failure must fail STS revocation")
+            })
+            .await;
+
+        assert!(err.to_string().contains("delete temporary account failed"));
+        assert!(
+            probe.observed.lock().expect("notification probe mutex poisoned").is_none(),
+            "peer invalidation must not run after local deletion fails"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_notification_task_panic() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, true);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect_err("notification task panic must fail STS revocation")
+            })
+            .await;
+
+        assert!(err.to_string().contains("panicked"));
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_notification_survives_caller_cancellation() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = Arc::new(IamSys::new(cache_manager));
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
+        let call = {
+            let iam_sys = Arc::clone(&iam_sys);
+            crate::LOAD_USER_NOTIFICATION_PROBE.scope(Arc::clone(&probe), async move {
+                iam_sys.delete_temp_account("deleted-notify-sts", true).await
+            })
+        };
+        let call = tokio::spawn(call);
+        probe.started.notified().await;
+
+        call.abort();
+        assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
+        probe
+            .release
+            .as_ref()
+            .expect("blocking probe must have a release signal")
+            .notify_one();
+        probe.completed.notified().await;
+
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "background peer invalidation must complete with the STS access key and user type"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_local_delete_survives_caller_cancellation() {
+        const ACCESS_KEY: &str = "cancelled-during-local-delete";
+
+        let store = StsTestMockStore::new(false);
+        store.saved_sts_users.lock().expect("saved_sts_users mutex poisoned").insert(
+            ACCESS_KEY.to_string(),
+            UserIdentity::from(Credentials {
+                access_key: ACCESS_KEY.to_string(),
+                secret_key: "temporary-user-secret".to_string(),
+                session_token: "session-token".to_string(),
+                status: ACCOUNT_ON.to_string(),
+                ..Default::default()
+            }),
+        );
+        store.block_delete.store(true, std::sync::atomic::Ordering::SeqCst);
+        let store_probe = store.clone();
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = Arc::new(IamSys::new(cache_manager));
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+        let call = {
+            let iam_sys = Arc::clone(&iam_sys);
+            crate::LOAD_USER_NOTIFICATION_PROBE
+                .scope(Arc::clone(&probe), async move { iam_sys.delete_temp_account(ACCESS_KEY, true).await })
+        };
+        let call = tokio::spawn(call);
+        store_probe.delete_started.notified().await;
+
+        call.abort();
+        assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
+        store_probe.release_delete.notify_one();
+        probe.completed.notified().await;
+
+        assert!(
+            !store_probe
+                .saved_sts_users
+                .lock()
+                .expect("saved_sts_users mutex poisoned")
+                .contains_key(ACCESS_KEY)
+        );
+    }
+
+    #[test]
+    fn notified_delete_without_tokio_runtime_returns_error() {
+        let runtime = tokio::runtime::Runtime::new().expect("create test runtime");
+        let iam_sys = runtime.block_on(async {
+            let store = StsTestMockStore::new(false);
+            let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+            IamSys::new(cache_manager)
+        });
+        drop(runtime);
+
+        let result = futures::executor::block_on(iam_sys.delete_temp_account("deleted-notify-sts", true));
+        let err = result.expect_err("notified deletion without a Tokio runtime must return an error");
+        assert!(err.to_string().contains("Tokio"));
+    }
+
+    #[tokio::test]
+    async fn notified_delete_without_notification_system_returns_error() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+
+        let err = iam_sys
+            .delete_temp_account("deleted-notify-sts", true)
+            .await
+            .expect_err("missing peer notification system must fail STS revocation");
+        assert!(err.to_string().contains("peer notification system is unavailable"));
+    }
+
+    #[tokio::test]
+    async fn missing_sts_notification_evicts_only_sts_cache_entry() {
+        const ACCESS_KEY: &str = "deleted-notify-sts";
+        const GROUP: &str = "deleted-notify-sts-group";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let regular_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "regular-user-secret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let sts_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "temporary-user-secret".to_string(),
+            session_token: "session-token".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let mapped_policy = MappedPolicy::new("readwrite");
+        let membership = HashSet::from([GROUP.to_string()]);
+        let group = GroupInfo::new(vec![ACCESS_KEY.to_string()]);
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(ACCESS_KEY, &regular_user, now);
+            cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
+            cache.add_or_update_group(GROUP, &group, now);
+            cache.add_or_update_user_group_membership(ACCESS_KEY, &membership, now);
+            cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
+        });
+
+        iam_sys
+            .load_user(ACCESS_KEY, UserType::Sts)
+            .await
+            .expect("process missing STS user notification");
+
+        let cache = iam_sys.store.cache.snapshot();
+        assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
+        assert!(
+            cache.users.contains_key(ACCESS_KEY),
+            "STS invalidation must not evict a same-name regular user"
+        );
+        assert!(cache.user_policies.contains_key(ACCESS_KEY));
+        assert!(cache.user_group_memberships.contains_key(ACCESS_KEY));
+        assert!(
+            cache
+                .groups
+                .get(GROUP)
+                .is_some_and(|group| group.members.contains(&ACCESS_KEY.to_string())),
+            "STS invalidation must preserve same-name regular-user group membership"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_preserves_same_name_regular_cache_state() {
+        const ACCESS_KEY: &str = "deleted-notify-sts";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let regular_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "regular-user-secret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let sts_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "temporary-user-secret".to_string(),
+            session_token: "session-token".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let mapped_policy = MappedPolicy::new("readwrite");
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(ACCESS_KEY, &regular_user, now);
+            cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
+            cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
+        });
+
+        iam_sys
+            .delete_temp_account(ACCESS_KEY, false)
+            .await
+            .expect("delete temporary account without peer notification");
+
+        let cache = iam_sys.store.cache.snapshot();
+        assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
+        assert!(cache.users.contains_key(ACCESS_KEY));
+        assert!(cache.user_policies.contains_key(ACCESS_KEY));
+        assert!(
+            iam_sys
+                .store
+                .api
+                .deleted_mapped_policies
+                .lock()
+                .expect("deleted_mapped_policies mutex poisoned")
+                .is_empty(),
+            "deleting one STS identity must not delete a parent-scoped STS policy mapping"
+        );
+    }
+
     #[tokio::test]
     async fn test_missing_user_notification_cleans_related_cache_state() {
         let store = StsTestMockStore::new(false);

crates/iam/tests/iam_bootstrap_no_lock_test.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![recursion_limit = "256"]
+
 //! Regression test for rustfs#4304: IAM bootstrap must not depend on the
 //! distributed namespace-lock quorum.
 //!

crates/io-metrics/src/internode_metrics.rs

@@ -27,6 +27,7 @@ pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner";
 pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
 pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
 pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple";
+pub const INTERNODE_OPERATION_GRPC_OTHER: &str = "grpc_other";
 pub const INTERNODE_TRANSPORT_BACKEND_TCP_HTTP: &str = "tcp-http";
 pub const INTERNODE_TRANSPORT_BACKEND_GRPC: &str = "grpc";
 pub const INTERNODE_TRANSPORT_BACKEND_UNKNOWN: &str = "unknown";
@@ -45,6 +46,9 @@ const CLASSIFICATION_LABEL: &str = "classification";
 const STAGE_LABEL: &str = "stage";
 const DOMINANT_ERROR_LABEL: &str = "dominant_error";
 const HTTP_VERSION_LABEL: &str = "http_version";
+const FAILURE_REASON_LABEL: &str = "failure_reason";
+const RPC_PATH_LABEL: &str = "rpc_path";
+const REASON_LABEL: &str = "reason";
 const DIRECTION_LABEL: &str = "direction";
 const MESSAGE_LABEL: &str = "message";
 const CODEC_LABEL: &str = "codec";
@@ -61,6 +65,7 @@ const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_int
 const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total";
 const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
     "rustfs_system_network_internode_operation_write_shutdown_errors_total";
+const INTERNODE_RPC_AUTH_FAILURES_TOTAL: &str = "rustfs_system_network_internode_rpc_auth_failures_total";
 const INTERNODE_OPERATION_PAYLOAD_BYTES: &str = "rustfs_system_network_internode_operation_payload_bytes";
 const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_internode_operation_large_payloads_total";
 const INTERNODE_MSGPACK_JSON_DECODE_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
@@ -70,6 +75,11 @@ const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
 const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
 const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
 const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
+const INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL: &str =
+    "rustfs_system_network_internode_replay_cache_overflow_by_operation_total";
+const INTERNODE_REPLAY_CACHE_ENTRIES: &str = "rustfs_system_network_internode_replay_cache_entries";
+const INTERNODE_REPLAY_CACHE_CAPACITY: &str = "rustfs_system_network_internode_replay_cache_capacity";
+const INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL: &str = "rustfs_system_network_internode_replay_cache_evictions_total";
 const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -82,6 +92,11 @@ const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL
 const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
     &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
 const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
+const SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS: &[&str] =
+    &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL];
+const SERVER_OPERATION_BACKEND_RPC_PATH_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL];
+const SERVER_LABELS: &[&str] = &[SERVER_LABEL];
+const SERVER_REASON_LABELS: &[&str] = &[SERVER_LABEL, REASON_LABEL];
 const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL];
 
 pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
@@ -133,6 +148,26 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
         name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
         labels: SERVER_OPERATION_BACKEND_LABELS,
     },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_RPC_AUTH_FAILURES_TOTAL,
+        labels: SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL,
+        labels: SERVER_OPERATION_BACKEND_RPC_PATH_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_REPLAY_CACHE_ENTRIES,
+        labels: SERVER_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_REPLAY_CACHE_CAPACITY,
+        labels: SERVER_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL,
+        labels: SERVER_REASON_LABELS,
+    },
     InternodeOperationMetricDescriptor {
         name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
         labels: SERVER_QUORUM_FAILURE_LABELS,
@@ -178,10 +213,14 @@ pub struct InternodeMetricsSnapshot {
     pub operation_http_versions_total: u64,
     pub operation_stall_timeouts_total: u64,
     pub operation_write_shutdown_errors_total: u64,
+    pub rpc_auth_failures_total: u64,
     pub signature_v1_fallback_total: u64,
     pub body_digest_fallback_total: u64,
     pub replay_scope_fallback_total: u64,
     pub replay_cache_overflow_total: u64,
+    pub replay_cache_entries: u64,
+    pub replay_cache_capacity: u64,
+    pub replay_cache_evictions_total: u64,
 }
 
 #[derive(Debug, Default)]
@@ -198,12 +237,20 @@ pub struct InternodeMetrics {
     operation_http_versions_total: AtomicU64,
     operation_stall_timeouts_total: AtomicU64,
     operation_write_shutdown_errors_total: AtomicU64,
+    rpc_auth_failures_total: AtomicU64,
     msgpack_json_decode_total: AtomicU64,
     msgpack_json_decode_error_total: AtomicU64,
     signature_v1_fallback_total: AtomicU64,
     body_digest_fallback_total: AtomicU64,
     replay_scope_fallback_total: AtomicU64,
     replay_cache_overflow_total: AtomicU64,
+    replay_cache_entries: AtomicU64,
+    replay_cache_capacity: AtomicU64,
+    replay_cache_evictions_total: AtomicU64,
+}
+
+fn usize_to_u64_saturating(value: usize) -> u64 {
+    u64::try_from(value).unwrap_or(u64::MAX)
 }
 
 impl InternodeMetrics {
@@ -423,6 +470,23 @@ impl InternodeMetrics {
         .increment(1);
     }
 
+    pub fn record_rpc_auth_failure_for_operation_and_backend(
+        &self,
+        operation: &'static str,
+        backend: &'static str,
+        failure_reason: &'static str,
+    ) {
+        self.rpc_auth_failures_total.fetch_add(1, Ordering::Relaxed);
+        counter!(
+            INTERNODE_RPC_AUTH_FAILURES_TOTAL,
+            SERVER_LABEL => current_server_label(),
+            OPERATION_LABEL => operation,
+            BACKEND_LABEL => backend,
+            FAILURE_REASON_LABEL => failure_reason
+        )
+        .increment(1);
+    }
+
     /// Record the payload size (bytes) of a completed internode operation into a histogram
     /// keyed by operation+backend. Used to size which unary `bytes`-carrying RPCs
     /// (`ReadAll`/`ReadMultiple`/`WriteAll`) would benefit from being moved off the shared
@@ -537,6 +601,46 @@ impl InternodeMetrics {
         counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
     }
 
+    pub fn record_replay_cache_overflow_for_operation_and_backend_path(
+        &self,
+        operation: &'static str,
+        backend: &'static str,
+        rpc_path: &str,
+    ) {
+        self.record_replay_cache_overflow();
+        counter!(
+            INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL,
+            SERVER_LABEL => current_server_label(),
+            OPERATION_LABEL => operation,
+            BACKEND_LABEL => backend,
+            RPC_PATH_LABEL => rpc_path.to_owned()
+        )
+        .increment(1);
+    }
+
+    pub fn record_replay_cache_state(&self, entries: usize, capacity: usize) {
+        let entries = usize_to_u64_saturating(entries);
+        let capacity = usize_to_u64_saturating(capacity);
+        self.replay_cache_entries.store(entries, Ordering::Relaxed);
+        self.replay_cache_capacity.store(capacity, Ordering::Relaxed);
+        gauge!(INTERNODE_REPLAY_CACHE_ENTRIES, SERVER_LABEL => current_server_label()).set(entries as f64);
+        gauge!(INTERNODE_REPLAY_CACHE_CAPACITY, SERVER_LABEL => current_server_label()).set(capacity as f64);
+    }
+
+    pub fn record_replay_cache_evictions(&self, reason: &'static str, count: usize) {
+        if count == 0 {
+            return;
+        }
+        let count = usize_to_u64_saturating(count);
+        self.replay_cache_evictions_total.fetch_add(count, Ordering::Relaxed);
+        counter!(
+            INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL,
+            SERVER_LABEL => current_server_label(),
+            REASON_LABEL => reason
+        )
+        .increment(count);
+    }
+
     pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
         counter!(
             ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
@@ -585,10 +689,14 @@ impl InternodeMetrics {
             operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
             operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
             operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
+            rpc_auth_failures_total: self.rpc_auth_failures_total.load(Ordering::Relaxed),
             signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
             body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
             replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
             replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
+            replay_cache_entries: self.replay_cache_entries.load(Ordering::Relaxed),
+            replay_cache_capacity: self.replay_cache_capacity.load(Ordering::Relaxed),
+            replay_cache_evictions_total: self.replay_cache_evictions_total.load(Ordering::Relaxed),
         }
     }
 
@@ -606,12 +714,16 @@ impl InternodeMetrics {
         self.operation_http_versions_total.store(0, Ordering::Relaxed);
         self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
         self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
+        self.rpc_auth_failures_total.store(0, Ordering::Relaxed);
         self.msgpack_json_decode_total.store(0, Ordering::Relaxed);
         self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
         self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
         self.body_digest_fallback_total.store(0, Ordering::Relaxed);
         self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
         self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
+        self.replay_cache_entries.store(0, Ordering::Relaxed);
+        self.replay_cache_capacity.store(0, Ordering::Relaxed);
+        self.replay_cache_evictions_total.store(0, Ordering::Relaxed);
     }
 }
 
@@ -778,7 +890,7 @@ mod tests {
     use super::*;
     use metrics::with_local_recorder;
     use metrics_util::debugging::DebuggingRecorder;
-    use std::collections::HashSet;
+    use std::collections::{HashMap, HashSet};
 
     #[test]
     fn snapshot_reports_recorded_values() {
@@ -829,6 +941,13 @@ mod tests {
             INTERNODE_TRANSPORT_BACKEND_GRPC,
         );
         metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
+        metrics.record_rpc_auth_failure_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_OTHER,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            "missing_v2_signature",
+        );
+        metrics.record_replay_cache_state(64, 1024);
+        metrics.record_replay_cache_evictions("expired", 3);
 
         let snapshot = metrics.snapshot();
         assert_eq!(snapshot.sent_bytes_total, 128);
@@ -836,11 +955,15 @@ mod tests {
         assert_eq!(snapshot.outgoing_requests_total, 1);
         assert_eq!(snapshot.incoming_requests_total, 1);
         assert_eq!(snapshot.errors_total, 1);
+        assert_eq!(snapshot.rpc_auth_failures_total, 1);
+        assert_eq!(snapshot.replay_cache_entries, 64);
+        assert_eq!(snapshot.replay_cache_capacity, 1024);
+        assert_eq!(snapshot.replay_cache_evictions_total, 3);
     }
 
     #[test]
     fn operation_metric_descriptors_include_backend_and_operation_labels() {
-        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 15);
+        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 20);
         for metric in &INTERNODE_OPERATION_METRICS[..6] {
             assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
         }
@@ -854,10 +977,22 @@ mod tests {
         for metric in &INTERNODE_OPERATION_METRICS[10..12] {
             assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
         }
-        assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[12].labels,
+            &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[13].labels,
+            &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
+        );
+        for metric in &INTERNODE_OPERATION_METRICS[14..16] {
+            assert_eq!(metric.labels, &[SERVER_LABEL]);
+        }
+        assert_eq!(INTERNODE_OPERATION_METRICS[16].labels, &[SERVER_LABEL, REASON_LABEL]);
+        assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
         // Payload histogram + large-payload counter carry operation+backend labels.
-        assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
-        assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
+        assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
+        assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
     }
 
     #[test]
@@ -867,6 +1002,7 @@ mod tests {
         assert_eq!(INTERNODE_OPERATION_WALK_DIR, "walk_dir");
         assert_eq!(INTERNODE_OPERATION_GRPC_READ_ALL, "grpc_read_all");
         assert_eq!(INTERNODE_OPERATION_GRPC_WRITE_ALL, "grpc_write_all");
+        assert_eq!(INTERNODE_OPERATION_GRPC_OTHER, "grpc_other");
 
         assert_eq!(INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, "tcp-http");
         assert_eq!(INTERNODE_TRANSPORT_BACKEND_GRPC, "grpc");
@@ -902,14 +1038,34 @@ mod tests {
         );
         assert_eq!(
             INTERNODE_OPERATION_METRICS[12].name,
-            "rustfs_system_storage_erasure_write_quorum_failures_total"
+            "rustfs_system_network_internode_rpc_auth_failures_total"
         );
         assert_eq!(
             INTERNODE_OPERATION_METRICS[13].name,
-            "rustfs_system_network_internode_operation_payload_bytes"
+            "rustfs_system_network_internode_replay_cache_overflow_by_operation_total"
         );
         assert_eq!(
             INTERNODE_OPERATION_METRICS[14].name,
+            "rustfs_system_network_internode_replay_cache_entries"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[15].name,
+            "rustfs_system_network_internode_replay_cache_capacity"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[16].name,
+            "rustfs_system_network_internode_replay_cache_evictions_total"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[17].name,
+            "rustfs_system_storage_erasure_write_quorum_failures_total"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[18].name,
+            "rustfs_system_network_internode_operation_payload_bytes"
+        );
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS[19].name,
             "rustfs_system_network_internode_operation_large_payloads_total"
         );
         assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple");
@@ -933,6 +1089,94 @@ mod tests {
             INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
             "rustfs_system_network_internode_signature_v1_fallback_total"
         );
+        assert_eq!(FAILURE_REASON_LABEL, "failure_reason");
+        assert_eq!(RPC_PATH_LABEL, "rpc_path");
+        assert_eq!(REASON_LABEL, "reason");
+    }
+
+    #[test]
+    fn rpc_auth_failure_counter_records_low_cardinality_labels() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let metrics = InternodeMetrics::default();
+
+        with_local_recorder(&recorder, || {
+            metrics.record_rpc_auth_failure_for_operation_and_backend(
+                INTERNODE_OPERATION_GRPC_READ_ALL,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                "invalid_v2_signature",
+            );
+        });
+
+        assert_eq!(metrics.snapshot().rpc_auth_failures_total, 1);
+        let entries: Vec<_> = snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_RPC_AUTH_FAILURES_TOTAL)
+            .collect();
+        assert_eq!(entries.len(), 1);
+        let labels: HashMap<_, _> = entries[0]
+            .0
+            .key()
+            .labels()
+            .map(|label| (label.key().to_string(), label.value().to_string()))
+            .collect();
+        assert_eq!(labels.get(OPERATION_LABEL).map(String::as_str), Some(INTERNODE_OPERATION_GRPC_READ_ALL));
+        assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
+        assert_eq!(labels.get(FAILURE_REASON_LABEL).map(String::as_str), Some("invalid_v2_signature"));
+        assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
+    }
+
+    #[test]
+    fn replay_cache_metrics_record_state_eviction_and_overflow_scope() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let metrics = InternodeMetrics::default();
+
+        with_local_recorder(&recorder, || {
+            metrics.record_replay_cache_state(7, 11);
+            metrics.record_replay_cache_evictions("expired", 5);
+            metrics.record_replay_cache_overflow_for_operation_and_backend_path(
+                INTERNODE_OPERATION_GRPC_READ_ALL,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                "/node_service.NodeService/ReadAll",
+            );
+        });
+
+        let snapshot = metrics.snapshot();
+        assert_eq!(snapshot.replay_cache_entries, 7);
+        assert_eq!(snapshot.replay_cache_capacity, 11);
+        assert_eq!(snapshot.replay_cache_evictions_total, 5);
+        assert_eq!(snapshot.replay_cache_overflow_total, 1);
+
+        let entries: Vec<_> = snapshotter.snapshot().into_vec();
+        assert!(
+            entries
+                .iter()
+                .any(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_ENTRIES)
+        );
+        assert!(
+            entries
+                .iter()
+                .any(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_CAPACITY)
+        );
+
+        let overflow: Vec<_> = entries
+            .iter()
+            .filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL)
+            .collect();
+        assert_eq!(overflow.len(), 1);
+        let labels: HashMap<_, _> = overflow[0]
+            .0
+            .key()
+            .labels()
+            .map(|label| (label.key().to_string(), label.value().to_string()))
+            .collect();
+        assert_eq!(labels.get(OPERATION_LABEL).map(String::as_str), Some(INTERNODE_OPERATION_GRPC_READ_ALL));
+        assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
+        assert_eq!(labels.get(RPC_PATH_LABEL).map(String::as_str), Some("/node_service.NodeService/ReadAll"));
+        assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
     }
 
     #[test]

crates/kms/src/backends/aws.rs

@@ -585,13 +585,16 @@ impl KmsBackend for AwsKmsBackend {
         // AWS rejects a `Limit` of zero, and clamping it up to one would return
         // a key to a caller that asked for none; the empty page is answered
         // here instead.
-        if list_keys_page_size(request.limit).is_none() {
+        let Some(page_size) = list_keys_page_size(request.limit) else {
             return Ok(empty_key_page());
-        }
+        };
 
+        // Taking the remote page size from the shared resolver keeps the AWS
+        // request under the same ceiling every other backend obeys; the AWS API
+        // maximum is the same 1000, so this never widens the remote page.
         let limit = request
             .limit
-            .map(|limit| i32::try_from(limit).unwrap_or(i32::MAX).clamp(1, 1000));
+            .map(|_| i32::try_from(page_size).unwrap_or(i32::MAX).clamp(1, 1000));
         let marker = request.marker.clone();
 
         let output = self
@@ -617,7 +620,17 @@ impl KmsBackend for AwsKmsBackend {
             let Some(key_id) = entry.key_id() else {
                 continue;
             };
-            let metadata = self.describe(key_id).await?;
+            let metadata = match self.describe(key_id).await {
+                Ok(metadata) => metadata,
+                // AWS `ListKeys` is eventually consistent, so a key destroyed
+                // between the listing and the describe is routine: it is
+                // dropped and the remote cursor still advances past it. There
+                // is no local record to be damaged here — key state lives in
+                // AWS — so `unreadable_key_ids` stays empty on this backend and
+                // every other failure fails the listing.
+                Err(KmsError::KeyNotFound { .. }) => continue,
+                Err(error) => return Err(error),
+            };
             if request
                 .usage_filter
                 .as_ref()
@@ -642,6 +655,8 @@ impl KmsBackend for AwsKmsBackend {
                 created_at: metadata.creation_date,
                 rotated_at: None,
                 created_by: None,
+                rotation_due: false,
+                rotation_due_reason: None,
             });
         }
 
@@ -649,6 +664,8 @@ impl KmsBackend for AwsKmsBackend {
             keys,
             next_marker: output.next_marker.clone(),
             truncated: output.truncated,
+            // AWS owns key state; nothing here can be present-but-unreadable.
+            unreadable_key_ids: Vec::new(),
         })
     }
 

crates/kms/src/backends/contract_tests.rs

@@ -183,6 +183,7 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) {
         .await
         .expect("decrypt with a disabled key must keep working");
     assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key");
+    assert_eq!(decrypted.key_id, key_id, "decrypt must report the master key that opened the envelope");
     // ...disable stays idempotent, cancel has nothing to cancel, and enable recovers.
     backend.disable_key(key_id).await.expect("disable must be idempotent");
     expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion");

crates/kms/src/backends/local.rs

@@ -15,8 +15,8 @@
 //! Local file-based KMS backend implementation
 
 use crate::backends::{
-    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
-    ensure_tag_keys_are_mutable, paginate_keys,
+    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, ListedKeyFailure, StateGatedOperation, UnreadableKeys,
+    classify_listed_key_failure, ensure_key_status_permits, ensure_tag_keys_are_mutable, paginate_keys, started_at_the_first_key,
 };
 use crate::config::KmsConfig;
 use crate::config::LocalConfig;
@@ -121,6 +121,16 @@ pub(crate) fn is_orphan_commit_temp_name(file_name: &str) -> bool {
     !prefix.is_empty() && suffix.len() == 36 && uuid::Uuid::try_parse(suffix).is_ok()
 }
 
+/// Mode every key-directory file is written with when the deployment does not
+/// name one.
+///
+/// `file_permissions` is optional in the persisted configuration and stays
+/// optional for compatibility, but "unspecified" must not mean "whatever the
+/// umask says": a `0` umask — the default in a good many container images —
+/// would publish master key records world-readable. Owner-only is the only
+/// defensible reading of an absent value for a file holding key material.
+pub(crate) const DEFAULT_KEY_FILE_MODE: u32 = 0o600;
+
 /// Durable single-file commit protocol for the key directory.
 ///
 /// Key material and metadata are unrecoverable state, so every mutation of the
@@ -253,6 +263,15 @@ pub(crate) mod durable_file {
         .map_err(io::Error::other)?
     }
 
+    /// The mode a key-directory file is published with.
+    ///
+    /// Exposed so the "absent means owner-only" rule can be asserted directly:
+    /// observing it through a written file only proves anything on a host whose
+    /// umask is not already masking the same bits.
+    pub(crate) fn resolved_file_mode(permissions: Option<u32>) -> u32 {
+        permissions.unwrap_or(super::DEFAULT_KEY_FILE_MODE)
+    }
+
     fn commit_blocking(
         temp_path: &Path,
         final_path: &Path,
@@ -260,6 +279,11 @@ pub(crate) mod durable_file {
         permissions: Option<u32>,
         publish: &Publish,
     ) -> Result<(), CommitError> {
+        // Resolved here rather than at each call site so no caller can publish
+        // a key-directory file at the umask's mercy by leaving the mode unset —
+        // the backup restore path did exactly that, and its files landed in the
+        // same directory as records written owner-only by every other path.
+        let permissions = Some(resolved_file_mode(permissions));
         let file = open_temp_exclusive(temp_path, permissions)?;
         match run_protocol(file, temp_path, final_path, content, permissions, publish) {
             // A simulated crash must leave the directory exactly as a real one
@@ -779,9 +803,10 @@ impl LocalKmsClient {
     pub async fn new(config: LocalConfig) -> Result<Self> {
         // Create key directory if it doesn't exist
         if !fs::try_exists(&config.key_dir).await? {
-            fs::create_dir_all(&config.key_dir).await?;
+            Self::create_key_dir(&config.key_dir).await?;
             debug!(path = ?config.key_dir, "KMS key directory created");
         }
+        Self::secure_key_dir(&config.key_dir).await?;
 
         // The restore-marker guard must run before anything else touches the
         // directory (in particular before salt load/creation): a directory
@@ -908,6 +933,95 @@ impl LocalKmsClient {
     /// only valid next steps are re-running the restore with the same bundle
     /// (roll forward) or explicitly aborting it. This mirrors the missing-salt
     /// guard: startup must never paper over a half-applied restore.
+    /// Create the key directory, and every directory leading to it, owner-only.
+    ///
+    /// `DirBuilder::mode` applies to each directory the recursive create makes,
+    /// so an intermediate component cannot be left at the umask's mercy, and it
+    /// closes the window a create-then-chmod pair leaves open — during which
+    /// the directory exists at whatever the umask allowed.
+    async fn create_key_dir(key_dir: &Path) -> Result<()> {
+        let key_dir = key_dir.to_path_buf();
+        tokio::task::spawn_blocking(move || {
+            let mut builder = std::fs::DirBuilder::new();
+            builder.recursive(true);
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::DirBuilderExt;
+                builder.mode(Self::KEY_DIR_MODE);
+            }
+            builder.create(&key_dir)
+        })
+        .await
+        .map_err(|error| KmsError::internal_error(format!("key directory creation task failed: {error}")))??;
+        Ok(())
+    }
+
+    /// Mode the key directory is held at: owner-only.
+    #[cfg(unix)]
+    pub(crate) const KEY_DIR_MODE: u32 = 0o700;
+
+    /// Bring the key directory down to owner-only, and keep it there.
+    ///
+    /// A directory wider than owner-only is rarely a decision anyone made. The
+    /// platform picks it: kubelet creates an `emptyDir` `0o777`, several PVC
+    /// provisioners `mkdir -m 0777`, a `--tmpfs` mount lands at `1777`, and
+    /// `create_dir_all` under a container's `0` umask does the same. Write
+    /// access here is the power to delete a key — destroying every object it
+    /// protects — or to plant a record for a key id that does not exist yet.
+    ///
+    /// So this narrows rather than refuses, matching how the observability
+    /// stack already treats its own directory (`ensure_dir_permissions`).
+    /// Refusing would turn every one of those platform defaults into a server
+    /// that will not start — `init_kms_system` propagates out of startup — and
+    /// would leave the exposure in place on the way out. Narrowing removes it.
+    /// Only a directory this process cannot secure is fatal: at that point the
+    /// mode is both dangerous and outside our control, and proceeding would
+    /// write key material into it anyway.
+    async fn secure_key_dir(key_dir: &Path) -> Result<()> {
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            // The full mode, sticky bit included: reporting `0o777` for a
+            // `1777` directory sends an operator looking for something
+            // `ls -ld` does not show.
+            let before = fs::metadata(key_dir).await?.permissions().mode() & 0o7777;
+            if before == Self::KEY_DIR_MODE {
+                return Ok(());
+            }
+
+            if let Err(error) = fs::set_permissions(key_dir, std::fs::Permissions::from_mode(Self::KEY_DIR_MODE)).await {
+                return Err(KmsError::configuration_error(format!(
+                    "Local KMS key directory {} has mode {before:#o} and cannot be narrowed to {:#o} ({error}); key material must not be written into a directory this process cannot secure",
+                    key_dir.display(),
+                    Self::KEY_DIR_MODE
+                )));
+            }
+
+            // Verified rather than assumed: a filesystem that ignores `chmod`
+            // would otherwise leave the directory wide open behind a log line
+            // saying it had been narrowed.
+            let after = fs::metadata(key_dir).await?.permissions().mode() & 0o7777;
+            if after != Self::KEY_DIR_MODE {
+                return Err(KmsError::configuration_error(format!(
+                    "Local KMS key directory {} is still mode {after:#o} after being set to {:#o}; this filesystem does not enforce permissions and must not hold key material",
+                    key_dir.display(),
+                    Self::KEY_DIR_MODE
+                )));
+            }
+
+            if before & 0o077 != 0 {
+                warn!(
+                    path = ?key_dir,
+                    previous_mode = format!("{before:#o}"),
+                    "Local KMS key directory was reachable beyond its owner and has been narrowed to 0o700"
+                );
+            }
+        }
+        #[cfg(not(unix))]
+        let _ = key_dir;
+        Ok(())
+    }
+
     async fn ensure_no_restore_marker(config: &LocalConfig) -> Result<()> {
         let marker = config.key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE);
         if fs::try_exists(&marker).await? {
@@ -1359,7 +1473,13 @@ impl LocalKmsClient {
                 key_ids.push(key_id.to_string());
                 continue;
             }
-            if entry.file_type().await?.is_file()
+            // `file_type` does not follow symlinks, and a symlink is exactly
+            // as much an orphan of this protocol as a regular file is: the
+            // commit protocol only ever creates temps with `create_new`, so
+            // anything wearing a temp name is either our own leftover or
+            // something planted, and neither belongs in the key directory.
+            // Requiring `is_file` left symlinked temp names behind forever.
+            if !entry.file_type().await?.is_dir()
                 && let Some(file_name) = path.file_name().and_then(|name| name.to_str())
                 && is_orphan_commit_temp_name(file_name)
             {
@@ -1409,14 +1529,7 @@ impl LocalKmsClient {
         ensure_key_status_permits(&request.master_key_id, &key_info.status, StateGatedOperation::GenerateDataKey)?;
 
         // Generate random data key material
-        let key_length = match request.key_spec.as_str() {
-            "AES_256" => 32,
-            "AES_128" => 16,
-            _ => return Err(KmsError::unsupported_algorithm(&request.key_spec)),
-        };
-
-        let mut plaintext_key = vec![0u8; key_length];
-        rand::rng().fill(&mut plaintext_key[..]);
+        let plaintext_key = generate_key_material(&request.key_spec)?;
 
         // Encrypt the data key with the master key
         let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.master_key_id, &plaintext_key).await?;
@@ -1476,11 +1589,19 @@ impl LocalKmsClient {
         })
     }
 
-    pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
+    /// Open a data-key envelope, returning the plaintext and the master key
+    /// that wrapped it.
+    pub(crate) async fn decrypt(
+        &self,
+        request: &DecryptRequest,
+        _context: Option<&OperationContext>,
+    ) -> Result<(Vec<u8>, String)> {
         debug!("Decrypting data");
 
-        // Parse the data key envelope from ciphertext
-        let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?;
+        // Parse the data key envelope from ciphertext. Mapped to the same
+        // error class the other backends report for unparseable ciphertext.
+        let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
+            .map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?;
 
         // NOTE: this comparison is an authorization check, not a cryptographic
         // binding. `DekCrypto` seals only the plaintext, so `encryption_context`
@@ -1514,7 +1635,7 @@ impl LocalKmsClient {
             .await?;
 
         debug!("Local KMS data decrypted");
-        Ok(plaintext)
+        Ok((plaintext, envelope.master_key_id))
     }
 
     /// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
@@ -1599,22 +1720,25 @@ impl LocalKmsClient {
         // Only the page is read from disk, so the cost of a list stays bounded
         // by the requested limit rather than by the size of the key set.
         let mut keys = Vec::with_capacity(page.items.len());
+        let mut unreadable = UnreadableKeys::default();
         for key_id in page.items {
             let key_info = match self.describe_key(key_id, None).await {
-                Ok(key_info) => key_info,
-                // A key that vanished between the scan and the read is dropped
-                // from the page: concurrent removal is normal, and the cursor
-                // is derived from the identifier list, so the listing still
-                // advances past it.
-                Err(KmsError::KeyNotFound { .. }) => {
-                    debug!(key_id, "skipping key removed while listing");
-                    continue;
+                Ok(key_info) => {
+                    unreadable.saw_readable();
+                    key_info
                 }
-                // Anything else means the record is still there and this build
-                // cannot interpret it. Dropping it would answer "these are
-                // your keys" with a set that silently omits one, and the
-                // deletion sweep would count a census it never fully saw.
-                Err(error) => return Err(error),
+                Err(error) => match classify_listed_key_failure(&error) {
+                    Some(ListedKeyFailure::Vanished) => {
+                        debug!(key_id, "skipping key removed while listing");
+                        continue;
+                    }
+                    Some(ListedKeyFailure::Unreadable) => {
+                        warn!(key_id, %error, "listing a key record this build cannot describe");
+                        unreadable.record(key_id, error);
+                        continue;
+                    }
+                    None => return Err(error),
+                },
             };
 
             if let Some(ref status_filter) = request.status_filter
@@ -1635,6 +1759,7 @@ impl LocalKmsClient {
             keys,
             next_marker: page.next_marker,
             truncated: page.truncated,
+            unreadable_key_ids: unreadable.into_reported_ids(!page.truncated && started_at_the_first_key(request))?,
         })
     }
 
@@ -1870,16 +1995,11 @@ impl KmsBackend for LocalKmsBackend {
     }
 
     async fn decrypt(&self, request: DecryptRequest) -> Result<DecryptResponse> {
-        let plaintext = self.client.decrypt(&request, None).await?;
-
-        // The envelope that was just opened names the master key that opened it.
-        // Reporting "unknown" left every caller unable to tell which key was
-        // actually used, which is what audit and key-rotation checks read.
-        let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?;
+        let (plaintext, key_id) = self.client.decrypt(&request, None).await?;
 
         Ok(DecryptResponse {
             plaintext,
-            key_id: envelope.master_key_id,
+            key_id,
             encryption_algorithm: Some("AES-256-GCM".to_string()),
         })
     }
@@ -1893,12 +2013,19 @@ impl KmsBackend for LocalKmsBackend {
             grant_tokens: Vec::new(),
         };
 
-        let data_key = self.client.generate_data_key(&generate_request, None).await?;
+        let mut data_key = self.client.generate_data_key(&generate_request, None).await?;
 
+        // Fields are taken, not destructured or cloned: `DataKeyInfo` has a
+        // `Drop` impl, and a clone would leave a second un-zeroized plaintext
+        // DEK on the heap.
+        let plaintext_key = data_key
+            .plaintext
+            .take()
+            .ok_or_else(|| KmsError::internal_error("Generated data key is missing plaintext"))?;
         Ok(GenerateDataKeyResponse {
             key_id: request.key_id,
-            plaintext_key: data_key.plaintext.clone().unwrap_or_default(),
-            ciphertext_blob: data_key.ciphertext.clone(),
+            plaintext_key,
+            ciphertext_blob: std::mem::take(&mut data_key.ciphertext),
         })
     }
 
@@ -2299,8 +2426,9 @@ mod tests {
         let decrypt_request =
             DecryptRequest::new(data_key.ciphertext.clone()).with_context("bucket".to_string(), "test-bucket".to_string());
 
-        let decrypted = client.decrypt(&decrypt_request, None).await.expect("Failed to decrypt");
+        let (decrypted, opened_by) = client.decrypt(&decrypt_request, None).await.expect("Failed to decrypt");
         assert_eq!(decrypted, data_key.plaintext.clone().expect("No plaintext"));
+        assert_eq!(opened_by, key_id, "decrypt must report the master key that opened the envelope");
     }
 
     #[tokio::test]
@@ -2331,7 +2459,7 @@ mod tests {
         // Pre-fix, each of those regenerated the master key, so this unwrap fails with an AEAD
         // error. Post-fix, the original material is preserved and the DEK still decrypts.
         let decrypt_request = DecryptRequest::new(ciphertext).with_context("bucket".to_string(), "b".to_string());
-        let decrypted = client
+        let (decrypted, _opened_by) = client
             .decrypt(&decrypt_request, None)
             .await
             .expect("DEK must still decrypt after status transitions");
@@ -2672,7 +2800,7 @@ mod tests {
         assert!(matches!(error, KmsError::InvalidOperation { .. }));
 
         for (index, (ciphertext, plaintext)) in batch.iter().enumerate() {
-            let decrypted = client
+            let (decrypted, _opened_by) = client
                 .decrypt(&DecryptRequest::new(ciphertext.clone()), None)
                 .await
                 .unwrap_or_else(|error| panic!("batch member {index} must decrypt: {error}"));
@@ -3625,8 +3753,13 @@ mod tests {
     /// key that is still on disk, and the deletion sweep — which counts the
     /// lifecycle gauges out of the pages it lists — would report a census it
     /// never fully saw as complete.
+    ///
+    /// It must not fail the whole listing either: one damaged record would then
+    /// stop every readable key from ever being listed, and with it every
+    /// scheduled deletion on this node. The identifier is reported alongside the
+    /// keys that did read, so the page is honest and the caller still advances.
     #[tokio::test]
-    async fn list_keys_fails_closed_on_a_record_it_cannot_interpret() {
+    async fn list_keys_reports_a_record_it_cannot_interpret_without_dropping_it() {
         let (client, _temp_dir) = create_test_client().await;
         client.create_key("alpha", "AES_256", None).await.expect("create alpha");
         client.create_key("beta", "AES_256", None).await.expect("create beta");
@@ -3641,10 +3774,27 @@ mod tests {
             .await
             .expect("write record");
 
-        let error = client
+        let response = client
             .list_keys(&ListKeysRequest::default(), None)
             .await
-            .expect_err("a listing must not quietly omit a key it cannot read");
+            .expect("one unreadable record must not fail the whole listing");
+        assert_eq!(
+            response.keys.iter().map(|key| key.key_id.as_str()).collect::<Vec<_>>(),
+            vec!["alpha"],
+            "the readable key must still be listed"
+        );
+        assert_eq!(
+            response.unreadable_key_ids,
+            vec!["beta".to_string()],
+            "a key this build cannot read must be named, not quietly omitted"
+        );
+
+        // Describing it directly still fails closed with the typed error, and
+        // the raw marker value stays out of the message.
+        let error = client
+            .describe_key("beta", None)
+            .await
+            .expect_err("describe must fail closed");
         assert!(
             matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
                 if key_id == "beta" && version == UNKNOWN_STORED_KEY_PROTECTION),
@@ -3653,6 +3803,110 @@ mod tests {
         assert!(!error.to_string().contains("secret-marker-value-must-not-leak"));
     }
 
+    /// Per-key attribution is only honest while some key on the page reads.
+    ///
+    /// When none does, the cause is almost certainly shared — a node reading
+    /// records written in a format it has no reader for, or a policy that
+    /// denies the whole subtree — and answering `200 OK` with an empty `keys`
+    /// list is indistinguishable, to every client that predates
+    /// `unreadable_key_ids`, from a deployment that simply has no keys. The
+    /// operator response to that is to provision a new key, which is the
+    /// destructive move the fail-closed rules exist to prevent.
+    #[tokio::test]
+    async fn a_page_whose_keys_are_all_unreadable_fails_instead_of_looking_empty() {
+        let (client, _temp_dir) = create_test_client().await;
+        for key_id in ["alpha", "beta"] {
+            client.create_key(key_id, "AES_256", None).await.expect("create key");
+            let key_path = client.master_key_path(key_id).expect("valid key id");
+            let mut record: serde_json::Value =
+                serde_json::from_slice(&fs::read(&key_path).await.expect("read record")).expect("decode record");
+            record["at_rest_protection"] = serde_json::json!({ "future_mode": ["opaque"] });
+            fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode record"))
+                .await
+                .expect("write record");
+        }
+
+        let error = client
+            .list_keys(&ListKeysRequest::default(), None)
+            .await
+            .expect_err("a page with nothing readable must fail, not report an empty key set");
+        assert!(
+            matches!(&error, KmsError::UnsupportedFormatVersion { .. }),
+            "the failure must name what went wrong: {error:?}"
+        );
+
+        // An empty marker is not the same as no marker to a caller, but it is
+        // to the pager: both start at the first key. A generated client that
+        // always emits its cursor parameter must not fall through the guard.
+        let error = client
+            .list_keys(
+                &ListKeysRequest {
+                    marker: Some(String::new()),
+                    ..Default::default()
+                },
+                None,
+            )
+            .await
+            .expect_err("an empty marker starts at the first key and must not bypass the guard");
+        assert!(matches!(&error, KmsError::UnsupportedFormatVersion { .. }), "got {error:?}");
+    }
+
+    /// The all-unreadable guard must never become a cursor trap.
+    ///
+    /// It only fires for a listing that both started at the beginning and
+    /// reached the end, because such a page has no successor to advance to.
+    /// Applying it per page instead would mean a caller with `limit=1` gets a
+    /// failure — and a failure carries no `next_marker` — the moment its page
+    /// lands on the damaged key, leaving every key behind it permanently
+    /// unreachable.
+    #[tokio::test]
+    async fn a_damaged_key_never_blocks_paging_past_it() {
+        let (client, _temp_dir) = create_test_client().await;
+        for key_id in ["a-first", "b-damaged", "c-last"] {
+            client.create_key(key_id, "AES_256", None).await.expect("create key");
+        }
+        let key_path = client.master_key_path("b-damaged").expect("valid key id");
+        let mut record: serde_json::Value =
+            serde_json::from_slice(&fs::read(&key_path).await.expect("read record")).expect("decode record");
+        record["at_rest_protection"] = serde_json::json!({ "future_mode": ["opaque"] });
+        fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode record"))
+            .await
+            .expect("write record");
+
+        // Walk the whole key set one key at a time, exactly as a client that
+        // pages until `truncated` is false would.
+        let mut marker = None;
+        let mut seen = Vec::new();
+        let mut reported_unreadable = Vec::new();
+        loop {
+            let page = client
+                .list_keys(
+                    &ListKeysRequest {
+                        limit: Some(1),
+                        marker: marker.clone(),
+                        ..Default::default()
+                    },
+                    None,
+                )
+                .await
+                .expect("a one-key page containing the damaged key must still be answerable");
+            seen.extend(page.keys.iter().map(|key| key.key_id.clone()));
+            reported_unreadable.extend(page.unreadable_key_ids.clone());
+            if !page.truncated {
+                break;
+            }
+            marker = page.next_marker;
+            assert!(marker.is_some(), "a truncated page must carry a cursor");
+        }
+
+        assert_eq!(
+            seen,
+            vec!["a-first".to_string(), "c-last".to_string()],
+            "paging must reach past the damage"
+        );
+        assert_eq!(reported_unreadable, vec!["b-damaged".to_string()]);
+    }
+
     #[tokio::test]
     async fn missing_salt_with_pre_marker_key_records_still_initializes() {
         let (dev_client, temp_dir) = create_dev_mode_client().await;
@@ -4035,4 +4289,238 @@ mod tests {
         assert!(!response.truncated);
         assert!(response.next_marker.is_none());
     }
+
+    // -----------------------------------------------------------------------
+    // Filesystem boundaries (rustfs/backlog#1562 P0.4).
+    //
+    // The commit protocol's durability argument rests on properties of the
+    // filesystem it runs on, and those properties are assumptions until
+    // something exercises them. Each test below pins one boundary the protocol
+    // depends on. Two boundaries in that item cannot be closed here and are
+    // recorded in `docs/operations/kms-backend-security.md` instead: a real
+    // cross-device rename needs a second filesystem (root or a privileged
+    // container), and detecting a key directory swapped between `rename` and
+    // the parent `fsync` needs directory file descriptors the protocol does not
+    // yet hold.
+    // -----------------------------------------------------------------------
+
+    #[cfg(unix)]
+    fn mode_of(path: &Path) -> u32 {
+        use std::os::unix::fs::PermissionsExt;
+        std::fs::metadata(path).expect("stat").permissions().mode() & 0o777
+    }
+
+    /// Every file the protocol publishes carries the requested mode, and the
+    /// umask cannot widen it — the mode is applied and re-read on the open file
+    /// before the content becomes durable, not left to the creation mask.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn published_files_carry_the_requested_mode_regardless_of_umask() {
+        let (client, _temp_dir) = create_test_client().await;
+        client.create_key("mode-key", "AES_256", None).await.expect("create key");
+
+        assert_eq!(mode_of(&client.master_key_path("mode-key").expect("valid key id")), 0o600);
+        let salt_path = LocalKmsClient::master_key_salt_path(&client.config);
+        assert_eq!(mode_of(&salt_path), 0o600);
+    }
+
+    /// The key directory ends up owner-only whoever created it and whatever
+    /// mode they left it at.
+    ///
+    /// The platform picks that mode far more often than an operator does:
+    /// kubelet creates an `emptyDir` `0o777`, several PVC provisioners
+    /// `mkdir -m 0777`, and a `--tmpfs` mount lands at `1777`. Each is a
+    /// directory holding every master key that any account on the host can
+    /// delete from. The modes below are exercised explicitly rather than left
+    /// to the umask, so the test keeps its teeth on a host with any umask.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn the_key_directory_is_narrowed_to_owner_only_whatever_it_was() {
+        use std::os::unix::fs::PermissionsExt;
+
+        // Created by this process, including the intermediate component.
+        let temp_dir = TempDir::new().expect("temp dir");
+        let key_dir = temp_dir.path().join("nested").join("keys");
+        assert!(!key_dir.exists());
+        let client = LocalKmsClient::new(LocalConfig {
+            key_dir: key_dir.clone(),
+            master_key: Some("test-master-key".to_string()),
+            file_permissions: Some(0o600),
+        })
+        .await
+        .expect("client must create its key directory");
+        client.create_key("nested-key", "AES_256", None).await.expect("create key");
+        assert_eq!(mode_of(&key_dir), 0o700);
+        assert_eq!(
+            mode_of(&temp_dir.path().join("nested")),
+            0o700,
+            "an intermediate directory must not be left at the umask's mercy"
+        );
+        drop(client);
+
+        // Placed by the platform at each mode a real one actually produces,
+        // sticky bit included.
+        for mode in [0o777, 0o1777, 0o770, 0o755] {
+            let placed = TempDir::new().expect("temp dir");
+            std::fs::set_permissions(placed.path(), std::fs::Permissions::from_mode(mode)).expect("widen mode");
+
+            let client = LocalKmsClient::new(LocalConfig {
+                key_dir: placed.path().to_path_buf(),
+                master_key: Some("test-master-key".to_string()),
+                file_permissions: Some(0o600),
+            })
+            .await
+            .unwrap_or_else(|error| panic!("mode {mode:#o} must start, not fail: {error:?}"));
+            client.create_key("placed-key", "AES_256", None).await.expect("create key");
+
+            assert_eq!(mode_of(placed.path()), 0o700, "a {mode:#o} key directory must be narrowed");
+        }
+    }
+
+    /// An absent `file_permissions` means owner-only, not "whatever the umask
+    /// says". A `0` umask is the default in a good many container images, and
+    /// under it an unspecified mode used to publish master key records
+    /// world-readable.
+    ///
+    /// The end-to-end assertion below cannot be the whole guard: on a host
+    /// whose umask already masks the group and other bits, the un-fixed code
+    /// would produce `0o600` by accident and the test would pass while
+    /// protecting nothing. So the resolution the protocol performs is asserted
+    /// directly, where the umask cannot reach it.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn unspecified_file_permissions_still_publish_owner_only() {
+        assert_eq!(durable_file::resolved_file_mode(None), DEFAULT_KEY_FILE_MODE);
+        assert_eq!(durable_file::resolved_file_mode(Some(0o640)), 0o640, "an explicit mode is still honoured");
+
+        let temp_dir = TempDir::new().expect("temp dir");
+        let client = LocalKmsClient::new(LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: Some("test-master-key".to_string()),
+            file_permissions: None,
+        })
+        .await
+        .expect("client with unspecified permissions must still start");
+        client.create_key("default-mode", "AES_256", None).await.expect("create key");
+
+        assert_eq!(
+            mode_of(&client.master_key_path("default-mode").expect("valid key id")),
+            DEFAULT_KEY_FILE_MODE
+        );
+        assert_eq!(mode_of(&LocalKmsClient::master_key_salt_path(&client.config)), DEFAULT_KEY_FILE_MODE);
+    }
+
+    /// Publishing must replace a symlink sitting at the destination, never
+    /// write through it. Following it would let anything with write access to
+    /// the key directory redirect a master key record — or a later read of it —
+    /// outside the confinement `master_key_path` enforces.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn publishing_replaces_a_symlink_instead_of_writing_through_it() {
+        let (client, temp_dir) = create_test_client().await;
+        let outside = temp_dir.path().join("outside.txt");
+        fs::write(&outside, b"untouched").await.expect("write decoy");
+
+        // `create_key` publishes with `hard_link`, which refuses a destination
+        // that already exists — including a symlink, dangling or not.
+        let key_path = client.master_key_path("symlinked").expect("valid key id");
+        std::os::unix::fs::symlink(&outside, &key_path).expect("plant symlink");
+        let error = client
+            .create_key("symlinked", "AES_256", None)
+            .await
+            .expect_err("a symlink at the destination must not be written through");
+        assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}");
+        assert_eq!(fs::read(&outside).await.expect("read decoy"), b"untouched");
+
+        // The update path publishes with `rename`, which replaces the link
+        // itself rather than the file it points at.
+        std::fs::remove_file(&key_path).expect("clear symlink");
+        client.create_key("symlinked", "AES_256", None).await.expect("create key");
+        std::fs::remove_file(&key_path).expect("remove record");
+        std::os::unix::fs::symlink(&outside, &key_path).expect("re-plant symlink");
+        let master_key = MasterKeyInfo::new("symlinked".to_string(), "AES_256".to_string(), None);
+        client
+            .save_master_key(&master_key, &[7u8; 32])
+            .await
+            .expect("save over symlink");
+
+        assert_eq!(fs::read(&outside).await.expect("read decoy"), b"untouched");
+        assert!(
+            !std::fs::symlink_metadata(&key_path).expect("stat").file_type().is_symlink(),
+            "the published record must be a regular file, not a link"
+        );
+    }
+
+    /// A hard link planted at the destination is refused exactly as a regular
+    /// file is: `hard_link` fails on an existing name, so no create can adopt
+    /// an inode it did not write.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn a_planted_hard_link_cannot_be_adopted_as_a_key_record() {
+        let (client, temp_dir) = create_test_client().await;
+        let outside = temp_dir.path().join("outside.txt");
+        fs::write(&outside, b"planted").await.expect("write decoy");
+        let key_path = client.master_key_path("linked").expect("valid key id");
+        std::fs::hard_link(&outside, &key_path).expect("plant hard link");
+
+        let error = client
+            .create_key("linked", "AES_256", None)
+            .await
+            .expect_err("a planted hard link must not be adopted");
+        assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}");
+        assert_eq!(fs::read(&outside).await.expect("read decoy"), b"planted");
+    }
+
+    /// Startup removes a commit temp that is a symlink, not only a regular
+    /// file. The protocol only ever creates temps with `create_new`, so a temp
+    /// name wearing any other file type is either our own leftover or something
+    /// planted; requiring a regular file left those behind forever.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn startup_removes_a_symlinked_commit_temp() {
+        let (client, temp_dir) = create_test_client().await;
+        client.create_key("live", "AES_256", None).await.expect("create key");
+        let key_path = client.master_key_path("live").expect("valid key id");
+
+        let symlinked_temp = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
+        std::os::unix::fs::symlink(&key_path, &symlinked_temp).expect("plant symlinked temp");
+        drop(client);
+
+        let client = LocalKmsClient::new(LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: Some("test-master-key".to_string()),
+            file_permissions: Some(0o600),
+        })
+        .await
+        .expect("restart");
+
+        assert!(
+            std::fs::symlink_metadata(&symlinked_temp).is_err(),
+            "a symlinked commit temp must not survive startup"
+        );
+        // Removing the link must not have touched the key it pointed at.
+        client.describe_key("live", None).await.expect("the key must survive");
+    }
+
+    /// The commit protocol never asks the filesystem to rename or link across a
+    /// device boundary, because the temp file is always created in the
+    /// destination's own directory. That is the invariant that makes `EXDEV`
+    /// unreachable; a real cross-device test needs a second filesystem and
+    /// cannot run here, so the invariant itself is what gets pinned.
+    #[tokio::test]
+    async fn commit_temps_always_share_the_destination_directory() {
+        let (client, temp_dir) = create_test_client().await;
+        client.create_key("same-dir", "AES_256", None).await.expect("create key");
+
+        // Every entry the protocol left behind, temps included, is in the key
+        // directory: nothing was staged anywhere a rename could have to cross a
+        // device to leave.
+        let mut entries = std::fs::read_dir(temp_dir.path()).expect("read key dir");
+        assert!(entries.any(|entry| entry.expect("entry").file_name() == "same-dir.key"));
+
+        let key_path = client.master_key_path("same-dir").expect("valid key id");
+        let salt_path = LocalKmsClient::master_key_salt_path(&client.config);
+        assert_eq!(salt_path.parent(), Some(temp_dir.path()));
+        assert_eq!(key_path.parent(), Some(temp_dir.path()));
+    }
 }

crates/kms/src/backends/mod.rs

@@ -162,6 +162,15 @@ pub(crate) fn ensure_rewrap_context_matches(
 /// Page size used when a [`ListKeysRequest`] does not ask for one.
 pub(crate) const DEFAULT_LIST_KEYS_PAGE_SIZE: u32 = 100;
 
+/// Largest page a single [`ListKeysRequest`] can be served.
+///
+/// A page is not a cheap slice: every listed identifier costs the backend one
+/// metadata lookup — a disk read on Local, an HTTP round trip on Vault Transit —
+/// so an unbounded `limit` turns one request into an unbounded fan-out against
+/// the key store. The ceiling is applied where the page is cut rather than at
+/// each caller, so no backend can opt out of it.
+pub(crate) const MAX_LIST_KEYS_PAGE_SIZE: u32 = 1_000;
+
 /// One page of a key set the backend has to slice itself.
 pub(crate) struct KeyPage<'a, T> {
     /// The identifiers this page covers, in listing order.
@@ -228,10 +237,14 @@ pub(crate) fn paginate_keys<'a, T>(sorted: &'a [T], request: &ListKeysRequest, k
 /// already gives `max-keys=0` on the S3 listing path — not a malformed one and
 /// not an omitted value. Rounding it up to a default would hand back a full
 /// page of keys to a caller that explicitly asked for none.
+///
+/// A larger request is clamped to [`MAX_LIST_KEYS_PAGE_SIZE`] rather than
+/// rejected: the caller still reaches every key by following `next_marker`, so
+/// clamping costs it an extra round trip where rejecting would break it.
 pub(crate) fn list_keys_page_size(limit: Option<u32>) -> Option<usize> {
     match limit.unwrap_or(DEFAULT_LIST_KEYS_PAGE_SIZE) {
         0 => None,
-        size => Some(size as usize),
+        size => Some(size.min(MAX_LIST_KEYS_PAGE_SIZE) as usize),
     }
 }
 
@@ -246,6 +259,106 @@ pub(crate) fn empty_key_page() -> ListKeysResponse {
         keys: Vec::new(),
         next_marker: None,
         truncated: false,
+        unreadable_key_ids: Vec::new(),
+    }
+}
+
+/// What a failed per-key describe means for the page being assembled.
+pub(crate) enum ListedKeyFailure {
+    /// The key disappeared between the identifier scan and the read. Concurrent
+    /// removal is normal and the cursor comes from the identifier list, so the
+    /// listing drops it and advances.
+    Vanished,
+    /// The record is still in the store and this build cannot interpret it.
+    /// Reported through [`ListKeysResponse::unreadable_key_ids`] rather than
+    /// omitted or turned into a whole-page failure.
+    Unreadable,
+}
+
+/// Decide whether a per-key describe failure may be attributed to that one key.
+///
+/// `None` means it may not: the error describes something outside the record,
+/// so the caller must fail the whole listing. Downgrading a timeout to "this key
+/// is unreadable" would report a Vault outage as mass key corruption.
+///
+/// The material-level variants are all per-record by construction: each names
+/// one key and stays true on re-read. That includes
+/// [`KmsError::MaterialAuthenticationFailed`], which on the local backend means
+/// one record's AEAD tag did not verify — bit rot or a torn write. The
+/// systemic reading of the same variant, a process holding the wrong master
+/// key, cannot reach a listing: it is rejected when the backend is constructed.
+/// The whole-key-set guard in [`UnreadableKeys`] covers whatever slips past
+/// that.
+pub(crate) fn classify_listed_key_failure(error: &KmsError) -> Option<ListedKeyFailure> {
+    match error {
+        KmsError::KeyNotFound { .. } => Some(ListedKeyFailure::Vanished),
+        KmsError::MaterialMissing { .. }
+        | KmsError::MaterialCorrupt { .. }
+        | KmsError::MaterialAuthenticationFailed { .. }
+        | KmsError::UnsupportedFormatVersion { .. }
+        | KmsError::BaselineVersionLost { .. } => Some(ListedKeyFailure::Unreadable),
+        _ => None,
+    }
+}
+
+/// Whether this request starts at the very beginning of the key set.
+///
+/// An empty `marker` is not the same as no marker to a caller, but it is to
+/// [`paginate_keys`]: no identifier sorts at or below the empty string, so the
+/// page starts at the first key either way. Testing `Option::is_none` alone
+/// would let `?marker=` — which a generated pager that always emits its cursor
+/// parameter sends on its first request — slip past the whole-key-set guard
+/// below and get the empty, healthy-looking page that guard exists to prevent.
+pub(crate) fn started_at_the_first_key(request: &ListKeysRequest) -> bool {
+    request.marker.as_deref().is_none_or(str::is_empty)
+}
+
+/// The unreadable identifiers of one page, and the guarantee that a *complete*
+/// listing never reports every key as damaged while looking empty.
+///
+/// The failure mode being guarded against is a shared cause — a mixed-version
+/// node reading records in a format it has no reader for — that makes every key
+/// unreadable at once. Answering that with `200 OK` and an empty `keys` list
+/// looks, to any client written before `unreadable_key_ids` existed, exactly
+/// like a deployment that has no keys.
+///
+/// The guard is deliberately scoped to a listing that reached the end of the key
+/// set on its first page. Firing it per page instead would be a trap: with
+/// `limit=1` a single damaged key would fail its own page, and since a failed
+/// page carries no `next_marker` the caller could never advance past it — every
+/// key behind the damaged one becomes permanently unreachable. A page that is
+/// truncated, or that resumed from a marker, always reports per-key so paging
+/// can advance; and an empty `keys` array on such a page is already a documented
+/// state, because filters are applied after the page is cut.
+#[derive(Default)]
+pub(crate) struct UnreadableKeys {
+    ids: Vec<String>,
+    first_error: Option<KmsError>,
+    readable: usize,
+}
+
+impl UnreadableKeys {
+    pub(crate) fn saw_readable(&mut self) {
+        self.readable += 1;
+    }
+
+    pub(crate) fn record(&mut self, key_id: &str, error: KmsError) {
+        self.ids.push(key_id.to_string());
+        self.first_error.get_or_insert(error);
+    }
+
+    /// The identifiers to report.
+    ///
+    /// `whole_key_set` says this page both started at the beginning and reached
+    /// the end, so there is no further page a caller could advance to — which is
+    /// what makes failing here safe. Every identifier has already been logged
+    /// individually by the backend, so the single returned error does not hide
+    /// the rest.
+    pub(crate) fn into_reported_ids(self, whole_key_set: bool) -> Result<Vec<String>> {
+        match self.first_error {
+            Some(error) if whole_key_set && self.readable == 0 => Err(error),
+            _ => Ok(self.ids),
+        }
     }
 }
 
@@ -860,6 +973,43 @@ mod tests {
         assert_eq!(page_of(&keys, Some(u32::MAX), Some("key-01")), (vec![keys[2].clone()], None, false));
     }
 
+    /// A page is capped however large a limit the caller asks for, and the
+    /// capped page still carries a cursor, so the caller reaches every key
+    /// instead of being cut off at the ceiling.
+    #[test]
+    fn page_size_is_capped_and_the_capped_page_still_advances() {
+        // Pinned as a number, not only symbolically: the published contract in
+        // `docs/operations/kms-admin-contract.md` states this exact ceiling, so
+        // raising it is an API change and has to be a deliberate edit here.
+        assert_eq!(MAX_LIST_KEYS_PAGE_SIZE, 1_000);
+        assert_eq!(
+            list_keys_page_size(Some(u32::MAX)),
+            Some(MAX_LIST_KEYS_PAGE_SIZE as usize),
+            "an unbounded limit must not become an unbounded per-key fan-out"
+        );
+        assert_eq!(list_keys_page_size(Some(MAX_LIST_KEYS_PAGE_SIZE)), Some(MAX_LIST_KEYS_PAGE_SIZE as usize));
+        // Under the ceiling the caller's limit is still honoured exactly.
+        assert_eq!(
+            list_keys_page_size(Some(MAX_LIST_KEYS_PAGE_SIZE - 1)),
+            Some(MAX_LIST_KEYS_PAGE_SIZE as usize - 1)
+        );
+
+        // Padded to a fixed width so the vector is sorted by identifier, which
+        // is what `paginate_keys` requires of its input.
+        let keys: Vec<String> = (0..MAX_LIST_KEYS_PAGE_SIZE as usize + 5)
+            .map(|index| format!("key-{index:04}"))
+            .collect();
+        let (items, next_marker, truncated) = page_of(&keys, Some(u32::MAX), None);
+        assert_eq!(items.len(), MAX_LIST_KEYS_PAGE_SIZE as usize);
+        assert!(truncated);
+        let next_marker = next_marker.expect("a capped page must hand back a cursor");
+        assert_eq!(next_marker, items.last().cloned().expect("page is non-empty"));
+
+        let (rest, _, still_truncated) = page_of(&keys, Some(u32::MAX), Some(&next_marker));
+        assert_eq!(rest.len(), 5, "following the cursor must reach the keys the cap held back");
+        assert!(!still_truncated);
+    }
+
     /// The cursor is an identifier, so a marker naming a key that no longer
     /// exists resumes after where it would have been instead of restarting.
     #[test]