test(e2e): activate conditional write regressions (#6454)

* fix(protocols): add missing test dependency

* style(ecstore): fix decommission formatting

.config/coverage-baselines.toml

@@ -0,0 +1,20 @@
+# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
+# Update counts only with a linked coverage run and a reviewed explanation.
+phase = "report-only"
+allowed_drop_percentage_points = 1.0
+
+[crates."crates/iam"]
+covered = 5149
+count = 8131
+
+[crates."crates/kms"]
+covered = 2950
+count = 4200
+
+[crates."crates/policy"]
+covered = 4636
+count = 5464
+
+[crates."crates/crypto"]
+covered = 469
+count = 494

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
-sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
+sha256-darwin=ae412c27e2e43f3fda3e0c21ef616b48a6403354faab8db8d4154202e3c4eec5
+sha256-linux=ef3be856bd3257c2c369428f66a48ad073dad8187229fe95840a600a81edf22b

.config/make/tests.mak

@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
 	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh

.config/nextest.toml

@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# The durable ILM decommission regressions build isolated multi-pool stores and
+# deliberately take source or target disks offline while checking fencing.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -305,7 +315,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #     the target incl. multipart and the resync path, SSE-C and
 #     target-without-KMS stay fail-closed), and one guards event/history
 #     observers.
-#   * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
+#   * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
 #   * 1 `_real_three_node` site-replication test.
 #   * 1 `_real_single_node` service-account round-trip test.
@@ -325,8 +335,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #
 # Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
 # workflow_dispatch), which builds the rustfs binary once, installs awscurl so
-# the STS dual-node test actually exercises its path (it skips gracefully with
-# a visible log line when awscurl is absent), and routes scheduled failures
+# the STS dual-node test actually exercises its path (the test fails when
+# awscurl is absent), and routes scheduled failures
 # through .github/actions/schedule-failure-issue (ci-8). Explicit division of
 # labor with e2e-full: these tests run only in the consolidated nightly
 # workflow, not in the merge/main lane.
@@ -396,7 +406,7 @@ path = "junit.xml"
 #     object_lambda) — too heavy for the merge budget; they run in the
 #     e2e-nightly serial cluster-fault lane.
 #   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
 #     manual-localhost:9000 reliant/policy tests are ci-13's migration.

.github/workflows/ci.yml

@@ -681,6 +681,19 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
@@ -803,6 +816,20 @@ jobs:
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
+      - name: Install mc
+        env:
+          MC_VERSION: RELEASE.2025-08-13T08-35-41Z
+          MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
+        run: |
+          MC_BINARY="mc.linux-amd64.${MC_VERSION}"
+          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
+          echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
+          chmod +x "$RUNNER_TEMP/mc"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Verify mc
+        run: mc --version
+
       - name: Install Vault
         run: |
           VAULT_VERSION="1.17.6"

.github/workflows/coverage.yml

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+# Workspace line-coverage baseline and security-crate calibration
+# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# trend, while relevant pull requests run a report-only security-crate
+# comparison. Neither job is a required check during calibration.
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,6 +29,17 @@
 name: coverage
 
 on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "crates/iam/**"
+      - "crates/kms/**"
+      - "crates/policy/**"
+      - "crates/crypto/**"
+      - ".config/coverage-baselines.toml"
+      - "scripts/coverage_per_crate.py"
+      - "scripts/check_security_coverage.py"
+      - ".github/workflows/coverage.yml"
   workflow_dispatch:
   schedule:
     # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
@@ -39,6 +48,10 @@ on:
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
     - cron: "43 7 * * 0"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
 permissions:
@@ -46,12 +59,14 @@ permissions:
 
 jobs:
   coverage:
-    name: Workspace coverage (weekly)
+    name: Workspace line coverage
     runs-on: sm-standard-4
     # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
+    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # full suite. Two later exact-head runs exhausted 150 minutes before the
+    # report steps, so allow one additional 90-minute cold-run margin while
+    # keeping the calibration job bounded.
+    timeout-minutes: 240
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +106,9 @@ jobs:
           cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
 
       - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload coverage artifact
         if: always()

.github/workflows/e2e-replication-nightly.yml

@@ -75,11 +75,7 @@ jobs:
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: 'false'
 
-      # awscurl lets the STS dual-node test actually exercise its path. Without
-      # it the test skips gracefully with a visible log line
-      # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
-      # still passes — installing it just upgrades that one test from skip to
-      # real coverage.
+      # The STS dual-node test requires awscurl and fails if it is unavailable.
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
@@ -87,7 +83,7 @@ jobs:
 
       - name: Install awscurl
         run: |
-          python3 -m pip install --user --upgrade pip awscurl
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
           echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
 
       - name: Verify awscurl
@@ -196,8 +192,11 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
-      - name: Verify protocol socket oracle
-        run: ss -tn state CLOSE-WAIT >/dev/null
+      - name: Install and verify protocol socket oracle
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq iproute2
+          ss -tn state CLOSE-WAIT >/dev/null
 
       # The suite owns fixed protocol ports and serializes its internal cases.
       - name: Verify protocol e2e membership

.github/workflows/e2e-s3tests.yml

@@ -21,6 +21,9 @@
 #     suite and reports promotion candidates. Regressions, unclassified tests,
 #     incomplete execution, and infrastructure errors fail the job; classified
 #     failures for not-yet-implemented features remain informational.
+#   - Non-blocking upstream HEAD canary: collects current upstream node IDs and
+#     reports new, removed, duplicate, or overlapping classifications without
+#     making upstream drift a release gate.
 #   - Manual runs (workflow_dispatch): same, with configurable mode/scope.
 #
 # All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -178,9 +181,14 @@ jobs:
 
       - name: Install Python tools
         run: |
-          python3 -m pip install --user --upgrade pip awscurl tox
+          python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
+      - name: Verify Python tools
+        run: |
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
+
       - name: Enable buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
@@ -354,6 +362,85 @@ jobs:
           name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
           path: artifacts/**
 
+  upstream-head-canary:
+    name: Upstream HEAD classification canary
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install collection tool
+        run: |
+          python3 -m pip install --user "tox==4.60.0"
+          python3 - <<'PY'
+          from importlib.metadata import version
+
+          assert version("tox") == "4.60.0"
+          PY
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Compare upstream HEAD classifications
+        id: upstream-compare
+        run: |
+          ARTIFACT_DIR="artifacts/s3tests-upstream-head"
+          UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
+          mkdir -p "${ARTIFACT_DIR}"
+          git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
+          git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
+          cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
+          (
+            cd "${UPSTREAM_DIR}"
+            S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
+              -q --collect-only s3tests/functional/test_s3.py \
+              -m "not rustfs_never_marker"
+          ) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
+          grep -E '^s3tests/functional/test_s3\.py::' \
+            "${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
+          python3 scripts/s3-tests/report_compat.py \
+            --lists-dir scripts/s3-tests \
+            --collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
+            --check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
+
+      - name: Publish canary report
+        if: always()
+        env:
+          CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
+        run: |
+          {
+            echo "## ceph/s3-tests upstream HEAD canary"
+            echo
+            if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
+              echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
+            fi
+            echo
+            echo '```text'
+            if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
+              cat artifacts/s3tests-upstream-head/classification-drift.txt
+            elif [ "${CANARY_OUTCOME}" != "success" ]; then
+              echo "Canary did not complete; inspect the collection log artifact."
+            else
+              echo "No classification drift detected."
+            fi
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload canary artifacts
+        if: always() && env.ACT != 'true'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: s3tests-upstream-head
+          path: artifacts/s3tests-upstream-head/**
+          retention-days: 14
+
   alert-on-failure:
     name: Alert on scheduled failure
     needs: [s3tests]

.github/workflows/performance-ab.yml

@@ -121,14 +121,14 @@ jobs:
           candidate_sha="$(git rev-parse HEAD)"
           if [[ "${{ github.event_name }}" == "schedule" ]]; then
             baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
-            if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
-              echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
-              exit 1
-            fi
           else
             baseline_sha="$(git rev-parse origin/main)"
           fi
           git cat-file -e "${baseline_sha}^{commit}"
+          if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+            echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
+            exit 1
+          fi
           echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
           echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
           echo "baseline commit: $baseline_sha"
@@ -342,6 +342,10 @@ jobs:
         if: always()
         run: |
           status="${{ steps.ab.outputs.status }}"
+          if [[ -z "$status" ]]; then
+            echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
+            exit 1
+          fi
           if [[ "$status" != "0" ]]; then
             echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
             exit "$status"

Cargo.lock

@@ -68,16 +68,16 @@ dependencies = [
 
 [[package]]
 name = "aes-gcm"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
+checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
 dependencies = [
  "aead",
  "aes 0.9.2",
  "cipher 0.5.2",
  "ctr",
+ "ctutils",
  "ghash",
- "subtle",
  "zeroize",
 ]
 
@@ -333,6 +333,12 @@ dependencies = [
  "password-hash",
 ]
 
+[[package]]
+name = "array-init"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.8"
@@ -813,11 +819,12 @@ dependencies = [
 
 [[package]]
 name = "async_zip"
-version = "0.0.18"
+version = "0.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
+checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
 dependencies = [
  "async-compression",
+ "binrw",
  "crc32fast",
  "futures-lite",
  "pin-project",
@@ -869,9 +876,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-config"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
+checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -964,9 +971,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-kms"
-version = "1.115.0"
+version = "1.116.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
+checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -990,9 +997,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.142.0"
+version = "1.143.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
+checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1027,9 +1034,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.106.0"
+version = "1.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
+checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1053,9 +1060,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.108.0"
+version = "1.109.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
+checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1079,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.111.0"
+version = "1.112.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
+checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1544,6 +1551,30 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "binrw"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
+dependencies = [
+ "array-init",
+ "binrw_derive",
+ "bytemuck",
+]
+
+[[package]]
+name = "binrw_derive"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
+dependencies = [
+ "either",
+ "owo-colors",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -1631,9 +1662,9 @@ dependencies = [
 
 [[package]]
 name = "blocking"
-version = "1.6.2"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
+checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
 dependencies = [
  "async-channel",
  "async-task",
@@ -1722,6 +1753,12 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
+[[package]]
+name = "bytemuck"
+version = "1.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -2325,9 +2362,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
 dependencies = [
  "cfg-if",
 ]
@@ -3934,9 +3971,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
 
 [[package]]
 name = "elliptic-curve"
@@ -4470,6 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
 dependencies = [
  "polyval",
+ "zeroize",
 ]
 
 [[package]]
@@ -5052,9 +5090,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
+checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
 dependencies = [
  "arc-swap",
  "async-channel",
@@ -5086,9 +5124,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
+checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5097,15 +5135,15 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
+checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
 
 [[package]]
 name = "hotpath-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
+checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
 dependencies = [
  "hotpath-macros-meta",
 ]
@@ -5372,9 +5410,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
 
 [[package]]
 name = "icu_provider"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
+checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -5761,9 +5799,9 @@ dependencies = [
 
 [[package]]
 name = "keccak"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
+checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
 dependencies = [
  "cfg-if",
  "cpufeatures 0.3.0",
@@ -6075,9 +6113,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
 
 [[package]]
 name = "lru"
@@ -6267,7 +6305,7 @@ dependencies = [
  "hashbrown 0.16.1",
  "indexmap 2.14.0",
  "metrics",
- "ordered-float 5.3.0",
+ "ordered-float 5.5.0",
  "quanta",
  "radix_trie",
  "rand 0.9.5",
@@ -7215,9 +7253,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "5.3.0"
+version = "5.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
 dependencies = [
  "num-traits",
 ]
@@ -7252,6 +7290,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
+[[package]]
+name = "owo-colors"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
+
 [[package]]
 name = "p12-keystore"
 version = "0.2.1"
@@ -7794,6 +7838,7 @@ dependencies = [
  "cpubits",
  "cpufeatures 0.3.0",
  "universal-hash",
+ "zeroize",
 ]
 
 [[package]]
@@ -8264,6 +8309,16 @@ name = "quick-xml"
 version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
 dependencies = [
  "encoding_rs",
  "memchr",
@@ -8928,9 +8983,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.62.7"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
+checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
 dependencies = [
  "aes 0.9.2",
  "aws-lc-rs",
@@ -9152,7 +9207,7 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rand 0.10.2",
  "rcgen",
  "regex",
@@ -9418,7 +9473,7 @@ dependencies = [
  "path-absolutize",
  "pin-project-lite",
  "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rand 0.10.2",
  "ratelimit",
  "rcgen",
@@ -9564,6 +9619,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serial_test",
+ "sha2 0.11.0",
  "temp-env",
  "tempfile",
  "thiserror 2.0.20",
@@ -9884,7 +9940,7 @@ dependencies = [
  "jiff",
  "metrics",
  "percent-encoding",
- "quick-xml",
+ "quick-xml 0.42.0",
  "rayon",
  "rustc-hash",
  "rustfs-config",
@@ -10657,9 +10713,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.14"
+version = "0.103.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -10703,7 +10759,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
+source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -10730,7 +10786,7 @@ dependencies = [
  "nom 8.0.0",
  "numeric_cast",
  "pin-project-lite",
- "quick-xml",
+ "quick-xml 0.41.0",
  "regex",
  "serde",
  "serde_json",
@@ -12675,9 +12731,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.24.1"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
+checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
 dependencies = [
  "getrandom 0.4.3",
  "js-sys",
@@ -13397,9 +13453,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
+checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async_zip = { default-features = false, version = "0.0.18" }
+async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.4"
-quick-xml = "0.41.0"
+quick-xml = "0.42.0"
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.0" }
+aes-gcm = { version = "=0.11.1" }
 argon2 = { version = "=0.6.0-rc.8" }
 blake2 = "=0.11.0-rc.6"
 chacha20poly1305 = { version = "=0.11.0" }
@@ -227,11 +227,11 @@ arc-swap = "1.9.2"
 astral-tokio-tar = "0.6.4"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.10.1" }
+aws-config = { version = "1.11.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.115.0" }
-aws-sdk-s3 = { default-features = false, version = "1.142.0" }
-aws-sdk-sts = { default-features = false, version = "1.111.0" }
+aws-sdk-kms = { default-features = false, version = "1.116.0" }
+aws-sdk-s3 = { default-features = false, version = "1.143.0" }
+aws-sdk-sts = { default-features = false, version = "1.112.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
 aws-smithy-runtime-api = { version = "1.15.0" }
 aws-smithy-types = { version = "1.6.2" }
@@ -291,7 +291,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -314,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.24.1" }
+uuid = { version = "1.25.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -343,7 +343,7 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.2" }
 rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.62.7" }
+russh = { version = "0.63.0" }
 russh-sftp = "2.4.0"
 
 # WebDAV
@@ -352,7 +352,7 @@ dav-server = "0.11.0"
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.0" }
 rustfs-mimalloc-sys = { version = "0.5.0" }
-hotpath = { version = "0.23.3", default-features = false }
+hotpath = { version = "0.24.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

crates/common/src/metrics.rs

@@ -918,7 +918,15 @@ pub struct Metrics {
     scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
     scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
     scanner_usage_last_save_unix_secs: AtomicU64,
+    scanner_usage_last_durable_success_unix_secs: AtomicU64,
+    scanner_usage_last_publication_unix_secs: AtomicU64,
+    scanner_usage_last_publication_state: Mutex<String>,
+    scanner_usage_last_publication_reason: Mutex<String>,
     scanner_usage_last_save_result: AtomicU8,
+    scanner_usage_deferred_pending: AtomicBool,
+    scanner_usage_deferred_total: AtomicU64,
+    scanner_usage_last_deferred_unix_secs: AtomicU64,
+    scanner_usage_last_deferred_reason: Mutex<String>,
     scanner_source_work: Vec<ScannerSourceWorkCounters>,
     current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
     last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1216,6 +1224,22 @@ pub struct ScannerUsageFreshnessSnapshot {
     pub last_usage_save_unix_secs: u64,
     pub last_usage_save_result: String,
     pub last_usage_save_result_code: u64,
+    #[serde(default)]
+    pub last_durable_success_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_state: String,
+    #[serde(default)]
+    pub last_publication_reason: String,
+    #[serde(default)]
+    pub deferred_pending: bool,
+    #[serde(default)]
+    pub deferred_total: u64,
+    #[serde(default)]
+    pub last_deferred_unix_secs: u64,
+    #[serde(default)]
+    pub last_deferred_reason: String,
 }
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1945,7 +1969,15 @@ impl Metrics {
             scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
             scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
             scanner_usage_last_save_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_state: Mutex::new(String::new()),
+            scanner_usage_last_publication_reason: Mutex::new(String::new()),
             scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
+            scanner_usage_deferred_pending: AtomicBool::new(false),
+            scanner_usage_deferred_total: AtomicU64::new(0),
+            scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_deferred_reason: Mutex::new(String::new()),
             scanner_source_work: ScannerWorkSource::all()
                 .iter()
                 .map(|_| ScannerSourceWorkCounters::default())
@@ -2270,6 +2302,44 @@ impl Metrics {
             .store(unix_now_secs(), Ordering::Relaxed);
     }
 
+    /// Record an intentional retryable usage publication deferral separately
+    /// from the last durable save result.
+    pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
+        let reason = reason.into();
+        self.record_scanner_usage_publication("deferred", reason.clone());
+        self.scanner_usage_deferred_pending.store(true, Ordering::Release);
+        self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
+        self.scanner_usage_last_deferred_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *last_reason = reason;
+    }
+
+    pub fn record_scanner_usage_durable_success(&self) {
+        self.record_scanner_usage_publication("success", "");
+        self.scanner_usage_last_durable_success_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        self.scanner_usage_deferred_pending.store(false, Ordering::Release);
+    }
+
+    pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
+        self.scanner_usage_last_publication_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_state = state.to_string();
+        let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_reason = reason.into();
+    }
+
     pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
         if let Some(counters) = self.scanner_source_work.get(source.index()) {
             counters.add(work);
@@ -3292,6 +3362,23 @@ impl Metrics {
             last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
             last_usage_save_result: usage_save_result.as_str().to_string(),
             last_usage_save_result_code: usage_save_result as u8 as u64,
+            last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
+            last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
+            last_publication_state: match self.scanner_usage_last_publication_state.lock() {
+                Ok(state) => state.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
+            deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
+            last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
+            last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
         };
         m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
         m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -4663,6 +4750,7 @@ mod tests {
         metrics.record_scanner_dirty_usage_cycle_snapshot(1);
         metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
         metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
+        metrics.record_scanner_usage_deferred("data_movement");
 
         let report = metrics.report().await;
 
@@ -4674,6 +4762,22 @@ mod tests {
         assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
         assert_eq!(report.usage_freshness.last_usage_save_result, "success");
         assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
+        assert!(report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_deferred_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
+
+        metrics.record_scanner_usage_durable_success();
+        let report = metrics.report().await;
+        assert!(!report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_publication_state, "success");
+
+        metrics.record_scanner_usage_publication("no_update", "no_update");
+        let report = metrics.report().await;
+        assert_eq!(report.usage_freshness.last_publication_state, "no_update");
+        assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
     }
 
     #[tokio::test]

crates/common/src/mrf_channel.rs

@@ -23,21 +23,32 @@
 //! unconsumed intents is the consumer's job (see `rustfs-heal`
 //! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
 
+use std::collections::HashMap;
+use std::collections::hash_map::RandomState;
+use std::hash::{BuildHasher, Hash};
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::AtomicUsize;
 use std::sync::{
-    Arc, OnceLock,
+    Arc, Mutex, OnceLock,
     atomic::{AtomicBool, Ordering},
 };
+use std::time::{Duration, Instant};
 use tokio::sync::mpsc;
 use uuid::Uuid;
 
 /// Bounded capacity of the global MRF channel. Backpressure is resolved by
 /// dropping (and counting) intents, never by blocking the producer.
 const MRF_CHANNEL_CAPACITY: usize = 8192;
+const MRF_COALESCER_SHARDS: usize = 16;
+const MRF_COALESCER_MAX_KEYS: usize = 8192;
+const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
+const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
+const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
 
 /// Why an intent was produced. Drives the heal priority mapping on the
 /// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
 /// PartialWrite -> Normal).
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum MrfKind {
     /// Erasure decode failed while serving a read (read path).
     DecodeFailure,
@@ -67,12 +78,52 @@ pub struct MrfIntent {
     /// Version the intent targets, as raw UUID bytes.
     pub version_id: Option<[u8; 16]>,
     pub kind: MrfKind,
+    /// Stable erasure-set scope when the producer has it. Kept optional so
+    /// metadata corruption and legacy producers do not invent a scope.
+    pub scope: Option<MrfScope>,
+    /// Generation of the node-local ingress lease. It is not persisted in
+    /// the journal; replayed records acquire a fresh lease when re-enqueued.
+    pub lease: Option<MrfIngressLease>,
     pub enqueued_at_ms: u64,
     /// Times this intent has already been offered to the heal manager.
     /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
     pub attempts: u8,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfScope {
+    pub pool_index: u32,
+    pub set_index: u32,
+}
+
+/// Opaque generation used to release exactly the admission that created an
+/// ingress entry. A generation prevents a late terminal callback from
+/// deleting a newer retry for the same identity (ABA).
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfIngressLease(u64);
+
+impl MrfIngressLease {
+    const fn new(value: u64) -> Self {
+        Self(value)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfDropReason {
+    Disabled,
+    Uninitialized,
+    Full,
+    OversizedIdentity,
+    CoalescerFull,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfIngressResult {
+    Enqueued,
+    Coalesced,
+    Dropped(MrfDropReason),
+}
+
 /// Consumer-side retry ceiling before an intent is given up on.
 pub const MRF_MAX_ATTEMPTS: u8 = 3;
 
@@ -87,6 +138,159 @@ impl MrfIntent {
 
 static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct MrfIdentityKey {
+    kind: MrfKind,
+    bucket: Arc<str>,
+    object: Arc<str>,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+}
+
+#[derive(Debug)]
+struct IngressEntry {
+    lease: MrfIngressLease,
+    expires_at: Instant,
+    bytes: usize,
+}
+
+type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
+type MrfCoalescer = Box<[MrfCoalescerShard]>;
+
+static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
+static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
+static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
+static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
+static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
+
+fn coalescer() -> &'static [MrfCoalescerShard] {
+    MRF_COALESCER.get_or_init(|| {
+        (0..MRF_COALESCER_SHARDS)
+            .map(|_| Mutex::new(HashMap::new()))
+            .collect::<Vec<_>>()
+            .into_boxed_slice()
+    })
+}
+
+fn key_shard(key: &MrfIdentityKey) -> usize {
+    let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
+    usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
+}
+
+fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
+    version_id
+        .filter(|version| !version.is_nil())
+        .map(|version| *version.as_bytes())
+}
+
+fn canonical_identity(
+    kind: MrfKind,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+) -> (Option<[u8; 16]>, Option<MrfScope>) {
+    let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
+    match kind {
+        MrfKind::MetadataCorruption => (None, None),
+        MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
+    }
+}
+
+fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
+    64usize
+        .saturating_add(key.bucket.len())
+        .saturating_add(key.object.len())
+        .saturating_add(key.version_id.map_or(0, |_| 16))
+        .saturating_add(key.scope.map_or(0, |_| 8))
+}
+
+fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
+    let mut current = counter.load(Ordering::Relaxed);
+    loop {
+        let Some(next) = current.checked_add(amount) else {
+            return false;
+        };
+        if next > limit {
+            return false;
+        }
+        match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return true,
+            Err(observed) => current = observed,
+        }
+    }
+}
+
+fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
+    let shard = key_shard(&key);
+    let mut entries = coalescer()[shard]
+        .lock()
+        .map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
+    let now = Instant::now();
+    let before = entries.len();
+    let mut expired_bytes = 0usize;
+    entries.retain(|_, entry| {
+        if entry.expires_at > now {
+            true
+        } else {
+            expired_bytes = expired_bytes.saturating_add(entry.bytes);
+            false
+        }
+    });
+    let evicted = before.saturating_sub(entries.len());
+    if evicted > 0 {
+        MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
+        let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
+        metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
+        metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
+    }
+    if entries.contains_key(&key) {
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    let bytes = identity_estimated_bytes(&key);
+    let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
+    let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
+    if !count_reserved || !bytes_reserved {
+        if count_reserved {
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        }
+        metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
+        return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
+    }
+    let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
+    if entries
+        .insert(
+            key,
+            IngressEntry {
+                lease,
+                expires_at: now + MRF_COALESCER_TTL,
+                bytes,
+            },
+        )
+        .is_some()
+    {
+        MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    Ok(lease)
+}
+
+fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
+    let Some(lease) = lease else {
+        return;
+    };
+    if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
+        let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
+        if should_remove {
+            let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+            MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        }
+    }
+}
+
 /// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
 /// this before touching the channel so the disabled path stays allocation- and
 /// sync-free.
@@ -122,21 +326,90 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
 /// This runs on IO error paths, so it stays synchronous and cheap: one
 /// bounded allocation for the two `Arc<str>` handles plus the channel slot.
 pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
+    matches!(
+        try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
+        MrfIngressResult::Enqueued
+    )
+}
+
+/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
+/// intent already exists; it is not a second executable or durable admission.
+pub fn try_send_mrf_intent_typed(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<Uuid>,
+    scope: Option<MrfScope>,
+) -> MrfIngressResult {
     if !mrf_delivery_enabled() {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Disabled);
     }
     let Some(sender) = GLOBAL_MRF_SENDER.get() else {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
     };
-    let intent = MrfIntent {
+    if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
+        return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
+    }
+    let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
+    let key = MrfIdentityKey {
+        kind,
         bucket: Arc::from(bucket),
         object: Arc::from(object),
-        version_id: version_id.map(|vid| *vid.as_bytes()),
+        version_id,
+        scope,
+    };
+    let lease = match coalescer_admit(key.clone()) {
+        Ok(lease) => lease,
+        Err(result) => return result,
+    };
+    let intent = MrfIntent {
+        bucket: key.bucket.clone(),
+        object: key.object.clone(),
+        version_id: key.version_id,
         kind,
+        scope,
+        lease: Some(lease),
         enqueued_at_ms: unix_now_ms(),
         attempts: 0,
     };
-    sender.try_send(intent).is_ok()
+    match sender.try_send(intent) {
+        Ok(()) => MrfIngressResult::Enqueued,
+        Err(mpsc::error::TrySendError::Full(_)) => {
+            coalescer_release(&key, Some(lease));
+            metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
+            MrfIngressResult::Dropped(MrfDropReason::Full)
+        }
+        Err(mpsc::error::TrySendError::Closed(_)) => {
+            coalescer_release(&key, Some(lease));
+            MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
+        }
+    }
+}
+
+/// Release the ingress key once the consumer owns the intent.
+pub fn release_mrf_intent(intent: &MrfIntent) {
+    release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
+}
+
+pub fn release_mrf_identity(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+    lease: Option<MrfIngressLease>,
+) {
+    let (version_id, scope) = canonical_identity(kind, version_id, scope);
+    coalescer_release(
+        &MrfIdentityKey {
+            kind,
+            bucket: Arc::from(bucket),
+            object: Arc::from(object),
+            version_id,
+            scope,
+        },
+        lease,
+    );
 }
 
 fn unix_now_ms() -> u64 {
@@ -144,7 +417,8 @@ fn unix_now_ms() -> u64 {
     // failure would be a bug rather than something to handle here.
     std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
-        .map(|d| d.as_millis() as u64)
+        .ok()
+        .and_then(|d| u64::try_from(d.as_millis()).ok())
         .unwrap_or(0)
 }
 
@@ -215,12 +489,60 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0u8; 16]),
             kind: MrfKind::DecodeFailure,
+            scope: None,
+            lease: None,
             enqueued_at_ms: 0,
             attempts: 0,
         };
         assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
     }
 
+    #[test]
+    fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
+        let key = MrfIdentityKey {
+            kind: MrfKind::DecodeFailure,
+            bucket: Arc::from("ingress-test-bucket"),
+            object: Arc::from("ingress-test-object"),
+            version_id: Some([9; 16]),
+            scope: Some(MrfScope {
+                pool_index: 3,
+                set_index: 4,
+            }),
+        };
+        let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
+        for _ in 0..999 {
+            assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
+        }
+        coalescer_release(&key, Some(lease));
+        let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
+        coalescer_release(&key, Some(retry_lease));
+    }
+
+    #[test]
+    fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
+        let (nil_version, nil_scope) = canonical_identity(
+            MrfKind::DecodeFailure,
+            Some([0; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
+        assert!(nil_scope.is_some());
+
+        let (metadata_version, metadata_scope) = canonical_identity(
+            MrfKind::MetadataCorruption,
+            Some([7; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(metadata_version, None);
+        assert_eq!(metadata_scope, None);
+    }
+
     #[tokio::test]
     async fn try_send_delivers_and_respects_capacity() {
         let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -230,6 +552,7 @@ mod tests {
         let intent = receiver.recv().await.expect("intent should arrive");
         assert_eq!(intent.kind, MrfKind::DecodeFailure);
         assert_eq!(intent.bucket.as_ref(), "b");
+        release_mrf_intent(&intent);
 
         // Disable delivery: producers become no-ops.
         set_mrf_delivery_enabled(false);
@@ -239,8 +562,8 @@ mod tests {
         // Fill the bounded channel past capacity: excess intents are dropped,
         // never blocking.
         let mut accepted = 0;
-        for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
-            if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
+        for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
+            if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
                 accepted += 1;
             }
         }

crates/e2e_test/README.md

@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `find_available_port` | Random free port (isolation primitive) |
 | `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
-| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
+| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
 | `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
 | `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
-# Replication nightly lane; install awscurl so STS paths do not skip
+# Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
 job) to dodge this; local single-node tests already use random ports, so a
 lingering orphan is usually the cause of a spurious bind failure.
 
-**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
-visible log line (`awscurl_available()`); install `awscurl` to actually run
-them.
+**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
+spawn error. Install the pinned CI version before running their profiles.
 
 ## Related
 
@@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is:
 2. **Single-node** — spawns its own server via
    `RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
    isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
-3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
-   no fixed protocol ports. Tools that may be absent on the runner (e.g.
-   `awscurl`) are acceptable only when the test skips gracefully with a
-   visible log line (see `bucket_policy_check_test.rs`).
+3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
+   Vault, and no fixed protocol ports. Any required CLI must be pinned and
+   installed by the workflow; a missing CLI must fail the test.
 4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
    ci-13 / backlog#1148 ilm-3), not smoke candidates.
 
@@ -278,4 +276,9 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
 moving e2e tests so acceptance numbers in the test-strategy issues
 (backlog#1147–#1155) stay auditable. When a profile membership change is
 intentional, review its JSON listing before updating the matching
-`.config/e2e-*-selection.txt` test-ID digest.
+`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
+produced the listing:
+
+```bash
+python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
+```

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[tokio::test]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
-        return Ok(());
-    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/cluster_concurrency_test.rs

@@ -15,16 +15,27 @@
 use crate::common::RustFSTestClusterEnvironment;
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
 use bytes::Bytes;
 use std::sync::Arc;
 use tokio::sync::Barrier;
 use tracing::{info, warn};
 
 const BUCKET: &str = "conditional-put-race-bucket";
+const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
 
-async fn cleanup_object(client: &Client, key: &str) {
-    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
-        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
+async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client.delete_object().bucket(BUCKET).key(key).send().await?;
+    Ok(())
+}
+
+async fn assert_bucket_cors_missing(client: &Client) {
+    let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
+    match result {
+        Err(SdkError::ServiceError(error)) => {
+            assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
+        }
+        result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
     }
 }
 
@@ -71,14 +82,13 @@ async fn run_race_iteration(
     test_key: &str,
     iteration: usize,
 ) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
-    cleanup_object(&clients[0], test_key).await;
+    cleanup_object(&clients[0], test_key).await?;
     tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
 
-    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
-
-    if head_result.is_ok() {
-        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
-        return Ok(0);
+    match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
+        Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
+        Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
+        Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
     }
 
     info!("\n=== Iteration {} ===", iteration);
@@ -120,14 +130,16 @@ async fn run_race_iteration(
 
     info!("Result: {} out of {} succeeded", success_count, clients.len());
 
+    if had_error {
+        return Err("one or more conditional PUTs failed unexpectedly".into());
+    }
+
     if success_count > 1 {
         info!(">>> RACE CONDITION DETECTED!");
     } else if success_count == 1 {
         info!(">>> Correct behavior: exactly 1 writer succeeded.");
-    } else if had_error {
-        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
     } else {
-        info!(">>> Unexpected: no writers succeeded.");
+        return Err("no conditional PUT succeeded".into());
     }
 
     Ok(success_count)
@@ -167,7 +179,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
             }
         }
 
-        cleanup_object(&clients[0], &test_key).await;
+        cleanup_object(&clients[0], &test_key).await?;
         tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
     }
 
@@ -177,7 +189,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
     info!("Total iterations:   {}", iterations);
     info!("Correct (1 winner): {}", correct_count);
     info!("Race conditions:    {}", races_detected);
-    info!("Errors (skipped):   {}", error_count);
+    info!("Failed iterations:  {}", error_count);
 
     assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
     assert_eq!(
@@ -185,6 +197,10 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
         "{} iteration(s) failed due to errors (e.g. cluster not ready)",
         error_count
     );
+    assert_eq!(
+        correct_count, iterations,
+        "only {correct_count}/{iterations} iterations observed exactly one winner"
+    );
 
     Ok(())
 }
@@ -201,7 +217,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
 
     let client = cluster.create_s3_client(0)?;
     let test_key = "basic-conditional-put";
-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;
 
     let result = client
         .put_object()
@@ -233,6 +249,51 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
         assert_eq!(code, "PreconditionFailed");
     }
 
-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.start().await?;
+    cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
+
+    let writer = cluster.create_s3_client(0)?;
+    let reader = cluster.create_s3_client(1)?;
+    assert_bucket_cors_missing(&reader).await;
+
+    let rule = CorsRule::builder()
+        .allowed_methods("GET")
+        .allowed_origins("https://example.com")
+        .build()?;
+    let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
+
+    writer
+        .put_bucket_cors()
+        .bucket(BUCKET_METADATA_RELOAD_BUCKET)
+        .cors_configuration(configuration)
+        .send()
+        .await?;
+
+    let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
+    let rules = response.cors_rules();
+    assert_eq!(
+        rules.len(),
+        1,
+        "peer should observe the committed CORS rule before the write response returns"
+    );
+    assert_eq!(rules[0].allowed_methods(), ["GET"]);
+    assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
+
+    writer
+        .delete_bucket_cors()
+        .bucket(BUCKET_METADATA_RELOAD_BUCKET)
+        .send()
+        .await?;
+    assert_bucket_cors_missing(&reader).await;
+    writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
     Ok(())
 }

crates/e2e_test/src/common.rs

@@ -494,15 +494,20 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
-pub fn awscurl_available() -> bool {
-    let path = awscurl_binary_path();
-    if path.components().count() > 1 || path.is_absolute() {
-        return path.is_file();
+fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
+    let output = Command::new(path).arg("--help").output()?;
+    if output.status.success() {
+        return Ok(());
     }
 
-    std::env::var_os("PATH")
-        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
-        .unwrap_or(false)
+    Err(std::io::Error::other(format!(
+        "awscurl prerequisite check failed: {}",
+        String::from_utf8_lossy(&output.stderr).trim()
+    )))
+}
+
+pub fn require_awscurl() -> std::io::Result<()> {
+    verify_awscurl_path(&awscurl_binary_path())
 }
 
 // Global initialization
@@ -1747,6 +1752,22 @@ mod tests {
         assert_eq!(normalize_rustfs_build_features(" , "), None);
     }
 
+    #[test]
+    fn missing_awscurl_is_a_prerequisite_failure() {
+        let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
+
+        let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
+
+        assert_eq!(error.kind(), ErrorKind::NotFound);
+    }
+
+    #[test]
+    fn available_awscurl_client_passes_prerequisite_check() {
+        let executable = std::env::current_exe().expect("the test executable should have a path");
+
+        verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
+    }
+
     #[test]
     fn capture_log_path_uses_temp_directory_basename() {
         assert_eq!(

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -16,9 +16,7 @@
 //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
 //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
@@ -175,11 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
 #[tokio::test]
 async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2eiamtag-{suffix}");
     let user_secret = "longSecretKeyForTest123!";
@@ -233,11 +226,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
 #[tokio::test]
 async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let user = format!("e2ebptag-{suffix}");
     let user_secret = "longSecretKeyForTest456!";
@@ -294,11 +282,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
 #[tokio::test]
 async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-par-{suffix}");
     let parent_secret = "longSecretKeyForParentSts99!";
@@ -370,11 +353,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
 #[tokio::test]
 async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
-        return Ok(());
-    }
-
     let suffix = Uuid::new_v4();
     let parent = format!("e2e-sts-del-par-{suffix}");
     let parent_secret = "longSecretKeyForParentDelete99!";

crates/e2e_test/src/group_delete_test.rs

@@ -14,7 +14,7 @@
 
 //! E2E tests for group management (fixes #2028).
 
-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
 
 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
     env.start_rustfs_server(vec![]).await?;
 
     // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
     let user_body = serde_json::json!({
         "secretKey": "testuser1secret",
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created testuser1");
 
     // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": "testgroup",
         "members": ["testuser1"],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added testuser1 to testgroup");
 
     // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
     info!("Delete of non-empty group correctly rejected");
 
     // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
         "isRemove": true,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
     info!("Removed testuser1 from testgroup");
 
     // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
     info!("Deleted empty testgroup successfully");
 
     // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
     info!("Confirmed testgroup no longer exists");
 
     Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
         "Statement": [{
             "Effect": "Allow",
             "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
         }]
     });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
     info!("Created canned policy {}", policy_name);
 
     // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {} with no explicit policy", user_name);
 
     // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
     info!("Attached policy {} to group {}", policy_name, group_name);
 
     // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
     let group_name = "soledeletegroup";
 
     // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {}", user_name);
 
     // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
     info!("Deleted user {}", user_name);
 
     // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
     info!("Deleted group {} after user was removed", group_name);
 
     Ok(())

crates/e2e_test/src/kms/common.rs

@@ -22,9 +22,7 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
@@ -59,15 +57,6 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
-pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
-    if awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping {} because awscurl is not available in PATH", test_name);
-    true
-}
-
 pub fn sse_customer_key_md5_base64(key: &str) -> String {
     let mut hasher = Md5::new();
     hasher.update(key.as_bytes());
@@ -490,10 +479,6 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
-        return Ok(());
-    }
-
     info!("Testing KMS key management APIs");
 
     // Test CreateKey

crates/e2e_test/src/kms/kms_local_test.rs

@@ -20,8 +20,7 @@
 //! - Complete encryption/decryption lifecycle
 
 use super::common::{
-    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    test_kms_key_management, test_sse_c_encryption,
+    LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
@@ -29,9 +28,6 @@ use tracing::{error, info};
 #[tokio::test]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -22,8 +22,8 @@ use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
+    test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
     test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
@@ -62,9 +62,6 @@ impl VaultKmsTestContext {
 #[tokio::test]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
-        return Ok(());
-    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -117,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
-        return Ok(());
-    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -203,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[tokio::test]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
-        return Ok(());
-    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -267,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
-        return Ok(());
-    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -297,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[tokio::test]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
-        return Ok(());
-    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/lib.rs

@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
+// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
+// replication metadata must not overwrite a newer local category state.
+#[cfg(test)]
+mod replication_lww_receiver_test;
+
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;

crates/e2e_test/src/mc_mirror_small_bucket_test.rs

@@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
     Ok(())
 }
 
-fn mc_available() -> bool {
-    Command::new("mc")
-        .arg("--version")
-        .output()
-        .is_ok_and(|output| output.status.success())
-}
-
 fn run_mc(args: &[&str]) -> TestResult {
     let output = Command::new("mc").args(args).output()?;
     if !output.status.success() {
@@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize {
 async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
     crate::common::init_logging();
     info!("Starting issue #3107 mc mirror regression test");
-    if !mc_available() {
-        info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
-        return Ok(());
-    }
+    run_mc(&["--version"])?;
 
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;

crates/e2e_test/src/multipart_auth_test.rs

@@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
 async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
-    if !crate::common::awscurl_available() {
-        return Ok(());
-    }
-
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/e2e_test/src/protocols/sftp_helpers.rs

@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use russh::client::{self, Handle};
 use russh::keys::ssh_key::LineEnding;
-use russh::keys::{Algorithm, PrivateKey, PublicKey};
+use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
 use russh_sftp::client::SftpSession;
 use russh_sftp::protocol::OpenFlags;
 use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
 impl client::Handler for AcceptAnyServerKey {
     type Error = anyhow::Error;
 
-    async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
+    async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
         Ok(true)
     }
 }

crates/e2e_test/src/quota_test.rs

@@ -18,15 +18,6 @@ use http::{Method, StatusCode};
 use tokio::time::{Duration, sleep, timeout};
 use tracing::{debug, info};
 
-fn skip_without_awscurl() -> bool {
-    if crate::common::awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping quota test because awscurl is not available");
-    true
-}
-
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -276,9 +267,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -320,9 +308,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -371,9 +356,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -406,9 +388,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -442,9 +421,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -480,9 +456,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -513,9 +486,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -553,9 +523,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -593,9 +560,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -628,9 +592,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -689,9 +650,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -744,9 +702,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -789,9 +744,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -847,9 +799,6 @@ mod integration_tests {
     #[tokio::test]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -1,52 +1,26 @@
 #![cfg(test)]
 
-use aws_config::meta::region::RegionProviderChain;
+use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use bytes::Bytes;
 use std::error::Error;
+use std::fmt::Debug;
 
-const ENDPOINT: &str = "http://localhost:9000";
-const ACCESS_KEY: &str = "rustfsadmin";
-const SECRET_KEY: &str = "rustfsadmin";
-const BUCKET: &str = "api-test";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
 
-async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
-    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
-    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
-        .region(region_provider)
-        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
-        .endpoint_url(ENDPOINT)
-        .load()
-        .await;
-
-    let client = Client::from_conf(
-        aws_sdk_s3::Config::from(&shared_config)
-            .to_builder()
-            .force_path_style(true)
-            .build(),
+fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
+where
+    T: Debug,
+    E: ProvideErrorMetadata + Debug,
+{
+    let error = result.expect_err("conditional request must fail");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some(expected),
+        "unexpected conditional request error: {error:?}"
     );
-    Ok(client)
-}
-
-/// Setup test bucket, creating it if it doesn't exist
-async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
-    match client.create_bucket().bucket(BUCKET).send().await {
-        Ok(_) => {}
-        Err(SdkError::ServiceError(e)) => {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            if !error_code.eq("BucketAlreadyExists") {
-                return Err(e.into());
-            }
-        }
-        Err(e) => {
-            return Err(e.into());
-        }
-    }
-    Ok(())
 }
 
 /// Generate test data of specified size
@@ -60,7 +34,12 @@ fn generate_test_data(size: usize) -> Vec<u8> {
 }
 
 /// Upload an object and return its ETag
-async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
+async fn upload_object_with_metadata(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+) -> Result<String, Box<dyn Error + Send + Sync>> {
     let response = client
         .put_object()
         .bucket(bucket)
@@ -69,188 +48,164 @@ async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, d
         .send()
         .await?;
 
-    let etag = response.e_tag().unwrap_or("").to_string();
-    Ok(etag)
+    response
+        .e_tag()
+        .map(str::to_owned)
+        .ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
 }
 
-/// Cleanup test objects from bucket
-async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
-    for key in keys {
-        let _ = client.delete_object().bucket(bucket).key(*key).send().await;
-    }
-}
-
-/// Generate unique test object key
-fn generate_test_key(prefix: &str) -> String {
-    use std::time::{SystemTime, UNIX_EPOCH};
-    let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
-    format!("{prefix}-{timestamp}")
+async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
+    let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
+    Ok(response.body.collect().await?.into_bytes())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_okay() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-ok");
+    let test_key = "conditional-put-ok";
     let initial_data = generate_test_data(1024); // 1KB test data
-    let updated_data = generate_test_data(2048); // 2KB updated data
+    let matching_data = generate_test_data(2048); // 2KB updated data
+    let non_matching_data = generate_test_data(3072); // 3KB updated data
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with matching If-Match condition (should succeed)
-    let response1 = client
+    client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(matching_data.clone()).into())
         .if_match(&initial_etag)
         .send()
-        .await;
-    assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
 
     // Test 2: PUT with non-matching If-None-Match condition (should succeed)
     let fake_etag = "\"fake-etag-12345\"";
-    let response2 = client
+    client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(non_matching_data.clone()).into())
         .if_none_match(fake_etag)
         .send()
-        .await;
-    assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_failed() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("conditional-put-failed");
+    let test_key = "conditional-put-failed";
     let initial_data = generate_test_data(1024);
     let updated_data = generate_test_data(2048);
 
     // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
 
     // Test 1: PUT with non-matching If-Match condition (should fail with 412)
     let fake_etag = "\"fake-etag-should-not-match\"";
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_match(fake_etag)
         .send()
         .await;
 
-    assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
 
     // Test 2: PUT with matching If-None-Match condition (should fail with 412)
     let response2 = client
         .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .body(Bytes::from(updated_data.clone()).into())
         .if_none_match(&initial_etag)
         .send()
         .await;
 
-    assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
-    if let Err(e) = response2 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
-
-    // Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+    assert_s3_error_code(response2, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
 
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let key = "some_key";
-    cleanup_objects(&client, BUCKET, &[key]).await;
+    let key = "conditional-put-missing";
 
     // When the object does not exist, the If-Match condition should always fail
     let response1 = client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
         .body(Bytes::from(generate_test_data(1024)).into())
         .if_match("*")
         .send()
         .await;
-    assert!(response1.is_err());
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("NoSuchKey", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "NoSuchKey");
 
     // When the object does not exist, the If-None-Match condition should be able to succeed
-    let response2 = client
+    let created_data = generate_test_data(1024);
+    client
         .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
         .key(key)
-        .body(Bytes::from(generate_test_data(1024)).into())
+        .body(Bytes::from(created_data.clone()).into())
         .if_none_match("*")
         .send()
-        .await;
-    assert!(response2.is_ok());
+        .await?;
+    assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
 
-    cleanup_objects(&client, BUCKET, &[key]).await;
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_multi_part_upload() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();
 
-    let test_key = generate_test_key("multipart-upload-ok");
+    let test_key = "conditional-multipart-upload";
     let test_data = generate_test_data(1024);
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
 
     let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
     let num_parts = 3;
     let mut parts = Vec::new();
+    let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
 
     // Initiate multipart upload
-    let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
+    let initiate_response = client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .send()
+        .await?;
 
     let upload_id = initiate_response
         .upload_id()
@@ -258,12 +213,13 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     // Upload parts
     for part_number in 1..=num_parts {
-        let part_data = generate_test_data(part_size);
+        let part_data = vec![u8::try_from(part_number)?; part_size];
+        expected_data.extend_from_slice(&part_data);
 
         let upload_part_response = client
             .upload_part()
-            .bucket(BUCKET)
-            .key(&test_key)
+            .bucket(TEST_BUCKET)
+            .key(test_key)
             .upload_id(upload_id)
             .part_number(part_number)
             .body(Bytes::from(part_data).into())
@@ -286,57 +242,62 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
     // Test 1: Multipart upload with wildcard If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match("*")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 2: Multipart upload with matching If-None-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_none_match(initial_etag.clone())
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
 
     // Test 3: Multipart upload with unmatching If-Match, should fail
     let complete_response = client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
         .multipart_upload(completed_upload.clone())
         .if_match("\"abcdef\"")
         .send()
         .await;
 
-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
+
+    let staged_parts = client
+        .list_parts()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
 
     // Test 4: Multipart upload with matching If-Match, should succeed
-    let complete_response = client
+    client
         .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
         .upload_id(upload_id)
-        .multipart_upload(completed_upload.clone())
+        .multipart_upload(completed_upload)
         .if_match(initial_etag)
         .send()
-        .await;
-
-    assert!(complete_response.is_ok());
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
 
     Ok(())
 }

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -13,207 +13,228 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::workspace_root;
+use crate::common::RustFSTestEnvironment;
 use crate::storage_api::node_interact::{
     TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
 };
-use futures::future::join_all;
+use aws_sdk_s3::primitives::ByteStream;
 use rmp_serde::{Deserializer, Serializer};
-use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
+use rustfs_filemeta::MetaCacheEntry;
 use rustfs_protos::proto_gen::node_service::WalkDirRequest;
 use rustfs_protos::{
     models::{PingBody, PingBodyBuilder},
-    proto_gen::node_service::{
-        ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
-    },
+    proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
 };
 use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::io::Cursor;
-use std::path::PathBuf;
-use tokio::spawn;
 use tonic::Request;
 use tonic::codegen::tokio_stream::StreamExt;
 
-const CLUSTER_ADDR: &str = "http://localhost:9000";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
 
 fn signature_interceptor() -> TonicInterceptor {
     TonicInterceptor::Signature(gen_tonic_signature_interceptor())
 }
 
+fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
+    std::io::Error::other(error.to_string())
+}
+
+async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
+    let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
+    let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
+    assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_without_cleanup_with_env(&[
+        ("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
+        ("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
+        ("RUST_LOG", "error"),
+    ])
+    .await?;
+    Ok(env)
+}
+
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn ping() -> Result<(), Box<dyn Error>> {
+async fn ping() -> TestResult {
+    let env = start_server().await?;
     let mut fbb = flatbuffers::FlatBufferBuilder::new();
     let payload = fbb.create_vector(b"hello world");
-
     let mut builder = PingBodyBuilder::new(&mut fbb);
     builder.add_payload(payload);
     let root = builder.finish();
     fbb.finish(root, None);
 
-    let finished_data = fbb.finished_data();
-
-    let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
-    assert!(decoded_payload.is_ok());
-
-    // Create client
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-
-    // Construct PingRequest
-    let request = Request::new(PingRequest {
-        version: 1,
-        body: bytes::Bytes::copy_from_slice(finished_data),
-    });
-
-    // Send request and get response
-    let response: PingResponse = client.ping(request).await?.into_inner();
-
-    // Print response
-    let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
-    if let Err(e) = ping_response_body {
-        eprintln!("{e}");
-    } else {
-        println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
-    }
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .ping(Request::new(PingRequest {
+            version: 1,
+            body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
+        }))
+        .await?
+        .into_inner();
 
+    assert_eq!(response.version, 1);
+    let body = flatbuffers::root::<PingBody>(&response.body)?;
+    assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn make_volume() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(MakeVolumeRequest {
-        disk: "data".to_string(),
-        volume: "dandan".to_string(),
-    });
+async fn make_volume() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-volume".to_string(),
+        }))
+        .await?
+        .into_inner();
 
-    let response = client.make_volume(request).await?.into_inner();
-    if response.success {
-        println!("success");
-    } else {
-        println!("failed: {:?}", response.error);
-    }
+    assert!(response.success, "make_volume failed: {:?}", response.error);
+    assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn list_volumes() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ListVolumesRequest {
-        disk: "data".to_string(),
-    });
+async fn list_volumes() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-listed-volume".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);
 
-    let response = client.list_volumes(request).await?.into_inner();
-    let volume_infos: Vec<VolumeInfo> = response
+    let response = client
+        .list_volumes(Request::new(ListVolumesRequest {
+            disk: env.temp_dir.clone(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "list_volumes failed: {:?}", response.error);
+    let volumes = response
         .volume_infos
-        .into_iter()
-        .filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
-        .collect();
-
-    println!("{volume_infos:?}");
+        .iter()
+        .map(|json| serde_json::from_str::<VolumeInfo>(json))
+        .collect::<Result<Vec<_>, _>>()?;
+    assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
     Ok(())
 }
 
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn walk_dir() -> Result<(), Box<dyn Error>> {
-    println!("walk_dir");
-    // TODO: use writer
+async fn walk_dir() -> TestResult {
+    let env = start_server().await?;
+    let s3 = env.create_s3_client();
+    let bucket = "node-rpc-walk-bucket";
+    let key = "prefix/object.txt";
+    env.create_test_bucket(bucket).await?;
+    s3.put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"walk payload"))
+        .send()
+        .await?;
+
     let opts = WalkDirOptions {
-        bucket: "dandan".to_owned(),
-        base_dir: "".to_owned(),
+        bucket: bucket.to_string(),
         recursive: true,
         ..Default::default()
     };
-    let (rd, mut wr) = tokio::io::duplex(1024);
-    let mut buf = Vec::new();
-    opts.serialize(&mut Serializer::new(&mut buf))?;
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
-        let mut path = workspace_root();
-        path.push("target");
-        path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
-        path.push("data");
-        path
-    });
-    let request = Request::new(WalkDirRequest {
-        disk: disk_path.to_string_lossy().into_owned(),
-        walk_dir_options: buf.into(),
-    });
-    let mut response = client.walk_dir(request).await?.into_inner();
+    let mut encoded = Vec::new();
+    opts.serialize(&mut Serializer::new(&mut encoded))?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let mut stream = client
+        .walk_dir(Request::new(WalkDirRequest {
+            disk: env.temp_dir.clone(),
+            walk_dir_options: encoded.into(),
+        }))
+        .await?
+        .into_inner();
 
-    let job1 = spawn(async move {
-        let mut out = MetacacheWriter::new(&mut wr);
-        loop {
-            match response.next().await {
-                Some(Ok(resp)) => {
-                    if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
-                    }
-                    let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
-                        .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
-                        .unwrap();
-                    out.write_obj(&entry).await.unwrap();
-                }
-                None => {
-                    let _ = out.close().await;
-                    break;
-                }
-                _ => {
-                    println!("Unexpected response: {response:?}");
-                    let _ = out.close().await;
-                    break;
-                }
-            }
-        }
-    });
-    let job2 = spawn(async move {
-        let mut reader = MetacacheReader::new(rd);
-        while let Ok(Some(entry)) = reader.peek().await {
-            println!("{entry:?}");
-        }
-    });
-
-    join_all(vec![job1, job2]).await;
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn read_all() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ReadAllRequest {
-        disk: "data".to_string(),
-        volume: "ff".to_string(),
-        path: "format.json".to_string(),
-    });
-
-    let response = client.read_all(request).await?.into_inner();
-    let volume_infos = response.data;
-
-    println!("{}", response.success);
-    println!("{volume_infos:?}");
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn storage_info() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(LocalStorageInfoRequest { metrics: true });
-
-    let response = client.local_storage_info(request).await?.into_inner();
-    if !response.success {
-        println!("{:?}", response.error_info);
-        return Ok(());
+    let mut entries = Vec::new();
+    while let Some(response) = stream.next().await {
+        let response = response?;
+        assert!(response.success, "walk_dir failed: {:?}", response.error_info);
+        entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
     }
-    let info = response.storage_info;
-
-    let mut buf = Deserializer::new(Cursor::new(info));
-    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
-    println!("{storage_info:?}");
+    assert!(
+        entries.iter().any(|entry| entry.name == key),
+        "walk_dir did not return {key}: {entries:?}"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn read_all() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let volume = "node-rpc-read-volume";
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);
+    tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
+
+    let response = client
+        .read_all(Request::new(ReadAllRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+            path: "payload.bin".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "read_all failed: {:?}", response.error);
+    assert_eq!(response.data.as_ref(), b"read payload");
+    Ok(())
+}
+
+#[tokio::test]
+async fn storage_info() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
+        .await?
+        .into_inner();
+    assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
+
+    let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
+    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
+    let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
+    assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
+    assert!(
+        storage_info
+            .disks
+            .iter()
+            .any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
+        "local_storage_info did not include the configured disk: {:?}",
+        storage_info.disks
+    );
     Ok(())
 }

crates/e2e_test/src/replication_extension_test.rs

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
     FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -57,7 +56,7 @@ use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
-use s3s::header::X_AMZ_REPLICATION_STATUS;
+use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::convert::Infallible;
@@ -2023,6 +2022,7 @@ async fn forward_replication_proxy_request(
     client: &reqwest::Client,
     request_count: &AtomicU64,
     mut replication_enabled: watch::Receiver<bool>,
+    mut held_tagging: watch::Receiver<Option<String>>,
 ) -> Response<Full<bytes::Bytes>> {
     let (parts, body) = request.into_parts();
     let is_replication = parts
@@ -2036,6 +2036,17 @@ async fn forward_replication_proxy_request(
                 return proxy_error_response("replication gate closed");
             }
         }
+        // Content-keyed hold: park only the replication request whose
+        // `x-amz-tagging` matches the held value, letting every other delivery
+        // through, so a test can make one specific (stale) delivery the last
+        // write the backend sees.
+        if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
+            while held_tagging.borrow().as_deref() == Some(tagging) {
+                if held_tagging.changed().await.is_err() {
+                    return proxy_error_response("replication tag hold closed");
+                }
+            }
+        }
     }
 
     let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2070,12 +2081,26 @@ async fn start_replication_counting_proxy(
     backend_url: &str,
     tasks: &mut JoinSet<()>,
 ) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
+    let (proxy_url, request_count, replication_enabled, _held_tagging) =
+        start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
+    Ok((proxy_url, request_count, replication_enabled))
+}
+
+/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
+/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
+/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
+/// counted); all other traffic flows. Send `None` to release them.
+async fn start_replication_counting_proxy_with_tag_hold(
+    backend_url: &str,
+    tasks: &mut JoinSet<()>,
+) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
     let listener = TcpListener::bind("127.0.0.1:0").await?;
     let proxy_url = format!("http://{}", listener.local_addr()?);
     let backend_url = backend_url.to_string();
     let request_count = Arc::new(AtomicU64::new(0));
     let task_request_count = request_count.clone();
     let (replication_enabled, task_replication_enabled) = watch::channel(true);
+    let (held_tagging, task_held_tagging) = watch::channel(None);
     tasks.spawn(async move {
         let client = local_http_client();
         let mut connections = JoinSet::new();
@@ -2087,12 +2112,14 @@ async fn start_replication_counting_proxy(
                     let client = client.clone();
                     let request_count = task_request_count.clone();
                     let replication_enabled = task_replication_enabled.clone();
+                    let held_tagging = task_held_tagging.clone();
                     connections.spawn(async move {
                         let service = service_fn(move |request| {
                             let backend_url = backend_url.clone();
                             let client = client.clone();
                             let request_count = request_count.clone();
                             let replication_enabled = replication_enabled.clone();
+                            let held_tagging = held_tagging.clone();
                             async move {
                                 Ok::<_, Infallible>(
                                     forward_replication_proxy_request(
@@ -2101,6 +2128,7 @@ async fn start_replication_counting_proxy(
                                         &client,
                                         &request_count,
                                         replication_enabled,
+                                        held_tagging,
                                     )
                                     .await,
                                 )
@@ -2113,7 +2141,7 @@ async fn start_replication_counting_proxy(
             }
         }
     });
-    Ok((proxy_url, request_count, replication_enabled))
+    Ok((proxy_url, request_count, replication_enabled, held_tagging))
 }
 
 async fn site_replication_remove(
@@ -6949,6 +6977,395 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
     }
 }
 
+/// Replication status a site reports for one object version via HEAD
+/// (`x-amz-replication-status`), or `None` when the header is absent.
+async fn head_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let head = client
+        .head_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(head.replication_status().map(|status| status.as_str().to_string()))
+}
+
+/// Poll one site until the version's replication status is one of `expected`.
+async fn wait_for_version_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &[&str],
+    site: &str,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let last = head_replication_status(client, bucket, key, version_id).await?;
+        if let Some(status) = last.as_deref()
+            && expected.contains(&status)
+        {
+            return Ok(status.to_string());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+async fn put_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .tagging(
+            aws_sdk_s3::types::Tagging::builder()
+                .tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn get_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string()))
+}
+
+/// Poll one site until the version's `tag_key` equals `expected`.
+async fn wait_for_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    expected: &str,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
+        if observed.as_deref() == Some(expected) {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+/// Tag key the dual-node LWW scenario edits on both sites.
+const LWW_TAG_KEY: &str = "owner";
+
+/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
+/// full quiet window (no late stale delivery flips it back).
+async fn assert_tag_stable_on_both_sites(
+    site_a_client: &Client,
+    site_b_client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &str,
+    quiet: Duration,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + quiet;
+    loop {
+        let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
+        assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(());
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+}
+
+/// Wait until the counting proxy in front of a site has admitted `expected`
+/// replication requests in total (requests held by a closed gate still count).
+async fn wait_for_proxy_replication_requests(
+    counter: &AtomicU64,
+    expected: u64,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = counter.load(Ordering::Relaxed);
+        if observed >= expected {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
+        }
+        sleep(Duration::from_millis(25)).await;
+    }
+}
+
+/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
+/// metadata categories, exercised end to end over the real dual-node
+/// active-active site-replication control plane — sender, worker, status
+/// bookkeeping and persisted failure recovery all participate (the single-server
+/// `replication_lww_receiver_test` only injects authorized replication PUTs).
+///
+/// Scenario on one versioned object:
+/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
+///    newer tag and leave the author COMPLETED / the receiver REPLICA;
+/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
+///    authors a newer edit that reaches A first; releasing the stale delivery
+///    must NOT roll B back — both sites settle on B's value and stay there
+///    through a quiet window, with no FAILED/PENDING status left behind;
+/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
+///    then B returns and the scanner-replayed edit converges both sites forward.
+/// Durable metadata-MRF serialization/reconstruction is covered separately by
+/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
+#[tokio::test]
+async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
+    init_logging();
+
+    match tokio::time::timeout(Duration::from_secs(420), async {
+        // The scanner is fast for the final persisted-failure recovery phase.
+        // Step 2 finishes and proves a quiet stable winner before that phase,
+        // so a later scanner pass cannot mask its stale-delivery assertion.
+        let mut site_env = replication_fast_env();
+        site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+        site_env.extend_from_slice(FAST_SCANNER_ENV);
+
+        let mut site_a_env = RustFSTestEnvironment::new().await?;
+        site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut site_b_env = RustFSTestEnvironment::new().await?;
+        site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut proxy_tasks = JoinSet::new();
+        let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
+        let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
+
+        let site_a_client = site_a_env.create_s3_client();
+        let site_b_client = site_b_env.create_s3_client();
+        let bucket = "site-repl-tag-lww";
+        let key = "lww.txt";
+
+        let add_status = site_replication_add(
+            &site_a_env,
+            &[
+                PeerSite {
+                    name: "lww-site-a".to_string(),
+                    endpoint: site_a_env.url.clone(),
+                    access_key: site_a_env.access_key.clone(),
+                    secret_key: site_a_env.secret_key.clone(),
+                    ..Default::default()
+                },
+                PeerSite {
+                    name: "lww-site-b".to_string(),
+                    endpoint: site_b_env.url.clone(),
+                    access_key: site_b_env.access_key.clone(),
+                    secret_key: site_b_env.secret_key.clone(),
+                    ..Default::default()
+                },
+            ],
+        )
+        .await?;
+        assert!(add_status.success, "unexpected site add result: {add_status:?}");
+
+        let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        // Route both directions through the counting proxies so inbound
+        // replication to B can be held (out-of-order delivery) and observed.
+        for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
+            let mut peer = site_info
+                .sites
+                .iter()
+                .find(|peer| peer.endpoint == *env_url)
+                .ok_or_else(|| format!("site {label} peer missing from replication info"))?
+                .clone();
+            peer.endpoint = proxy_url.clone();
+            peer.sync_state = SyncStatus::Enable;
+            let edit = site_replication_edit(&site_a_env, "", &peer).await?;
+            assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
+        }
+        for env in [&site_a_env, &site_b_env] {
+            wait_for_site_replication_info(env, |info| {
+                info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
+                    && info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
+            })
+            .await?;
+        }
+
+        site_a_client.create_bucket().bucket(bucket).send().await?;
+        wait_for_bucket_on_target(&site_b_client, bucket).await?;
+
+        let version_id = site_a_client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(b"tag lww payload"))
+            .send()
+            .await?
+            .version_id()
+            .ok_or("site A PUT omitted version ID")?
+            .to_string();
+        wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
+
+        // --- 1. reciprocal edits in real order: A then B ----------------------
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
+
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
+            .await?;
+
+        // --- 2. concurrent edits, stale delivery last ------------------------
+        // Both sites edit the same version while each other's delivery is
+        // parked at the peer's inbound proxy (content-keyed: only the
+        // `owner=a2` / `owner=b2` replication PUTs wait, everything else
+        // flows). B's edit is the newer one. Releasing A's stale `a2` first
+        // makes it the last write B sees while A itself still holds `a2`, so
+        // nothing A could re-deliver carries the winner: only receiver-side
+        // LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
+        site_b_held_tagging.send(Some("owner=a2".to_string()))?;
+        site_a_held_tagging.send(Some("owner=b2".to_string()))?;
+        let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
+        let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
+        sleep(Duration::from_millis(50)).await;
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
+        wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
+        assert_eq!(
+            get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("a2")
+        );
+        assert_eq!(
+            get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("b2")
+        );
+
+        // Release the stale a2 delivery onto B: the newer local b2 must
+        // survive, and the delivery itself must still succeed (A reaches
+        // COMPLETED instead of looping through MRF with the stale value).
+        site_b_held_tagging.send(None)?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
+        loop {
+            assert_eq!(
+                get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                    .await?
+                    .as_deref(),
+                Some("b2"),
+                "a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
+            );
+            if tokio::time::Instant::now() >= stale_deadline {
+                break;
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        // Release b2 onto A: the newer edit wins there and both sites settle.
+        // B's own version may legitimately read REPLICA here: the stale inbound
+        // a2 write re-labelled it as a replica write (keeping B's tags); what
+        // must not remain is PENDING/FAILED.
+        site_a_held_tagging.send(None)?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
+            .await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
+            .await?;
+        for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
+            let status = head_replication_status(client, bucket, key, &version_id).await?;
+            assert!(
+                matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
+                "{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
+            );
+        }
+
+        // --- 3. persisted FAILED state survives a source restart ------------
+        site_b_env.stop_server();
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
+        site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
+            .await?;
+
+        // The object itself never forked: one version on each side.
+        tokio::time::timeout(
+            Duration::from_secs(70),
+            assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
+        )
+        .await??;
+        let state = list_replication_state(&site_a_client, bucket).await?;
+        assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
+        assert_eq!(state[0].version_id, version_id);
+
+        proxy_tasks.abort_all();
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err("site replication tagging LWW test timed out".into()),
+    }
+}
 #[tokio::test]
 async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
@@ -7281,11 +7698,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
 async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
     init_logging();
 
-    if !awscurl_available() {
-        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
-        return Ok(());
-    }
-
     let mut source_env = RustFSTestEnvironment::new().await?;
     source_env
         .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)

crates/e2e_test/src/replication_lww_receiver_test.rs

@@ -0,0 +1,155 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
+//! A4/P1-6).
+//!
+//! In an active-active topology both sites' metadata states arrive at the
+//! peer as authorized replication PUTs carrying per-category source
+//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
+//! family). Before the fix the receiver applied them unconditionally, so a
+//! stale delivery overwrote a newer local state and the two sites diverged
+//! permanently while both reported COMPLETED. This test drives one live
+//! `rustfs` server with simulated inbound replication PUTs for the same
+//! object version and asserts the newer tagging state wins regardless of
+//! delivery order, while a stale delivery still succeeds at the object level
+//! (a failure would loop through MRF re-delivering the stale value).
+//!
+//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
+//! is covered by
+//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
+//! this file stays as the fast, single-process receiver check.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
+const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
+const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
+const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
+
+const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
+const T_STALE: &str = "2026-01-01T00:00:01Z";
+const T_LOCAL: &str = "2026-02-01T00:00:00Z";
+const T_NEWER: &str = "2026-03-01T00:00:00Z";
+
+/// Simulated inbound authorized replication PUT: same object version, tags and
+/// the source-authored tagging timestamp carried in transport headers.
+async fn inbound_replication_put(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tags: &str,
+    tagging_timestamp: &str,
+) -> TestResult {
+    let version_id = version_id.to_string();
+    let tagging_timestamp = tagging_timestamp.to_string();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"lww-e2e-body"))
+        .tagging(tags)
+        .customize()
+        .mutate_request(move |req| {
+            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
+            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
+            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
+            req.headers_mut()
+                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
+        })
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await
+        .expect("object tagging should be readable");
+    tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let client = env.create_s3_client();
+
+    let bucket = "replication-lww-receiver";
+    let key = "object";
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // First delivery establishes version V with tags stamped T_LOCAL.
+    let version_id = uuid::Uuid::new_v4().to_string();
+    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "the first delivery must establish the tagged version"
+    );
+
+    // A stale delivery (older source timestamp) must succeed at the object
+    // level but must NOT overwrite the newer tags.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
+    );
+
+    // A newer delivery still converges the version onto the newest state.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("newer"),
+        "a newer inbound delivery must overwrite older tags"
+    );
+
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await?;
+    env.delete_test_bucket(bucket).await.ok();
+    Ok(())
+}

crates/e2e_test/src/security_boundary_test.rs

@@ -21,12 +21,11 @@
 //! - SSRF prevention (internal/private endpoints rejected for tiering)
 //! - Race condition handling (concurrent writes converge without corruption)
 
-use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
 use std::error::Error;
-use tracing::info;
 
 /// Oversized tagging payloads must be rejected by the per-object tag limit.
 ///
@@ -225,16 +224,12 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
 /// outcome — the internal endpoint is not accepted — is asserted here.
 ///
 /// The admin API is exercised via signed `awscurl` requests, matching the
-/// pattern used by the other admin-API E2E tests in this crate; the test is
-/// skipped when `awscurl` is not installed.
+/// pattern used by the other admin-API E2E tests in this crate. The full E2E
+/// lane installs and verifies the pinned `awscurl` prerequisite.
 #[tokio::test]
 async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
-    if !awscurl_available() {
-        info!("Skipping tiering URL validation test because awscurl is not available");
-        return Ok(());
-    }
-
+    require_awscurl()?;
     let mut env = RustFSTestEnvironment::new().await?;
     env.start_rustfs_server(vec![]).await?;
 

crates/ecstore/src/api/mod.rs

@@ -161,7 +161,7 @@ pub mod bucket {
         pub mod objectlock_sys {
             pub use crate::bucket::object_lock::objectlock_sys::{
                 BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
-                check_retention_for_modification, is_retention_active,
+                check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
             };
         }
     }
@@ -187,22 +187,24 @@ pub mod bucket {
         pub use crate::bucket::replication::{
             BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
             DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
-            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
-            REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
-            REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-            ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
+            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
+            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
+            REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+            REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
             ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
             ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
-            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
-            get_global_replication_stats, get_proxy_targets, init_background_replication,
-            invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
-            replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
-            resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
-            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
+            VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
+            complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
+            get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
+            invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
+            merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
+            read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
+            replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
+            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
+            should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
             unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
             version_purge_status_to_filemeta,
         };
@@ -456,11 +458,13 @@ pub mod rpc {
         ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
         check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
         gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
-        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
-        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
-        verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
-        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
+        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+        verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
+        verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
 use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
-    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
-    is_rustfs_header, is_standard_header, is_storageclass_header,
+    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
+    is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
 };
 use rustfs_utils::http::{
     SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,6 +1774,22 @@ impl PutObjectOptions {
             Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
         }
 
+        // MinIO PutObjectOptions.Header parity: object tags travel on the
+        // `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
+        // fills `user_tags` from the source version; without this header the
+        // whole-object transport delivered a tagless replica, so tag edits
+        // never reached the peer and the receiver-side LWW comparison
+        // (rustfs/backlog#1953) had nothing to judge.
+        if !self.user_tags.is_empty() {
+            let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
+            tags.sort();
+            let mut encoded = url::form_urlencoded::Serializer::new(String::new());
+            for (key, value) in tags {
+                encoded.append_pair(key, value);
+            }
+            Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
+        }
+
         for (k, v) in &self.user_metadata {
             let Ok(header_value) = HeaderValue::from_str(v) else {
                 warn!("skipping user metadata header with invalid value: {}", k);
@@ -3195,6 +3211,29 @@ mod tests {
         }
     }
 
+    #[test]
+    fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
+        // rustfs/backlog#1953: tag edits replicate through the whole-object
+        // transport, so the source tags must travel on x-amz-tagging.
+        let mut opts = PutObjectOptions::default();
+        opts.user_tags.insert("owner".to_string(), "site a".to_string());
+        opts.user_tags.insert("env".to_string(), "prod".to_string());
+
+        let header = opts.header();
+        let tagging = header
+            .get(AMZ_OBJECT_TAGGING_LOWER)
+            .expect("user tags must be transported on x-amz-tagging")
+            .to_str()
+            .expect("tag header must be ASCII");
+        // Deterministic key order; values are form-urlencoded.
+        assert_eq!(tagging, "env=prod&owner=site+a");
+
+        assert!(
+            PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
+            "a tagless source must not send an empty x-amz-tagging header"
+        );
+    }
+
     #[test]
     fn put_object_headers_omit_unset_replication_timestamps() {
         // UNIX_EPOCH means "never modified on the source"; sending it would

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport {
     }
 
     pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
+        let previous = self.clone();
+        let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
         let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
         for (reason, count) in &scan_report.tier_failure_by_reason {
             let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
-            tier_failure_by_reason.insert(*reason, current.max(*count));
+            let merged = if resumed_after_checkpoint {
+                current.saturating_add(*count)
+            } else {
+                current.max(*count)
+            };
+            tier_failure_by_reason.insert(*reason, merged);
         }
         let transition_completed = self.transition_completed;
         let transition_failed = self.transition_failed;
         *self = scan_report.clone();
+        if resumed_after_checkpoint {
+            self.scanned = previous.scanned.saturating_add(scan_report.scanned);
+            self.eligible = previous.eligible.saturating_add(scan_report.eligible);
+            self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
+            self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
+            self.skipped_not_transition = previous
+                .skipped_not_transition
+                .saturating_add(scan_report.skipped_not_transition);
+            self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
+            self.skipped_delete_marker = previous
+                .skipped_delete_marker
+                .saturating_add(scan_report.skipped_delete_marker);
+            self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
+            self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
+            self.skipped_already_transitioned = previous
+                .skipped_already_transitioned
+                .saturating_add(scan_report.skipped_already_transitioned);
+            self.skipped_already_in_flight = previous
+                .skipped_already_in_flight
+                .saturating_add(scan_report.skipped_already_in_flight);
+            self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
+            self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
+            self.skipped_queue_timeout = previous
+                .skipped_queue_timeout
+                .saturating_add(scan_report.skipped_queue_timeout);
+            self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
+        }
+        self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
+        self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
+        self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
+        self.cancelled = previous.cancelled || scan_report.cancelled;
         self.transition_completed = transition_completed;
         self.transition_failed = transition_failed;
-        self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
+        self.tier_failure = self.tier_failure.saturating_add(transition_failed);
         self.tier_failure_by_reason = tier_failure_by_reason;
     }
 
@@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken {
     version_marker: Option<String>,
 }
 
-fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
+pub(super) fn encode_manual_transition_continuation_token(
+    marker: Option<String>,
+    version_marker: Option<String>,
+) -> Option<String> {
     if marker.is_none() && version_marker.is_none() {
         return None;
     }
@@ -9136,6 +9177,7 @@ mod tests {
         assert_eq!(loaded.report.scanned, 37);
         assert_eq!(loaded.report.eligible, 11);
         assert_eq!(loaded.report.enqueued, 5);
+        assert_eq!(loaded.cursor_revision, Some(37));
         assert!(loaded.lease_expires_at_unix_nanos > 0);
         let token = loaded
             .report
@@ -9154,6 +9196,30 @@ mod tests {
         assert_eq!(admission.lease_id, loaded.lease_id);
         assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
 
+        let mut same_marker_report = report.clone();
+        same_marker_report.scanned += 1;
+        persist_manual_transition_page_checkpoint(
+            &checkpoint_options,
+            &same_marker_report,
+            Some("logs/page-end".to_string()),
+            Some("opaque-next-version".to_string()),
+        )
+        .await
+        .expect("same-marker version checkpoint should persist through the durable progress sink");
+        let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
+            .await
+            .expect("same-marker version checkpoint should reload");
+        assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
+        let (_, version_marker) = decode_manual_transition_continuation_token(
+            same_marker_checkpointed
+                .report
+                .continuation_token
+                .as_deref()
+                .expect("same-marker version checkpoint should persist a cursor"),
+        )
+        .expect("same-marker version cursor should decode");
+        assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
+
         create_test_bucket(&ecstore, &bucket).await;
         let lifecycle_xml = format!(
             r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9276,7 @@ mod tests {
         assert_eq!(checkpointed.report.scanned, 1000);
         assert_eq!(checkpointed.report.eligible, 1000);
         assert_eq!(checkpointed.report.dry_run_eligible, 1000);
+        assert_eq!(checkpointed.cursor_revision, Some(1000));
         let token = checkpointed
             .report
             .continuation_token

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use uuid::Uuid;
 
+#[cfg(test)]
+use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{
     ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
 };
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::{
+    MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
+    MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
@@ -34,10 +40,10 @@ use crate::store::ECStore;
 pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
 pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
 pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
-pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
-pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
-pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
-pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
+pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
 pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
 pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
 pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord {
     pub updated_at_unix_nanos: i128,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub completed_at_unix_nanos: Option<i128>,
+    #[serde(default, skip_serializing)]
+    pub cursor_revision: Option<u64>,
     pub report: ManualTransitionRunReport,
     pub queue_snapshot: ManualTransitionQueueSnapshot,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +232,7 @@ impl ManualTransitionJobRecord {
             created_at_unix_nanos: now,
             updated_at_unix_nanos: now,
             completed_at_unix_nanos: None,
+            cursor_revision: None,
             report: ManualTransitionRunReport {
                 bucket: bucket.to_string(),
                 prefix: options.prefix.clone(),
@@ -238,7 +247,7 @@ impl ManualTransitionJobRecord {
 
     pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         self.scan_completed = true;
-        self.report.merge_scan_report_preserving_worker(&report);
+        self.merge_scan_report(&report);
         self.queue_snapshot = queue_snapshot;
         self.error = None;
         self.mark_terminal_if_worker_drained();
@@ -316,7 +325,7 @@ impl ManualTransitionJobRecord {
             }
         }
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
     }
 
@@ -359,14 +368,14 @@ impl ManualTransitionJobRecord {
         self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
         self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
         true
     }
 
     pub fn mark_cancel_requested(&mut self) {
         self.cancel_requested = true;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
     }
 
     pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +389,7 @@ impl ManualTransitionJobRecord {
     pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
         if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
             self.lease_expires_at_unix_nanos = 0;
-            self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+            self.advance_updated_at();
         }
     }
 
@@ -398,7 +407,7 @@ impl ManualTransitionJobRecord {
 
     pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
         self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
         self.queue_snapshot = queue_snapshot;
     }
@@ -438,11 +447,16 @@ impl ManualTransitionJobRecord {
 
     pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         if self.state == ManualTransitionJobState::Running {
-            self.report.merge_scan_report_preserving_worker(&report);
+            self.merge_scan_report(&report);
             self.renew_lease(queue_snapshot);
         }
     }
 
+    fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
+        self.report.merge_scan_report_preserving_worker(report);
+        self.cursor_revision = manual_transition_cursor_revision(&self.report);
+    }
+
     pub fn mark_unknown_if_unowned(&mut self) {
         if self.state == ManualTransitionJobState::Running {
             self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
     }
 
     fn mark_updated_terminal(&mut self) {
+        self.advance_updated_at();
+        self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
+    }
+
+    fn advance_updated_at(&mut self) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
-        self.completed_at_unix_nanos = Some(now);
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
     }
 
     fn mark_terminal_if_worker_drained(&mut self) {
@@ -543,6 +561,7 @@ impl ManualTransitionJobRecord {
         if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
             job.report.cancelled = true;
         }
+        job.cursor_revision = manual_transition_cursor_revision(&job.report);
         job.validate()?;
         Ok(job)
     }
@@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
 pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current(
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
     config_boundary::save_config_with_opts_quiet(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 /// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     match config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
             Err(Error::PreconditionFailed)
         }
         result => result,
-    }
+    }?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
     job_id: Uuid,
     lease_id: Uuid,
 ) -> EcstoreResult<bool> {
-    let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
-        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
+    let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
+        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
         Ok(_) => return Ok(false),
         Err(Error::ConfigNotFound) => return Ok(true),
         Err(err) => return Err(err),
     };
     let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
+    let data = serde_json::to_vec(&admission).map_err(Error::other)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config_if_match(api, &object, &etag).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
         Err(Error::PreconditionFailed) => Ok(false),
@@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
     Error::other(err)
 }
 
+fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
+    report.continuation_token.as_ref()?;
+    (report.scanned > 0).then_some(report.scanned)
+}
+
 pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
     OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
 }
@@ -2210,6 +2240,76 @@ mod tests {
         );
     }
 
+    #[test]
+    fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let first_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                lifecycle_config_found: true,
+                scanned: 1000,
+                eligible: 800,
+                enqueued: 50,
+                dry_run_eligible: 10,
+                skipped_not_transition: 2,
+                skipped_tier: 3,
+                skipped_delete_marker: 4,
+                skipped_directory: 5,
+                skipped_replication: 6,
+                skipped_already_transitioned: 7,
+                skipped_already_in_flight: 8,
+                skipped_queue_full: 9,
+                skipped_queue_closed: 10,
+                skipped_queue_timeout: 11,
+                tier_failure: 12,
+                truncated_by_duration: true,
+                continuation_token: first_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        let next_token =
+            encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 3,
+                eligible: 2,
+                enqueued: 1,
+                dry_run_eligible: 1,
+                skipped_not_transition: 1,
+                tier_failure: 1,
+                continuation_token: next_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        assert_eq!(record.report.scanned, 1003);
+        assert_eq!(record.report.eligible, 802);
+        assert_eq!(record.report.enqueued, 51);
+        assert_eq!(record.report.dry_run_eligible, 11);
+        assert_eq!(record.report.skipped_not_transition, 3);
+        assert_eq!(record.report.skipped_tier, 3);
+        assert_eq!(record.report.skipped_delete_marker, 4);
+        assert_eq!(record.report.skipped_directory, 5);
+        assert_eq!(record.report.skipped_replication, 6);
+        assert_eq!(record.report.skipped_already_transitioned, 7);
+        assert_eq!(record.report.skipped_already_in_flight, 8);
+        assert_eq!(record.report.skipped_queue_full, 9);
+        assert_eq!(record.report.skipped_queue_closed, 10);
+        assert_eq!(record.report.skipped_queue_timeout, 11);
+        assert_eq!(record.report.tier_failure, 13);
+        assert!(record.report.lifecycle_config_found);
+        assert!(record.report.truncated_by_duration);
+        assert_eq!(record.report.continuation_token, next_token);
+        assert_eq!(record.cursor_revision, Some(1003));
+    }
+
     #[test]
     fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
         let options = ManualTransitionRunOptions::default();
@@ -2577,6 +2677,98 @@ mod tests {
         assert!(decoded.report.tier_failure_by_reason.is_empty());
     }
 
+    #[test]
+    fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 9,
+                continuation_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        let encoded = record.encode().expect("job record should encode");
+        let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        value["job"]
+            .as_object_mut()
+            .expect("job should be object")
+            .remove("cursor_revision");
+        let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
+        value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
+        let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
+
+        let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
+
+        assert_eq!(decoded.cursor_revision, Some(9));
+    }
+
+    #[test]
+    fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyPersistedManualTransitionJobRecord {
+            schema: String,
+            content_sha256: String,
+            job: LegacyManualTransitionJobRecord,
+        }
+
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyManualTransitionJobRecord {
+            job_id: Uuid,
+            scope_key: String,
+            bucket: String,
+            prefix: String,
+            tier: Option<String>,
+            dry_run: bool,
+            max_objects: Option<u64>,
+            max_duration: Option<std::time::Duration>,
+            owner_id: String,
+            lease_id: Uuid,
+            lease_expires_at_unix_nanos: i128,
+            state: ManualTransitionJobState,
+            scan_completed: bool,
+            cancel_requested: bool,
+            created_at_unix_nanos: i128,
+            updated_at_unix_nanos: i128,
+            completed_at_unix_nanos: Option<i128>,
+            report: ManualTransitionRunReport,
+            queue_snapshot: ManualTransitionQueueSnapshot,
+            error: Option<String>,
+        }
+
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 7,
+                continuation_token: continuation_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        assert_eq!(record.cursor_revision, Some(7));
+
+        let encoded = record.encode().expect("job record should encode");
+        let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        assert!(value["job"].get("cursor_revision").is_none());
+        let legacy: LegacyPersistedManualTransitionJobRecord =
+            serde_json::from_slice(&encoded).expect("old reader should accept new job record");
+
+        assert_eq!(legacy.job.job_id, record.job_id);
+        assert_eq!(legacy.job.report.continuation_token, continuation_token);
+    }
+
     #[test]
     fn manual_transition_job_record_rejects_unknown_report_fields() {
         let options = ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
 mod config_boundary;
 pub mod core;
+mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;
 pub mod transition_transaction;
+
+pub(crate) use durable_namespace::{
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
+};

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, warn};
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
     Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
-pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
+pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
         )
         .await?;
     }
+    let path = tier_delete_journal_object_name(je);
+    let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
+    let target_pool_indices = api
+        .record_durable_ilm_decommission_terminal_target_pools(&path, &data)
+        .await
+        .map_err(std::io::Error::other)?;
+    if let Some(target_pool_indices) = target_pool_indices {
+        for target_pool_idx in target_pool_indices {
+            match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
+                Ok(()) | Err(Error::ConfigNotFound) => {}
+                Err(err) => return Err(std::io::Error::other(err)),
+            }
+        }
+        return Ok(());
+    }
     remove_tier_delete_journal_entry(api, je).await
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -21,6 +21,7 @@ use tracing::{debug, warn};
 use uuid::Uuid;
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
-pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
+pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
 pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
 
 pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
     let object =
         transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
     let data = transaction.encode().map_err(transition_transaction_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
     TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
 }
 
-pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
-    let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
+pub(crate) async fn delete_transition_transaction_record(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<()> {
+    let object =
+        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
+    let data = transaction.encode().map_err(transition_transaction_store_error)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config(api, &object).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
         Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
     if probe != TransitionOperatorProbe::Missing {
         return Err(TransitionOperatorError::CandidateNotMissing(probe));
     }
-    delete_transition_transaction_record(api, transaction_id)
+    delete_transition_transaction_record(api, &transaction)
         .await
         .map_err(TransitionOperatorError::Store)
 }
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
     match transaction.state {
         TransitionTransactionState::Uploaded => {
             delete_transition_remote_candidate(api.clone(), transaction).await?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
             Ok(true) => {
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
             }
             Ok(false) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) if transition_source_is_missing(&err) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
         TransitionTransactionState::LocalCommitStarted => {
             match local_commit_matches_transaction(api.clone(), transaction).await {
                 Ok(true) => {
-                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    delete_transition_transaction_record(api, transaction).await?;
                     Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
                 }
                 Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
             }
         }
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
         .map_err(Error::other)?
     {
         TransitionCandidateProbe::Missing => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
             )
             .await
             .map_err(Error::other)?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
         .map_err(transition_transaction_store_error)?;
     save_transition_transaction_record(api.clone(), &cleanup).await?;
     delete_transition_remote_candidate(api.clone(), &cleanup).await?;
-    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    delete_transition_transaction_record(api, &cleanup).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -15,7 +15,7 @@
 use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
 use crate::bucket::object_lock::objectlock;
 use crate::error::{Error, Result, StorageError};
-use crate::object_api::ObjectInfo;
+use crate::object_api::{ObjectInfo, ObjectOptions};
 use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use std::sync::Arc;
@@ -136,12 +136,50 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
 
 /// Check if an object has legal hold enabled.
 /// Returns true if legal hold is ON.
-#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
     let lhold = objectlock::get_object_legalhold_meta(user_defined);
     matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
 }
 
+/// Whether an authorized replication write (`ObjectOptions::replication_request`)
+/// may overwrite a locked destination version.
+///
+/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
+/// skips the existing-version check for replicas), and a source-side hold
+/// release or retention change reaches this site only through this write. The
+/// overwrite is allowed only when the write carries the source timestamp of
+/// every category that currently locks the version, so receiver-side LWW
+/// (`merge_replication_metadata_lww`) judges each of them: a category locked
+/// more recently here is kept, otherwise the source's newer state wins. A write
+/// without that timestamp carries no source decision for the category — the
+/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
+///
+/// The locking categories come from the same authoritative evaluation as the
+/// commit-time WORM gate (`check_object_lock_for_deletion_with_state`): the
+/// bucket default retention locks a version that carries no explicit
+/// retention keys, so it is judged here too rather than read off the keys.
+/// Malformed persisted lock metadata or a non-authoritative bucket
+/// configuration is an error, never a pass.
+pub fn replication_write_may_pass_worm_gate(
+    state: &ObjectLockConfigState,
+    obj_info: &ObjectInfo,
+    opts: &ObjectOptions,
+) -> Result<bool> {
+    if !opts.replication_request {
+        return Ok(false);
+    }
+    if obj_info.delete_marker {
+        // Delete markers are never locked (same as the WORM gate).
+        return Ok(true);
+    }
+    let config = object_lock_config_from_state(state)?;
+    if legal_hold_locks(obj_info)? && opts.replication_legalhold_timestamp.is_none() {
+        return Ok(false);
+    }
+    let retention_locked = active_retention(config, obj_info)?.is_some();
+    Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
+}
+
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -239,69 +277,101 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
         return Ok(None);
     }
 
-    if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
-        if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
-            return Ok(Some(ObjectLockBlockReason::LegalHold));
-        }
-        if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
-            return Err(Error::other("persisted object legal-hold metadata is invalid"));
-        }
+    if legal_hold_locks(obj_info)? {
+        return Ok(Some(ObjectLockBlockReason::LegalHold));
     }
 
-    let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
-    let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
-    let explicit_ret = match (mode, retain_until) {
-        (None, None) => None,
+    if let Some((mode_str, retain_until)) = active_retention(config, obj_info)?
+        && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
+    {
+        return Ok(Some(reason));
+    }
+
+    Ok(None)
+}
+
+/// A cleared retention / legal hold is persisted as empty strings (the MinIO
+/// on-disk shape, `parse_object_lock_retention`); read it as "no lock" rather
+/// than as corrupt metadata.
+fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a String> {
+    obj_info.user_defined.get(key).filter(|value| !value.is_empty())
+}
+
+/// Whether the version's persisted legal hold is ON. Any other non-empty
+/// value than ON/OFF is malformed metadata and fails closed.
+fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
+    let Some(status) = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) else {
+        return Ok(false);
+    };
+    if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
+        return Ok(true);
+    }
+    if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
+        return Err(Error::other("persisted object legal-hold metadata is invalid"));
+    }
+    Ok(false)
+}
+
+/// The retention that currently locks the version, if any: the explicit
+/// persisted retention when the keys are present, otherwise the bucket
+/// default retention computed from the version's modification time. Returns
+/// `(mode, retain_until)` only while the retention is still active.
+fn active_retention<'a>(
+    config: Option<&'a ObjectLockConfiguration>,
+    obj_info: &ObjectInfo,
+) -> Result<Option<(&'a str, OffsetDateTime)>> {
+    let mode = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_MODE.as_str());
+    let retain_until = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
+    match (mode, retain_until) {
+        (None, None) => {}
         (Some(mode), Some(retain_until)) => {
             let mode =
                 objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
             let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
                 .map(Date::from)
                 .map_err(|_| Error::other("persisted object retention date is invalid"))?;
-            Some((mode, retain_until))
+            let mode_str = match mode.as_str() {
+                ObjectLockRetentionMode::COMPLIANCE => ObjectLockRetentionMode::COMPLIANCE,
+                ObjectLockRetentionMode::GOVERNANCE => ObjectLockRetentionMode::GOVERNANCE,
+                _ => return Err(Error::other("persisted object retention mode is invalid")),
+            };
+            return Ok(is_retention_active(mode_str, Some(&retain_until)).then(|| (mode_str, OffsetDateTime::from(retain_until))));
         }
         _ => return Err(Error::other("persisted object retention metadata is incomplete")),
+    }
+
+    let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
+        return Ok(None);
     };
-
-    if let Some((mode, retain_until)) = &explicit_ret {
-        let mode_str = mode.as_str();
-        if is_retention_active(mode_str, Some(retain_until))
-            && let Some(reason) =
-                check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
-        {
-            return Ok(Some(reason));
-        }
+    let Some(mode) = &default_retention.mode else {
+        return Ok(None);
+    };
+    let mode_str = mode.as_str();
+    if mode_str != ObjectLockRetentionMode::COMPLIANCE && mode_str != ObjectLockRetentionMode::GOVERNANCE {
+        return Ok(None);
     }
+    // Calculate retention expiration date from object modification time
+    let mod_time = obj_info
+        .mod_time
+        .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
+    let now = objectlock::utc_now_ntp();
+    let retain_until = if let Some(days) = default_retention.days {
+        mod_time.saturating_add(time::Duration::days(i64::from(days)))
+    } else {
+        let years = default_retention
+            .years
+            .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
+        add_years(mod_time, years)
+    };
+    Ok((retain_until.unix_timestamp() > now.unix_timestamp()).then_some((mode_str, retain_until)))
+}
 
-    if explicit_ret.is_none()
-        && let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
-        && let Some(mode) = &default_retention.mode
-    {
-        let mode_str = mode.as_str();
-        if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
-            // Calculate retention expiration date from object modification time
-            let mod_time = obj_info
-                .mod_time
-                .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
-            let now = objectlock::utc_now_ntp();
-            let retain_until = if let Some(days) = default_retention.days {
-                mod_time.saturating_add(time::Duration::days(i64::from(days)))
-            } else {
-                let years = default_retention
-                    .years
-                    .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
-                add_years(mod_time, years)
-            };
-
-            if retain_until.unix_timestamp() > now.unix_timestamp()
-                && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
-            {
-                return Ok(Some(reason));
-            }
-        }
+fn object_lock_config_from_state(state: &ObjectLockConfigState) -> Result<Option<&ObjectLockConfiguration>> {
+    match state {
+        ObjectLockConfigState::Configured { config, .. } => Ok(Some(config)),
+        ObjectLockConfigState::ConfirmedAbsent => Ok(None),
+        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
     }
-
-    Ok(None)
 }
 
 pub(crate) fn check_object_lock_for_deletion_with_state(
@@ -309,13 +379,7 @@ pub(crate) fn check_object_lock_for_deletion_with_state(
     obj_info: &ObjectInfo,
     bypass_governance: bool,
 ) -> Result<Option<ObjectLockBlockReason>> {
-    match state {
-        ObjectLockConfigState::Configured { config, .. } => {
-            check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
-        }
-        ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
-        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
-    }
+    check_object_lock_for_deletion_with_config(object_lock_config_from_state(state)?, obj_info, bypass_governance)
 }
 
 /// Compatibility wrapper for callers that predate fallible metadata lookup.
@@ -486,6 +550,210 @@ mod tests {
         }
     }
 
+    fn replication_opts(hold_ts: bool, retention_ts: bool) -> ObjectOptions {
+        ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        }
+    }
+
+    fn lock_metadata(entries: &[&[(&str, &str)]]) -> std::collections::HashMap<String, String> {
+        entries
+            .iter()
+            .flat_map(|entries| entries.iter())
+            .map(|(key, value)| (key.to_string(), value.to_string()))
+            .collect()
+    }
+
+    fn lock_object_info(user_defined: std::collections::HashMap<String, String>) -> ObjectInfo {
+        ObjectInfo {
+            user_defined: Arc::new(user_defined),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        }
+    }
+
+    /// A replication write passes the WORM gate only when it carries the
+    /// source timestamp of every category that currently locks the version.
+    #[test]
+    fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
+        let retention = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
+        ];
+        let expired = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
+        ];
+        let absent = ObjectLockConfigState::ConfirmedAbsent;
+        let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
+            replication_write_may_pass_worm_gate(state, &lock_object_info(lock_metadata(entries)), opts)
+                .expect("well-formed lock metadata must be judged")
+        };
+
+        assert!(passes(&absent, &[&hold, &retention], &replication_opts(true, true)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(false, true)));
+
+        assert!(passes(&absent, &[&hold], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold], &replication_opts(false, true)));
+        assert!(passes(&absent, &[&retention], &replication_opts(false, true)));
+        assert!(!passes(&absent, &[&retention], &replication_opts(true, false)));
+
+        // Expired retention and a released hold no longer lock anything.
+        assert!(passes(
+            &absent,
+            &[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
+            &replication_opts(false, false)
+        ));
+
+        // Never for a non-replication write, whatever it carries.
+        let local = ObjectOptions {
+            replication_request: false,
+            ..replication_opts(true, true)
+        };
+        assert!(!passes(&absent, &[&hold], &local));
+    }
+
+    /// The bucket default retention locks a version that carries no explicit
+    /// retention keys (`check_object_lock_for_deletion_with_config` judges it
+    /// from the modification time), so the replication bypass must demand the
+    /// retention source timestamp for it too — a tagging-only replication
+    /// write must not overwrite the default-protected version unjudged.
+    #[test]
+    fn replication_write_under_bucket_default_retention_requires_retention_timestamp() {
+        use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER};
+
+        for mode in [ObjectLockRetentionMode::COMPLIANCE, ObjectLockRetentionMode::GOVERNANCE] {
+            let state = ObjectLockConfigState::Configured {
+                config: default_retention_config(mode),
+                updated_at: OffsetDateTime::now_utc(),
+            };
+            let no_keys = lock_object_info(std::collections::HashMap::new());
+            assert!(
+                check_object_lock_for_deletion_with_state(&state, &no_keys, false)
+                    .expect("default retention must be judged")
+                    .is_some(),
+                "{mode}: the gate must report the default retention lock"
+            );
+
+            let tagging_only = ObjectOptions {
+                replication_request: true,
+                replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            assert!(
+                !replication_write_may_pass_worm_gate(&state, &no_keys, &tagging_only).expect("judged"),
+                "{mode}: a tagging-only replication write must not pass the default retention lock"
+            );
+            assert!(
+                replication_write_may_pass_worm_gate(&state, &no_keys, &replication_opts(false, true)).expect("judged"),
+                "{mode}: the retention source timestamp lets LWW judge the default retention"
+            );
+
+            // Default retention plus a legal hold: both categories need a timestamp.
+            let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
+            assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
+
+            // A version whose default retention has already expired (old
+            // mod_time) is not locked by the default any more.
+            let expired_default = ObjectInfo {
+                mod_time: Some(make_datetime(2000, 1, 1)),
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &expired_default, &tagging_only).expect("judged"));
+
+            // A delete marker is never locked, so there is nothing to judge.
+            let delete_marker = ObjectInfo {
+                delete_marker: true,
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
+
+            // Cleared (empty) explicit keys fall back to the bucket default.
+            let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
+        }
+    }
+
+    /// The replication bypass never judges from a non-authoritative bucket
+    /// state or malformed persisted lock metadata; both are errors, not a pass.
+    #[test]
+    fn replication_write_worm_gate_fails_closed_on_unverifiable_lock_state() {
+        use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
+
+        let opts = replication_opts(true, true);
+        let err = replication_write_may_pass_worm_gate(
+            &ObjectLockConfigState::Fabricated,
+            &lock_object_info(std::collections::HashMap::new()),
+            &opts,
+        )
+        .expect_err("fabricated bucket lock metadata must not be judged");
+        assert!(err.to_string().contains("not authoritative"));
+
+        let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
+        let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
+            .expect_err("malformed legal hold must not be judged");
+        assert!(err.to_string().contains("legal-hold"));
+
+        let state = ObjectLockConfigState::Configured {
+            config: default_retention_config(ObjectLockRetentionMode::COMPLIANCE),
+            updated_at: OffsetDateTime::now_utc(),
+        };
+        let no_mod_time = ObjectInfo::default();
+        let err = replication_write_may_pass_worm_gate(&state, &no_mod_time, &opts)
+            .expect_err("default retention without a modification time must not be judged");
+        assert!(err.to_string().contains("modification time"));
+    }
+
+    /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
+    /// lock keys as empty strings (the MinIO on-disk shape, see
+    /// `parse_object_lock_retention`); that is "no lock", not corruption, and
+    /// must not wedge later explicit-version PUTs or deletes
+    /// (rustfs/backlog#1953).
+    #[test]
+    fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let cases: [(&str, &[&str]); 3] = [
+            (
+                "cleared retention",
+                &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
+            ),
+            ("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
+            (
+                "all cleared",
+                &[
+                    AMZ_OBJECT_LOCK_MODE_LOWER,
+                    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+                    AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
+                ],
+            ),
+        ];
+
+        for (case, keys) in cases {
+            let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
+            let obj_info = ObjectInfo {
+                user_defined: Arc::new(user_defined),
+                ..Default::default()
+            };
+
+            let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
+            assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
+        }
+    }
+
     #[test]
     fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
         let mut user_defined = std::collections::HashMap::new();

crates/ecstore/src/bucket/replication/mod.rs

@@ -44,11 +44,14 @@ mod replication_versioning_boundary;
 mod runtime_boundary;
 
 pub use replication_config_boundary::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
+    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
+    is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -13,9 +13,12 @@
 // limitations under the License.
 
 pub use rustfs_replication::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
+    ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
+    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -4194,6 +4194,30 @@ mod tests {
         assert_eq!(ri.checksum, Some(checksum));
     }
 
+    #[test]
+    fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
+        let target = "arn:rustfs:replication:target-a";
+        let object = ObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            user_tags: Arc::new("owner=a3".to_string()),
+            ..Default::default()
+        };
+        let live =
+            replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
+        let persisted = live.to_mrf_entry();
+        let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
+        let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
+
+        assert_eq!(decoded[0].op, MrfOpKind::Metadata);
+        assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
+        let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
+        assert_eq!(replayed.op_type, ReplicationType::Metadata);
+        assert_eq!(replayed.user_tags, "owner=a3");
+        assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
+    }
+
     #[tokio::test]
     async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
         let (tx, mut rx) = mpsc::channel(1);

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -76,7 +76,8 @@ use metrics::counter;
 use rmp_serde;
 use rustfs_s3_types::EventName;
 use rustfs_utils::http::{
-    AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
+    AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
+    has_internal_suffix, insert_str,
 };
 use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
 #[cfg(test)]
@@ -174,6 +175,14 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
     err.raw_response().is_some_and(|r| r.status().as_u16() == status)
 }
 
+fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
+    op_type == ReplicationType::Metadata
+        && object_info
+            .user_defined
+            .get(AMZ_BUCKET_REPLICATION_STATUS)
+            .is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
+}
+
 const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
 
 /// Targets that already produced a version-identity-drift warning this
@@ -3494,6 +3503,7 @@ async fn resolve_replicate_all_action(
         start_time,
         ssec_audit_required,
     } = ctx;
+    let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
     let replication_action;
     match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
         Ok(oi) => {
@@ -3555,7 +3565,13 @@ async fn resolve_replicate_all_action(
                 // Version-ID format mismatch: retry without versionId and compare ETags.
                 match head_object_fallback(tgt_client, object).await {
                     Ok(Some(oi)) => {
-                        replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
+                        let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
+                        if require_existing_target && !etags_match {
+                            rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
+                        replication_action = if etags_match {
                             if ssec_audit_required
                                 && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
                             {
@@ -3568,6 +3584,11 @@ async fn resolve_replicate_all_action(
                         };
                     }
                     Ok(None) => {
+                        if require_existing_target {
+                            rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
                         replication_action = ReplicationAction::All;
                     }
                     Err(e2) => {
@@ -3593,7 +3614,12 @@ async fn resolve_replicate_all_action(
                         return None;
                     }
                 }
-            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
+            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
+                if require_existing_target {
+                    rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                    rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                    return None;
+                }
                 replication_action = ReplicationAction::All;
             } else {
                 rinfo.error = Some(e.to_string());
@@ -3868,6 +3894,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                 actual_size,
                 object_info.etag.clone().unwrap_or_default(),
                 object_info.mod_time,
+                &put_opts.internal,
             ),
         )
         .await
@@ -3921,6 +3948,113 @@ mod tests {
         })
     }
 
+    fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
+        use std::io::{Read, Write};
+
+        let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
+        let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
+        let handle = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
+            let mut request = [0_u8; 8192];
+            let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
+            assert!(bytes_read > 0, "test HTTP request should not be empty");
+            assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
+            write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("test HTTP response should be written");
+        });
+        (endpoint, handle)
+    }
+
+    #[tokio::test]
+    async fn replica_metadata_missing_target_stops_before_full_put() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            // Normal metadata writes replace REPLICA with per-target PENDING
+            // before constructing the worker request.
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            user_defined: Arc::new(HashMap::from([(
+                AMZ_BUCKET_REPLICATION_STATUS.to_string(),
+                ReplicationStatusType::Replica.as_str().to_string(),
+            )])),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(
+            rinfo.error.as_deref(),
+            Some("replica metadata target does not contain this object version")
+        );
+        server.join().expect("test HTTP server should finish");
+    }
+
+    #[tokio::test]
+    async fn source_metadata_missing_target_rebuilds_object() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(matches!(action, Some((ReplicationAction::All, _))));
+        assert!(rinfo.error.is_none());
+        server.join().expect("test HTTP server should finish");
+    }
+
     async fn register_test_target(target: &Arc<TargetClient>) {
         ReplicationTargetStore::register_test_target(target).await;
     }

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
     actual_size: String,
     source_etag: String,
     source_mtime: Option<OffsetDateTime>,
+    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
             // mtime must degrade to epoch so header() suppresses the header
             // instead of asserting the replication time as the object's mtime.
             source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
+            // Carry the per-category LWW timestamps on the complete request as
+            // well: the receiver's CompleteMultipartUpload options builder
+            // parses the same headers, so the multipart transport gets the
+            // same receiver-side LWW as the single-PUT transport
+            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
+            tagging_timestamp: source_internal.tagging_timestamp,
+            retention_timestamp: source_internal.retention_timestamp,
+            legalhold_timestamp: source_internal.legalhold_timestamp,
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -663,20 +672,39 @@ mod tests {
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
         let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
+        let source_internal = AdvancedPutOptions {
+            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
+            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
+            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
+            ..Default::default()
+        };
         let options = replication_complete_multipart_options(
             "1024".to_string(),
             "0123456789abcdef0123456789abcdef-3".to_string(),
             Some(source_mtime),
+            &source_internal,
         );
         assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
         assert_eq!(options.internal.source_mtime, source_mtime);
 
+        // The complete request must carry the same per-category LWW timestamps
+        // as the initiate request; the receiver reads them from the complete
+        // headers (rustfs/backlog#1953).
+        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
+        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
+        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
+
         // Absent source mtime must degrade to epoch (header suppressed), not
         // the AdvancedPutOptions default of now_utc() — that default would
         // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence.
-        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
+        // multipart HEAD convergence. Unset category timestamps stay epoch so
+        // header() keeps suppressing them.
+        let options_no_mtime =
+            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
         assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -32,6 +32,10 @@ pub struct Credentials {
     pub access_key: String,
     #[serde(rename = "secretKey")]
     pub secret_key: String,
+    // The aliases accept madmin's JSON tags (MinIO-written bucket-targets
+    // metadata and mc request bodies) without changing the snake_case
+    // persisted/peer wire format this struct serializes to.
+    #[serde(alias = "sessionToken")]
     pub session_token: Option<String>,
     pub expiration: Option<Timestamp>,
 }
@@ -202,12 +206,14 @@ pub struct BucketTarget {
     #[serde(default)]
     pub region: String,
 
-    #[serde(alias = "bandwidth", default)]
+    // madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
+    // alias kept for inputs written before the madmin tag was verified.
+    #[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
     pub bandwidth_limit: i64,
 
     #[serde(rename = "replicationSync", default)]
     pub replication_sync: bool,
-    #[serde(default)]
+    #[serde(alias = "storageclass", default)]
     pub storage_class: String,
     #[serde(rename = "skipTlsVerify", default)]
     pub skip_tls_verify: bool,
@@ -220,7 +226,7 @@ pub struct BucketTarget {
 
     #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
     pub reset_before_date: Option<OffsetDateTime>,
-    #[serde(default)]
+    #[serde(alias = "resetID", default)]
     pub reset_id: String,
     #[serde(rename = "totalDowntime", with = "duration_seconds", default)]
     pub total_downtime: Duration,
@@ -233,7 +239,7 @@ pub struct BucketTarget {
     #[serde(default)]
     pub latency: LatencyStat,
 
-    #[serde(default)]
+    #[serde(alias = "deploymentID", default)]
     pub deployment_id: String,
 
     #[serde(default)]
@@ -531,6 +537,85 @@ mod tests {
         assert_eq!(value["totalDowntime"], 90);
     }
 
+    #[test]
+    fn bucket_target_persisted_wire_keys_stay_snake_case() {
+        // bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
+        // in the admin set/remove handlers) and the msgpack struct-map form
+        // (`BucketTargets::marshal_msg`) both come straight from this struct's
+        // serde field names. madmin naming is applied only in the admin
+        // response layer (`remote_target_admin_json`); renaming here would
+        // silently break every existing deployment's persisted metadata.
+        let targets = BucketTargets {
+            targets: vec![BucketTarget {
+                credentials: Some(Credentials {
+                    access_key: "ak".to_string(),
+                    secret_key: "sk".to_string(),
+                    session_token: Some("token".to_string()),
+                    expiration: None,
+                }),
+                bandwidth_limit: 5,
+                storage_class: "STANDARD".to_string(),
+                reset_id: "reset-1".to_string(),
+                deployment_id: "deploy-1".to_string(),
+                ..Default::default()
+            }],
+        };
+
+        let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
+        let msgpack: serde_json::Value =
+            rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
+                .expect("msgpack struct map should decode into a JSON value");
+
+        for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
+            assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
+            assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
+            assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
+            assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
+            assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
+        }
+    }
+
+    #[test]
+    fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
+        // A MinIO-written bucket-targets.json carries madmin's JSON tags
+        // (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
+        // `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
+        // On migration these must land in the matching fields instead of
+        // silently defaulting (backlog#1951).
+        let targets: BucketTargets = serde_json::from_value(serde_json::json!({
+            "targets": [{
+                "sourcebucket": "src",
+                "endpoint": "minio.example:9000",
+                "credentials": {
+                    "accessKey": "ak",
+                    "secretKey": "sk",
+                    "sessionToken": "minio-session-token"
+                },
+                "targetbucket": "dst",
+                "type": "replication",
+                "replicationSync": true,
+                "bandwidthlimit": 107374182400i64,
+                "storageclass": "STANDARD",
+                "resetID": "reset-789",
+                "deploymentID": "deploy-123"
+            }]
+        }))
+        .expect("MinIO-written bucket-targets.json must deserialize");
+
+        let target = &targets.targets[0];
+        assert_eq!(target.bandwidth_limit, 107374182400);
+        assert_eq!(target.storage_class, "STANDARD");
+        assert_eq!(target.reset_id, "reset-789");
+        assert_eq!(target.deployment_id, "deploy-123");
+        assert_eq!(
+            target
+                .credentials
+                .as_ref()
+                .and_then(|credentials| credentials.session_token.as_deref()),
+            Some("minio-session-token")
+        );
+    }
+
     #[test]
     fn test_bucket_target_debug_redacts_credentials() {
         let target = BucketTarget {

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
 const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
+const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
     get_env_bool(
@@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
 }
 
-fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
+fn update_ns_scanner_capability_mac(
+    mac: &mut HmacSha256,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) {
     mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
     mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
     mac.update(challenge.as_bytes());
     mac.update(server_epoch.as_bytes());
+    if supports_tier_registry_generation {
+        // The optional response capability is part of the authenticated
+        // scope. A proxy cannot turn an old/unsupported peer into a worker
+        // that receives generation-fenced scanner work.
+        mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
+    }
 }
 
-fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
+fn generate_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     Ok(mac.finalize().into_bytes().to_vec())
 }
 
-fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
+fn verify_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
     if challenge.is_nil() || server_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
     }
     let mut mac =
         <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
     mac.verify_slice(proof)
         .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
 }
 
 pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
-    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
+    sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
 }
 
 pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
-    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
+    verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
+}
+
+pub fn sign_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
+    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
+}
+
+pub fn verify_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
+    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
 }
 
 #[derive(Clone, Copy)]
@@ -1709,13 +1749,28 @@ mod tests {
         let secret = "test-scanner-capability-secret";
         let challenge = Uuid::new_v4();
         let server_epoch = Uuid::new_v4();
-        let proof =
-            generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("capability proof should be generated");
 
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
+    }
+
+    #[test]
+    fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
+        let secret = "test-scanner-capability-secret";
+        let challenge = Uuid::new_v4();
+        let server_epoch = Uuid::new_v4();
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
+            .expect("generation capability proof should be generated");
+
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("legacy capability proof should be generated");
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
     }
 
     /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -13,17 +13,18 @@
 // limitations under the License.
 
 use crate::cluster::rpc::{
-    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
+    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
+    verify_put_file_capability,
 };
 use crate::disk::error::{Error, Result};
 use crate::disk::{FileReader, FileWriter};
 use crate::storage_api_contracts::internode::{
     NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
     NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
-    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
-    PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
-    PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
-    WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
+    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
+    NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
+    PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
+    PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
 };
 use async_trait::async_trait;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -137,6 +138,12 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
     status == 404
 }
 
+fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
+    [400, 404, 405, 426]
+        .into_iter()
+        .any(|status| error.is_internode_http_status(status))
+}
+
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 #[allow(
     dead_code,
@@ -220,6 +227,7 @@ pub struct NsScannerStreamRequest {
 #[derive(Debug, Clone)]
 pub struct NsScannerCapabilityRequest {
     pub endpoint: String,
+    pub supports_tier_registry_generation: bool,
 }
 
 /// Data-plane stream opener used by `RemoteDisk`.
@@ -252,6 +260,15 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
     async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
         Err(Error::MethodNotAllowed)
     }
+    async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
+        let server_epoch = self.probe_ns_scanner(request).await?;
+        Ok(NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch,
+            proof: Vec::new(),
+            supports_tier_registry_generation: None,
+        })
+    }
     // Interface facet nobody calls yet: every transport implements both, but no
     // caller negotiates on them. Kept for the internode transport split
     // (backlog#1350); deleting them would delete the seam and six impls.
@@ -335,27 +352,44 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
     }
 
     async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
-        let challenge = Uuid::new_v4();
-        let url = build_ns_scanner_capability_url(&request, challenge);
-        let mut headers = msgpack_headers();
-        build_auth_headers(&url, &Method::GET, &mut headers)?;
-        let reader = HttpReader::new(url, Method::GET, headers, None).await?;
-        let mut body = Vec::new();
-        reader
-            .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
-            .read_to_end(&mut body)
-            .await?;
-        if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
-            return Err(Error::other("invalid remote namespace scanner capability response size"));
+        Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
+    }
+
+    async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
+        if request.supports_tier_registry_generation {
+            return match self.probe_ns_scanner_capability_once(&request).await {
+                Ok(response) => Ok(response),
+                Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
+                    // A v3 peer may reject the additive query marker, ignore
+                    // it, or return its legacy proof. Retry once without the
+                    // marker and only downgrade after that legacy response is
+                    // authenticated; an unverified epoch is never trusted.
+                    let legacy_request = NsScannerCapabilityRequest {
+                        endpoint: request.endpoint.clone(),
+                        supports_tier_registry_generation: false,
+                    };
+                    match self.probe_ns_scanner_capability_once(&legacy_request).await {
+                        Ok(mut response) => {
+                            response.supports_tier_registry_generation = None;
+                            Ok(response)
+                        }
+                        Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
+                            // Some old deployments expose only the legacy
+                            // protocol response (or advertise 426). Treat
+                            // the pair as an explicit unsupported result so
+                            // the scanner can use its coordinator fallback.
+                            Err(Error::MethodNotAllowed)
+                        }
+                        Err(_) => Err(marked_error),
+                    }
+                }
+                // A server failure, network failure, or authentication error
+                // is not evidence of an old parser. Do not issue an
+                // unauthenticated legacy probe or silently downgrade.
+                Err(marked_error) => Err(marked_error),
+            };
         }
-        let response: NsScannerCapabilityResponse =
-            rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
-        if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
-            return Err(Error::other("incompatible remote namespace scanner capability response"));
-        }
-        verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
-            .map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
-        Ok(response.server_epoch)
+        self.probe_ns_scanner_capability_once(&request).await
     }
 
     fn name(&self) -> &'static str {
@@ -368,6 +402,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
 }
 
 impl TcpHttpInternodeDataTransport {
+    async fn probe_ns_scanner_capability_once(
+        &self,
+        request: &NsScannerCapabilityRequest,
+    ) -> Result<NsScannerCapabilityResponse> {
+        let challenge = Uuid::new_v4();
+        let url = build_ns_scanner_capability_url(request, challenge);
+        let mut headers = msgpack_headers();
+        build_auth_headers(&url, &Method::GET, &mut headers)?;
+        let reader = HttpReader::new(url, Method::GET, headers, None).await?;
+        let mut body = Vec::new();
+        reader
+            .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
+            .read_to_end(&mut body)
+            .await?;
+        if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
+            return Err(Error::other("invalid remote namespace scanner capability response size"));
+        }
+        let mut response: NsScannerCapabilityResponse =
+            rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
+        if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
+            return Err(Error::other("incompatible remote namespace scanner capability response"));
+        }
+        if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
+            challenge,
+            response.server_epoch,
+            &response.proof,
+            request.supports_tier_registry_generation,
+        ) {
+            // A permissive older peer can ignore the additive marker and
+            // return a valid legacy-scope proof with HTTP 200. Accept that
+            // response only after independently authenticating the legacy
+            // scope; all other verification failures remain fail-closed.
+            if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
+                response.supports_tier_registry_generation = None;
+                return Ok(response);
+            }
+            return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
+        }
+        // The proof authenticates the requested capability scope, not the
+        // optional response field. Derive the client-facing bit from that
+        // verified scope so an intermediary cannot strip or rewrite the field
+        // and force a silent downgrade after a successful generation-bound
+        // handshake.
+        normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
+        Ok(response)
+    }
+
     async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
         resolve_put_file_auth_capability(endpoint, || async {
             tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
@@ -649,6 +730,14 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
     )
 }
 
+fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
+    response.supports_tier_registry_generation = requested_generation_support.then_some(true);
+}
+
+fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
+    verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
+}
+
 fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
     let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
     format!(
@@ -675,13 +764,18 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
 
 fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
     format!(
-        "{}{}?{}={}&{}={}",
+        "{}{}?{}={}&{}={}{}",
         request.endpoint,
         NS_SCANNER_PATH,
         NS_SCANNER_PROTOCOL_VERSION_QUERY,
         NS_SCANNER_PROTOCOL_VERSION,
         NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
-        challenge
+        challenge,
+        if request.supports_tier_registry_generation {
+            format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
+        } else {
+            String::new()
+        }
     )
 }
 
@@ -794,6 +888,7 @@ mod tests {
         let probe_err = transport
             .probe_ns_scanner(NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: false,
             })
             .await
             .expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1387,6 +1482,7 @@ mod tests {
         let url = build_ns_scanner_capability_url(
             &NsScannerCapabilityRequest {
                 endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: false,
             },
             challenge,
         );
@@ -1399,6 +1495,85 @@ mod tests {
         );
     }
 
+    #[test]
+    fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
+        let challenge = Uuid::new_v4();
+        let url = build_ns_scanner_capability_url(
+            &NsScannerCapabilityRequest {
+                endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: true,
+            },
+            challenge,
+        );
+
+        assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
+    }
+
+    #[test]
+    fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
+        for status in [400, 404, 405, 426] {
+            let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+                rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
+            ));
+            assert!(
+                ns_scanner_capability_error_allows_legacy(&error),
+                "status {status} should permit legacy retry"
+            );
+        }
+
+        let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
+        ));
+        let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
+        ));
+        let authentication_error = Error::other("remote namespace scanner capability authentication failed");
+        assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
+        assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
+        assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
+    }
+
+    #[test]
+    fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
+        let mut response = NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch: Uuid::new_v4(),
+            proof: Vec::new(),
+            supports_tier_registry_generation: None,
+        };
+
+        normalize_ns_scanner_capability_response(&mut response, true);
+        assert_eq!(response.supports_tier_registry_generation, Some(true));
+
+        response.supports_tier_registry_generation = Some(false);
+        normalize_ns_scanner_capability_response(&mut response, false);
+        assert_eq!(response.supports_tier_registry_generation, None);
+    }
+
+    #[test]
+    fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let challenge = Uuid::new_v4();
+        let response = NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch: Uuid::new_v4(),
+            proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
+                .expect("placeholder proof should be generated"),
+            supports_tier_registry_generation: None,
+        };
+        // A proof bound to a different challenge cannot authorize the legacy
+        // fallback, even though the response has the expected shape.
+        assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
+
+        let server_epoch = response.server_epoch;
+        let valid_response = NsScannerCapabilityResponse {
+            proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
+                .expect("legacy proof should be generated"),
+            ..response
+        };
+        assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
+    }
+
     #[test]
     fn transport_config_defaults_to_tcp_http() {
         let transport = build_internode_data_transport(None).unwrap();

crates/ecstore/src/cluster/rpc/mod.rs

@@ -35,8 +35,9 @@ pub use http_auth::{
     TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
     gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
     set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
-    tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
+    sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+    verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
     verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
     verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
     verify_tonic_rpc_signature_with_bootstrap,

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -85,6 +85,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
 const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
 const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
 const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
+const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
 
 /// Error for a peer that reported `success = false` without an `error_info` payload.
 ///
@@ -1089,7 +1090,9 @@ impl PeerRestClient {
                     .await?
                     .max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
                 let response = match client
-                    .background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
+                    .background_heal_status(Request::new(BackgroundHealStatusRequest {
+                        protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
+                    }))
                     .await
                 {
                     Ok(response) => response.into_inner(),
@@ -1328,27 +1331,38 @@ impl PeerRestClient {
     }
 
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
-        self.finalize_result(
-            async {
-                let mut client = self.get_client().await?;
-                let mut request = Request::new(LoadBucketMetadataRequest {
-                    bucket: bucket.to_string(),
-                    scanner_maintenance_change,
-                });
-                set_tonic_mutation_body_digest(&mut request)?;
-
-                let response = client.load_bucket_metadata(request).await?.into_inner();
-                if !response.success {
-                    if let Some(msg) = response.error_info {
-                        return Err(Error::other(msg));
-                    }
-                    return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
-                }
-                Ok(())
+        let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
+            let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
+            if let Err(err) = &result
+                && Self::is_network_like_error(err)
+            {
+                self.prepare_retry().await;
+                return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
             }
-            .await,
-        )
+            result
+        })
         .await
+        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
+        self.finalize_result(result).await
+    }
+
+    async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
+        let mut client = self.get_client().await?;
+        let mut request = Request::new(LoadBucketMetadataRequest {
+            bucket: bucket.to_string(),
+            scanner_maintenance_change,
+        });
+        set_tonic_mutation_body_digest(&mut request)?;
+        request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
+
+        let response = client.load_bucket_metadata(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::other(msg));
+            }
+            return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
+        }
+        Ok(())
     }
 
     pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -781,14 +781,16 @@ impl RemoteDisk {
         if self.health.is_faulty() {
             return Err(DiskError::FaultyDisk);
         }
-        let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
+        let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
             endpoint: self.endpoint.grid_host(),
+            supports_tier_registry_generation: true,
         });
         let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
             .await
             .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
         match result {
-            Ok(server_epoch) => Ok(Some(server_epoch)),
+            Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
+            Ok(_) => Ok(None),
             // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
             Err(DiskError::MethodNotAllowed) => Ok(None),
             Err(err)
@@ -4040,10 +4042,21 @@ mod tests {
         NsScannerProbe(NsScannerCapabilityRequest),
     }
 
-    #[derive(Debug, Clone, Default)]
+    #[derive(Debug, Clone)]
     struct RecordingInternodeDataTransport {
         calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
         ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
+        ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
+    }
+
+    impl Default for RecordingInternodeDataTransport {
+        fn default() -> Self {
+            Self {
+                calls: Arc::default(),
+                ns_scanner_probe_status: Arc::default(),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+            }
+        }
     }
 
     #[derive(Clone, Debug)]
@@ -4263,6 +4276,15 @@ mod tests {
             Self {
                 calls: Arc::default(),
                 ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+            }
+        }
+
+        fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
+            Self {
+                calls: Arc::default(),
+                ns_scanner_probe_status: Arc::default(),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
             }
         }
 
@@ -4945,6 +4967,23 @@ mod tests {
             Ok(Uuid::from_u128(1))
         }
 
+        async fn probe_ns_scanner_capability(
+            &self,
+            request: NsScannerCapabilityRequest,
+        ) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
+            let server_epoch = self.probe_ns_scanner(request).await?;
+            let supports_tier_registry_generation = *self
+                .ns_scanner_generation_support
+                .lock()
+                .expect("namespace scanner generation support lock poisoned");
+            Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
+                version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
+                server_epoch,
+                proof: Vec::new(),
+                supports_tier_registry_generation,
+            })
+        }
+
         fn name(&self) -> &'static str {
             "recording"
         }
@@ -6757,6 +6796,22 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
+        for support in [None, Some(false)] {
+            let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
+            let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
+
+            assert_eq!(
+                remote_disk
+                    .ns_scanner_server_epoch()
+                    .await
+                    .expect("missing generation support should be classified as unsupported"),
+                None
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
         let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;

crates/ecstore/src/config/com.rs

@@ -406,6 +406,25 @@ where
     Ok(data)
 }
 
+pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
+where
+    S: EcstoreObjectIO,
+{
+    let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
+    Ok(data)
+}
+
+pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
+    api: Arc<S>,
+    file: &str,
+    max_bytes: usize,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
+}
+
 /// Read an existing config object without treating an empty payload as absent.
 /// Callers that validate their own payload format need to distinguish corruption
 /// from `ConfigNotFound`.

crates/ecstore/src/data_movement/mod.rs

@@ -813,7 +813,7 @@ pub(crate) fn is_equivalent_data_movement_metadata(
             .all(|(key, value)| source.user_defined.get(key) == Some(value))
 }
 
-fn is_equivalent_data_movement_object_identity(
+pub(crate) fn is_equivalent_data_movement_object_identity(
     source: &ObjectInfo,
     target: &ObjectInfo,
     compare_mod_time: bool,

crates/ecstore/src/data_usage/mod.rs

@@ -73,6 +73,16 @@ struct CachedBucketUsage {
     // mutation. A strictly later generation is required before the mutation
     // evidence can be discarded.
     pending_scanner_position: Option<(u64, u64)>,
+    // Deletes are visible to admin immediately, but quota admission keeps
+    // them pending until a complete scanner generation reconciles the set.
+    // This marker intentionally remains process-local: the delete request
+    // updates this overlay before the scanner writes a durable snapshot. If
+    // the process restarts first, loading the persisted complete snapshot
+    // restores the pre-reconciliation (larger) baseline, which is
+    // conservative for quota admission. A persisted post-delete snapshot is
+    // necessarily a complete scanner reconciliation and therefore creates a
+    // fresh cache entry with no pending hold.
+    pending_negative_delta: u64,
 }
 
 type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -378,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
             "nonconverged data usage observations cannot replace the quota-authoritative snapshot",
         ));
     }
+    let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
+        return Err(Error::other("data usage publication is blocked by data movement"));
+    };
     // Prevent older data from overwriting newer persisted stats
-    if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
+    let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
+    if let Ok((existing, source)) = existing_snapshot
         && source.is_authoritative()
         && let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
     {
@@ -390,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
         return Ok(());
     }
 
-    save_data_usage_in_backend(data_usage_info, store).await
+    save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
 }
 
-async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
+async fn save_data_usage_in_backend(
+    data_usage_info: DataUsageInfo,
+    store: Arc<ECStore>,
+    expected_publication_epoch: u64,
+) -> Result<(), Error> {
     let data =
         serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
 
     // Save to backend using the same mechanism as original code
+    let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
+        return Err(Error::other("data usage publication is blocked by data movement"));
+    };
+    if publication_epoch != expected_publication_epoch {
+        return Err(Error::other("data usage publication epoch changed before save"));
+    }
     crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
         .await
         .map_err(Error::other)?;
+    drop(publication_guard);
 
-    cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
+    cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
+        .await;
 
     // Invalidate the cached snapshot so readers observe the new save on their
     // next request instead of waiting out the remaining TTL. The next cached
@@ -439,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
     }
 }
 
-async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
-where
+async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
+    store: &S,
+    authoritative: &DataUsageInfo,
+    publication_store: Option<&ECStore>,
+) where
     S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
 {
-    let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
+    let observed_read_epoch = match publication_store {
+        Some(publication_store) => {
+            let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
+                return;
+            };
+            Some(epoch)
+        }
+        None => None,
+    };
+    let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
+    let (observed, revision) = match observed_snapshot {
         Ok(Some(snapshot)) => snapshot,
         Ok(None) => return,
         Err(err) => {
@@ -459,6 +498,19 @@ where
         return;
     }
 
+    let publication_guard = match publication_store {
+        Some(publication_store) => {
+            let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
+            else {
+                return;
+            };
+            if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
+                return;
+            }
+            Some(guard)
+        }
+        None => None,
+    };
     match store.delete_observed_data_usage_snapshot(&revision).await {
         Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
         Err(err) => {
@@ -469,6 +521,15 @@ where
             );
         }
     }
+    drop(publication_guard);
+}
+
+#[cfg(test)]
+async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
+where
+    S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
+{
+    cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
 }
 
 fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -509,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
 
 pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
     prepare_bucket_usage_for_namespace_change(bucket, None).await?;
-    remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
+    remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
 }
 
 pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -532,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
     Ok(())
 }
 
+#[cfg(test)]
 pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
     store: &S,
     bucket: &str,
@@ -541,6 +603,24 @@ where
     S: EcstoreObjectIO + ?Sized,
 {
     let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
+    invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
+    result
+}
+
+pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
+    store: &ECStore,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+) -> Result<(), Error> {
+    let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
+    invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
+    result
+}
+
+async fn invalidate_bucket_usage_snapshot_caches(
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    bucket: &str,
+) -> Result<(), Error> {
     let mut snapshot_cache = data_usage_snapshot_cache().write().await;
     ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
     clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -548,7 +628,7 @@ where
     let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
     ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
     clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
-    result
+    Ok(())
 }
 
 async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -607,48 +687,83 @@ fn ensure_bucket_namespace_guard(
     Ok(())
 }
 
+#[cfg(test)]
 async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
     store: &S,
     bucket: &str,
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error>
+where
+    S: EcstoreObjectIO + ?Sized,
+{
+    remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
+}
+
+async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
+    store: &S,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    publication_store: Option<&ECStore>,
+) -> Result<(), Error>
 where
     S: EcstoreObjectIO + ?Sized,
 {
     ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
+    let primary_seed_epoch = match publication_store {
+        Some(publication_store) => Some(
+            publication_store
+                .scanner_data_usage_publication_epoch()
+                .await
+                .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
+        ),
+        None => None,
+    };
     let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
-    remove_bucket_usage_from_object_with_retries(
+    remove_bucket_usage_from_object_with_retries_and_publication(
         store,
         DATA_USAGE_OBJ_NAME_PATH.as_str(),
         bucket,
         DATA_USAGE_REMOVE_CAS_RETRIES,
-        Some(&primary_seed),
+        primary_seed.as_ref(),
         guard,
+        publication_store.map(|store| (store, primary_seed_epoch)),
     )
     .await?;
 
     ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
+    let backup_seed_epoch = match publication_store {
+        Some(publication_store) => Some(
+            publication_store
+                .scanner_data_usage_publication_epoch()
+                .await
+                .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
+        ),
+        None => None,
+    };
     let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
         .await?
-        .map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
-    remove_bucket_usage_from_object_with_retries(
+        .map(|(data_usage_info, _)| data_usage_info)
+        .or_else(|| primary_seed.clone());
+    remove_bucket_usage_from_object_with_retries_and_publication(
         store,
         DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
         bucket,
         DATA_USAGE_REMOVE_CAS_RETRIES,
-        Some(&backup_seed),
+        backup_seed.as_ref(),
         guard,
+        publication_store.map(|store| (store, backup_seed_epoch)),
     )
     .await?;
 
     ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
-    if let Err(err) = remove_bucket_usage_from_object_with_retries(
+    if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
         store,
         DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
         bucket,
         DATA_USAGE_REMOVE_CAS_RETRIES,
         None,
         guard,
+        publication_store.map(|store| (store, None)),
     )
     .await
     {
@@ -662,12 +777,21 @@ where
         LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
         LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
     ] {
-        remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
+        remove_bucket_usage_from_object_with_retries_and_publication(
+            store,
+            object,
+            bucket,
+            DATA_USAGE_REMOVE_CAS_RETRIES,
+            None,
+            guard,
+            publication_store.map(|store| (store, None)),
+        )
+        .await?;
     }
     Ok(())
 }
 
-async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
+async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
 where
     S: EcstoreObjectIO + ?Sized,
 {
@@ -680,12 +804,13 @@ where
             if !authoritative {
                 data_usage_info.usage_snapshot_complete = false;
             }
-            return Ok(data_usage_info);
+            return Ok(Some(data_usage_info));
         }
     }
-    Ok(DataUsageInfo::default())
+    Ok(None)
 }
 
+#[cfg(test)]
 async fn remove_bucket_usage_from_object_with_retries<S>(
     store: &S,
     object: &str,
@@ -694,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
     missing_seed: Option<&DataUsageInfo>,
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error>
+where
+    S: EcstoreObjectIO + ?Sized,
+{
+    remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
+        .await
+}
+
+async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
+    store: &S,
+    object: &str,
+    bucket: &str,
+    cas_retries: usize,
+    missing_seed: Option<&DataUsageInfo>,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    publication: Option<(&ECStore, Option<u64>)>,
+) -> Result<(), Error>
 where
     S: EcstoreObjectIO + ?Sized,
 {
     for attempt in 0..=cas_retries {
         ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
-        let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
+        let read_epoch = match publication {
+            Some((publication_store, expected_publication_epoch)) => {
+                let epoch = publication_store
+                    .scanner_data_usage_publication_epoch()
+                    .await
+                    .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
+                if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
+                    return Err(Error::other("data usage publication epoch changed before snapshot read"));
+                }
+                Some(epoch)
+            }
+            None => None,
+        };
+        let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
+        let (mut data_usage_info, revision) = match loaded_snapshot {
             Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
             None => match missing_seed {
                 Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -723,6 +878,22 @@ where
             },
         };
         ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
+        let publication_guard = match publication {
+            Some((publication_store, expected_publication_epoch)) => {
+                let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
+                else {
+                    return Err(Error::other("data usage publication is blocked by data movement"));
+                };
+                if expected_publication_epoch
+                    .or(read_epoch)
+                    .is_some_and(|expected| expected != publication_epoch)
+                {
+                    return Err(Error::other("data usage publication epoch changed before snapshot commit"));
+                }
+                Some(guard)
+            }
+            None => None,
+        };
         let save_result = store
             .put_object(
                 RUSTFS_META_BUCKET,
@@ -735,6 +906,7 @@ where
                 },
             )
             .await;
+        drop(publication_guard);
         match save_result {
             Ok(_) => return Ok(()),
             Err(err) => {
@@ -948,7 +1120,12 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
     };
 
     match parse_usage_snapshot(&data) {
-        Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
+        Ok(info)
+            if info.usage_snapshot_converged == Some(false)
+                && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
+        {
+            Some(info)
+        }
         Ok(_) => {
             error!(
                 event = "data_usage_snapshot_load_failed",
@@ -993,7 +1170,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
 }
 
 fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
-    if !data_usage_info.is_complete_bucket_usage_snapshot() {
+    if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
         data_usage_info.usage_snapshot_complete = false;
         data_usage_info.buckets_usage.clear();
         data_usage_info.bucket_sizes.clear();
@@ -1643,6 +1820,7 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
         dirty: false,
         stale_snapshot_pending: false,
         pending_scanner_position: None,
+        pending_negative_delta: 0,
     }
 }
 
@@ -1656,6 +1834,7 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
         dirty: false,
         stale_snapshot_pending: false,
         pending_scanner_position: None,
+        pending_negative_delta: 0,
     }
 }
 
@@ -1808,6 +1987,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
         .or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
 
     entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
+    entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
     if removed_current_object {
         entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
         entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
@@ -1863,7 +2043,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
     cache
         .get(bucket)
         .filter(|cached| cached.authoritative)
-        .map(|cached| cached.usage.size)
+        .map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
 }
 
 async fn update_usage_cache_if_needed() {
@@ -2268,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::layout::endpoints::EndpointServerPools;
+    use crate::runtime::instance::InstanceContext;
     use crate::storage_api_contracts::object::ObjectIO as _;
     use rustfs_data_usage::BucketUsageInfo;
     use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2290,6 +2472,7 @@ mod tests {
         error_after_commit_put: Option<usize>,
         advance_time_on_put: Option<Duration>,
         advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
+        advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
         advance_time_before_put: Option<(usize, Duration)>,
         advance_time_after_put: Option<(usize, Duration)>,
         put_count: usize,
@@ -2367,10 +2550,21 @@ mod tests {
                 }
                 _ => None,
             };
+            let advance_publication_epoch = match state.advance_publication_epoch_after_get {
+                Some((expected_slot, ref ctx)) if expected_slot == slot => {
+                    let ctx = Arc::clone(ctx);
+                    state.advance_publication_epoch_after_get = None;
+                    Some(ctx)
+                }
+                _ => None,
+            };
             drop(state);
             if let Some(duration) = advance {
                 tokio::time::advance(duration).await;
             }
+            if let Some(ctx) = advance_publication_epoch {
+                ctx.advance_data_movement_operation_epoch();
+            }
             Ok(crate::object_api::GetObjectReader {
                 stream: Box::new(Cursor::new(data)),
                 object_info: ObjectInfo {
@@ -2571,6 +2765,23 @@ mod tests {
             .to_string()
     }
 
+    fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
+        let endpoint_pools = EndpointServerPools::default();
+        Arc::new(ECStore {
+            id: uuid::Uuid::new_v4(),
+            disk_map: HashMap::new(),
+            pools: Vec::new(),
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
+            pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
+            rebalance_meta: RwLock::new(None),
+            decommission_cancelers: RwLock::new(Vec::new()),
+            start_gate: TokioMutex::new(()),
+            pool_meta_save_gate: TokioMutex::new(()),
+            ctx,
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
     #[test]
     fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
         // `to_object_err` rewrites the raw storage variants before they reach
@@ -2943,6 +3154,45 @@ mod tests {
         assert_eq!(selected.usage_snapshot_converged, Some(true));
     }
 
+    #[test]
+    fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
+        let authoritative = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        };
+        let mut partial = authoritative.clone();
+        partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        partial.scanner_cycle = Some(11);
+        partial.usage_snapshot_complete = false;
+        partial.usage_snapshot_partial = true;
+        partial.usage_snapshot_converged = Some(false);
+        partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
+        partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
+            pool_index: 0,
+            set_index: 0,
+            scanner_cycle: Some(10),
+            scanner_epoch: Some(4),
+            scan_plan_digest: Some([1; 32]),
+            complete: false,
+            tombstone: false,
+        }];
+        partial.buckets_usage.insert(
+            "bucket".to_string(),
+            BucketUsageInfo {
+                size: 100,
+                ..Default::default()
+            },
+        );
+        partial.buckets_count = 1;
+
+        let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
+        assert!(selected.usage_snapshot_partial);
+        assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
+    }
+
     #[tokio::test]
     async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
         let store = UsageCasStore::default();
@@ -4276,7 +4526,15 @@ mod tests {
                 .expect("namespace lock acquisition should not fail")
                 .expect("namespace lock should be acquired"),
         );
-        let store = Arc::new(UsageCasStore::default());
+        let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
+        let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
+        let store = Arc::new(UsageCasStore {
+            state: Mutex::new(UsageCasState {
+                object: Some((encoded.clone(), 1)),
+                backup_object: Some((encoded, 1)),
+                ..Default::default()
+            }),
+        });
         let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4375,21 +4633,17 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
+    async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
         let store = Arc::new(UsageCasStore::default());
 
         remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
             .await
-            .expect("bucket removal should create both usage fences");
+            .expect("bucket removal should remain a no-op without a usage baseline");
 
         let state = store.state.lock().await;
-        assert_eq!(state.put_count, 2);
-        for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
-            let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
-            assert_eq!(*revision, 1);
-            assert!(saved.last_update.is_some());
-            assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
-        }
+        assert_eq!(state.put_count, 0);
+        assert!(state.object.is_none());
+        assert!(state.backup_object.is_none());
     }
 
     #[tokio::test]
@@ -4541,6 +4795,39 @@ mod tests {
         assert_eq!(backup_err, Error::PreconditionFailed);
     }
 
+    #[tokio::test]
+    async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
+        let ctx = Arc::new(InstanceContext::new());
+        let publication_store = build_publication_store(ctx.clone());
+        let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
+        let store = Arc::new(UsageCasStore {
+            state: Mutex::new(UsageCasState {
+                object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
+                advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
+                ..Default::default()
+            }),
+        });
+        let expected_epoch = publication_store
+            .scanner_data_usage_publication_epoch()
+            .await
+            .expect("idle publication store should admit the initial read");
+
+        let err = remove_bucket_usage_from_object_with_retries_and_publication(
+            store.as_ref(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            "bucket-a",
+            0,
+            None,
+            None,
+            Some((publication_store.as_ref(), Some(expected_epoch))),
+        )
+        .await
+        .expect_err("a movement epoch flip during the read must fence the commit");
+
+        assert!(err.to_string().contains("epoch changed"));
+        assert_eq!(store.state.lock().await.put_count, 0);
+    }
+
     #[tokio::test]
     async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
         let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
@@ -4665,6 +4952,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn partial_usage_is_observational_not_authoritative_for_quota() {
+        clear_usage_memory_cache_for_test().await;
+
+        let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
+        partial.usage_snapshot_complete = false;
+        partial.usage_snapshot_partial = true;
+        replace_bucket_usage_memory_from_info(&partial).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
+        clear_usage_memory_cache_for_test().await;
+
+        let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        record_bucket_object_write_memory("bucket-a", None, 25).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn negative_delta_waits_for_set_reconciliation() {
+        clear_usage_memory_cache_for_test().await;
+
+        let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        record_bucket_object_delete_memory("bucket-a", 25, true).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
+
+        // Simulate a process restart: the request-path overlay is gone, but
+        // the persisted authoritative snapshot is still the pre-reconciliation
+        // baseline. Quota must remain conservative until a complete scanner
+        // result proves the delete.
+        clear_usage_memory_cache_for_test().await;
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
+
+        let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
+        replace_bucket_usage_memory_from_info(&reconciled).await;
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
+    }
+
     #[tokio::test]
     #[serial]
     async fn memory_overlay_counts_versioned_overwrite_as_new_version() {

crates/ecstore/src/disk/local.rs

@@ -8000,10 +8000,15 @@ impl DiskAPI for LocalDisk {
             use std::io::Write as _;
 
             let file_path = self.io_get_object_path(volume, path)?;
-            let lock_path = file_path.with_extension("rustfs-cas.lock");
             let path = path.to_string();
             let sync_metadata = effective_durability(volume).syncs_commit_metadata();
             return Ok(tokio::task::spawn_blocking(move || {
+                // A persistent directory lock bounds metadata growth. Removing
+                // per-target lock files can split flock ownership across inodes.
+                let lock_path = file_path
+                    .parent()
+                    .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
+                    .join(".rustfs-cas.lock");
                 let lock = std::fs::OpenOptions::new()
                     .create(true)
                     .truncate(false)
@@ -8068,7 +8073,25 @@ impl DiskAPI for LocalDisk {
             .map_err(DiskError::from)??);
         }
 
-        #[cfg(not(unix))]
+        #[cfg(windows)]
+        {
+            let file_path = self.io_get_object_path(volume, path)?;
+            let sync_metadata = effective_durability(volume).syncs_commit_metadata();
+            let publication_root = self.publication_root.clone();
+            return Ok(tokio::task::spawn_blocking(move || {
+                os::compare_and_update_control_file(
+                    &file_path,
+                    expected.as_deref(),
+                    replacement.as_deref(),
+                    sync_metadata,
+                    &publication_root,
+                )
+            })
+            .await
+            .map_err(DiskError::from)??);
+        }
+
+        #[cfg(not(any(unix, windows)))]
         {
             let _ = (volume, path, expected, replacement);
             Err(DiskError::MethodNotAllowed)
@@ -8940,6 +8963,7 @@ impl DiskAPI for LocalDisk {
         )
         .await?;
 
+        out.close().await?;
         Ok(())
     }
 
@@ -21823,9 +21847,9 @@ mod test {
         assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
     }
 
-    #[cfg(unix)]
+    #[cfg(any(unix, windows))]
     #[tokio::test]
-    async fn conditional_file_update_never_deletes_a_new_owner() {
+    async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
         use tempfile::tempdir;
 
         let dir = tempdir().expect("temp dir should be created");
@@ -21856,8 +21880,18 @@ mod test {
             disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
                 .await
                 .expect("new owner marker should remain"),
-            owner_b
+            owner_b.clone()
         );
+        assert_eq!(
+            disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
+                .await
+                .expect("current owner should remove marker"),
+            ConditionalFileUpdate::Updated
+        );
+        assert!(matches!(
+            disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
+            Err(DiskError::FileNotFound)
+        ));
     }
 
     #[cfg(unix)]
@@ -21872,7 +21906,10 @@ mod test {
         let marker_path = disk
             .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
             .expect("marker path should resolve");
-        let lock_path = marker_path.with_extension("rustfs-cas.lock");
+        let lock_path = marker_path
+            .parent()
+            .expect("marker path should have a parent")
+            .join(".rustfs-cas.lock");
         let lock = std::fs::OpenOptions::new()
             .create(true)
             .truncate(false)
@@ -21893,6 +21930,40 @@ mod test {
         assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
     }
 
+    #[cfg(windows)]
+    #[tokio::test]
+    async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
+        let marker_path = disk
+            .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
+            .expect("marker path should resolve");
+        let lock_path = marker_path
+            .parent()
+            .expect("marker path should have a parent")
+            .join(".rustfs-cas.lock");
+        let lock = std::fs::OpenOptions::new()
+            .create(true)
+            .truncate(false)
+            .read(true)
+            .write(true)
+            .open(lock_path)
+            .expect("marker lock should open");
+        lock.try_lock().expect("marker lock should be held");
+
+        let err = tokio::time::timeout(
+            Duration::from_secs(1),
+            disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
+        )
+        .await
+        .expect("contended conditional update must not block")
+        .expect_err("contended conditional update must retry");
+
+        assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
+    }
+
     #[cfg(target_os = "linux")]
     #[tokio::test]
     async fn replacement_io_paths_stay_under_the_mount_lease() {

crates/ecstore/src/disk/os.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(windows)]
+use crate::disk::ConditionalFileUpdate;
 use crate::disk::error::DiskError;
 use crate::disk::error::Result;
 use crate::disk::error_conv::to_file_error;
@@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
     Ok(Some(data))
 }
 
+#[cfg(windows)]
+pub(crate) fn compare_and_update_control_file(
+    file_path: &Path,
+    expected: Option<&[u8]>,
+    replacement: Option<&[u8]>,
+    sync_metadata: bool,
+    publication_root: &PublicationRoot,
+) -> io::Result<ConditionalFileUpdate> {
+    use windows_sys::{
+        Wdk::Storage::FileSystem::{
+            FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
+        },
+        Win32::Storage::FileSystem::{
+            DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
+        },
+    };
+
+    let parent = file_path
+        .parent()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
+    let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
+    let lock = open_windows_relative(
+        parent_guard.last_handle()?,
+        std::ffi::OsStr::new(".rustfs-cas.lock"),
+        SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
+        FILE_SHARE_READ | FILE_SHARE_WRITE,
+        FILE_OPEN_IF,
+        FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
+        FILE_ATTRIBUTE_NORMAL,
+        true,
+    )?;
+    validate_windows_owned_file(&lock)?;
+    match lock.as_file().try_lock() {
+        Ok(()) => {}
+        Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
+        Err(std::fs::TryLockError::Error(err)) => return Err(err),
+    }
+
+    let current = read_windows_relative_file(file_path, &parent_guard)?;
+    let matches = match (&current, expected) {
+        (None, None) => true,
+        (Some(current), Some(expected)) => current.as_slice() == expected,
+        _ => false,
+    };
+    if !matches {
+        return Ok(match current {
+            None => ConditionalFileUpdate::Missing,
+            Some(_) => ConditionalFileUpdate::Mismatch,
+        });
+    }
+
+    match replacement {
+        Some(replacement) => RenameDestinationPathGuard {
+            directory: parent.to_path_buf(),
+            _directory_guard: parent_guard,
+        }
+        .write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
+        None => {
+            let file_name = file_path
+                .file_name()
+                .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
+            let file = open_windows_relative(
+                parent_guard.last_handle()?,
+                file_name,
+                DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
+                FILE_SHARE_READ,
+                FILE_OPEN,
+                FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
+                0,
+                true,
+            )?;
+            validate_windows_owned_file(&file)?;
+            set_windows_file_delete_on_close(&file, true)?;
+            drop(file);
+            if sync_metadata {
+                fsync_dir_std(parent)?;
+            }
+        }
+    }
+
+    Ok(ConditionalFileUpdate::Updated)
+}
+
 #[cfg(windows)]
 fn open_windows_directory_component(
     parent: &WindowsDirectoryHandle,

crates/ecstore/src/runtime/instance.rs

@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
 use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
 use s3s::region::Region;
 use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, OnceLock};
+use std::sync::{
+    Arc, OnceLock,
+    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
+};
 use tokio::sync::{OnceCell, RwLock};
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
+const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
+const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
+const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -160,10 +167,22 @@ pub struct InstanceContext {
     /// workers (scanner/heal/tier/lifecycle) without touching another instance.
     /// Replaces the process-global cancel-token static.
     background_cancel_token: OnceLock<CancellationToken>,
-    /// Serializes decommission data-movement operations with cancellation and
-    /// a subsequent restart. Readers are held across one object side effect;
-    /// the transition path takes the writer after cancelling the routine.
-    decommission_operation_gate: Arc<RwLock<()>>,
+    /// Serializes data-movement transitions with scanner publication commits.
+    /// Readers are held across one publication commit; movement transitions
+    /// take the writer at their durable state commit boundary.
+    data_movement_operation_gate: Arc<RwLock<()>>,
+    /// Monotonic admission epoch paired with the operation gate. A
+    /// publication admitted before a movement transition must never be
+    /// mistaken for one admitted after the transition.
+    data_movement_operation_epoch: AtomicU64,
+    /// Once the admission epoch reaches its reserved terminal value, no new
+    /// publication may be admitted. Keeping this state separate from the
+    /// saturating counter prevents an unchanged `u64::MAX` value from being
+    /// mistaken for a fresh epoch after overflow.
+    data_movement_operation_epoch_exhausted: AtomicBool,
+    /// Last storage-owned movement snapshot observed under the operation
+    /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
+    scanner_publication_state: AtomicU8,
     /// Resolves object-encryption material at the application boundary.
     object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -204,7 +223,10 @@ impl InstanceContext {
             local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
             bucket_metadata_sys: std::sync::Mutex::new(None),
             background_cancel_token: OnceLock::new(),
-            decommission_operation_gate: Arc::new(RwLock::new(())),
+            data_movement_operation_gate: Arc::new(RwLock::new(())),
+            data_movement_operation_epoch: AtomicU64::new(0),
+            data_movement_operation_epoch_exhausted: AtomicBool::new(false),
+            scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
             object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -223,8 +245,54 @@ impl InstanceContext {
         self.lock_manager.clone()
     }
 
-    pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
-        Arc::clone(&self.decommission_operation_gate)
+    pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
+        Arc::clone(&self.data_movement_operation_gate)
+    }
+
+    pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
+        self.data_movement_operation_epoch.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
+        self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
+        !self.data_movement_operation_epoch_exhausted()
+            && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
+    }
+
+    pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
+        self.scanner_publication_state.store(
+            if blocked {
+                SCANNER_PUBLICATION_STATE_BLOCKED
+            } else {
+                SCANNER_PUBLICATION_STATE_ALLOWED
+            },
+            Ordering::Release,
+        );
+    }
+
+    pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
+        self.scanner_publication_state
+            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
+        let _ = self
+            .data_movement_operation_epoch
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
+        let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
+        if result == u64::MAX {
+            self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
+        }
+        result
+    }
+
+    #[cfg(test)]
+    pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
+        self.data_movement_operation_epoch.store(epoch, Ordering::Release);
+        self.data_movement_operation_epoch_exhausted
+            .store(epoch == u64::MAX, Ordering::Release);
+        self.scanner_publication_state
+            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
     }
 
     /// Install the application-owned object-encryption resolver once.

crates/ecstore/src/services/metrics_realtime.rs

@@ -220,6 +220,14 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
             last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
             last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
             last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
+            last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
+            last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
+            last_publication_state: metrics.usage_freshness.last_publication_state,
+            last_publication_reason: metrics.usage_freshness.last_publication_reason,
+            deferred_pending: metrics.usage_freshness.deferred_pending,
+            deferred_total: metrics.usage_freshness.deferred_total,
+            last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
+            last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
         },
         maintenance_control: MadminScannerMaintenanceControlSnapshot {
             primary_control: metrics.maintenance_control.primary_control,
@@ -816,6 +824,14 @@ mod test {
                 last_usage_save_unix_secs: 12,
                 last_usage_save_result: "success".to_string(),
                 last_usage_save_result_code: 1,
+                last_durable_success_unix_secs: 13,
+                last_publication_unix_secs: 14,
+                last_publication_state: "published".to_string(),
+                last_publication_reason: "complete".to_string(),
+                deferred_pending: true,
+                deferred_total: 15,
+                last_deferred_unix_secs: 16,
+                last_deferred_reason: "data_movement".to_string(),
             },
             ..Default::default()
         });
@@ -828,6 +844,14 @@ mod test {
         assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
         assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
         assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
+        assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
+        assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
+        assert_eq!(scanner.usage_freshness.last_publication_state, "published");
+        assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
+        assert!(scanner.usage_freshness.deferred_pending);
+        assert_eq!(scanner.usage_freshness.deferred_total, 15);
+        assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
+        assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
     }
 
     #[test]

crates/ecstore/src/services/rebalance/control.rs

@@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi
         .unwrap_or_default()
 }
 
-fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
+fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
     if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
-        clear_rebalance_status_refresh(current);
-        return;
+        return clear_rebalance_status_refresh(current);
     }
 
+    let before = current.clone();
     match current.as_mut() {
         Some(current_meta) => {
             if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
@@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted
             *current = Some(persisted);
         }
     }
+
+    match (before.as_ref(), current.as_ref()) {
+        (None, None) => false,
+        (None, Some(_)) | (Some(_), None) => true,
+        (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
+    }
 }
 
-fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
+fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
     if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
-        *current = None;
+        current.take().is_some()
+    } else {
+        false
     }
 }
 
+fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
+    let Some(current) = current else {
+        return true;
+    };
+
+    current.id != persisted.id
+        || current.stopped_at != persisted.stopped_at
+        || current.pool_stats.len() != persisted.pool_stats.len()
+        || current
+            .pool_stats
+            .iter()
+            .zip(persisted.pool_stats.iter())
+            .any(|(current, persisted)| {
+                current.participating != persisted.participating
+                    || current.info.status != persisted.info.status
+                    || current.info.stopping != persisted.info.stopping
+            })
+}
+
 impl ECStore {
     pub(super) async fn save_rebalance_meta_with_merge<S>(
         &self,
@@ -121,7 +148,10 @@ impl ECStore {
             "Loading rebalance metadata"
         );
         let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
+            let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
             {
                 let mut rebalance_meta = self.rebalance_meta.write().await;
 
@@ -130,6 +160,10 @@ impl ECStore {
                 drop(rebalance_meta);
             }
 
+            if movement_changed {
+                self.ctx.advance_data_movement_operation_epoch();
+            }
+            drop(_movement_guard);
             resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
             debug!(
                 event = EVENT_REBALANCE_STATE,
@@ -139,10 +173,15 @@ impl ECStore {
                 "Loaded rebalance metadata"
             );
         } else {
+            let movement_changed = self.rebalance_meta.read().await.is_some();
             {
                 let mut rebalance_meta = self.rebalance_meta.write().await;
                 *rebalance_meta = None;
             }
+            if movement_changed {
+                self.ctx.advance_data_movement_operation_epoch();
+            }
+            drop(_movement_guard);
             debug!(
                 event = EVENT_REBALANCE_STATE,
                 component = LOG_COMPONENT_ECSTORE,
@@ -160,14 +199,20 @@ impl ECStore {
     pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
         let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
         let mut persisted = RebalanceMeta::new();
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         match persisted.load(pool).await {
             Ok(()) => {
                 let mut rebalance_meta = self.rebalance_meta.write().await;
-                merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
+                if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
+                    self.ctx.advance_data_movement_operation_epoch();
+                }
             }
             Err(Error::ConfigNotFound) => {
                 let mut rebalance_meta = self.rebalance_meta.write().await;
-                clear_rebalance_status_refresh(&mut rebalance_meta);
+                if clear_rebalance_status_refresh(&mut rebalance_meta) {
+                    self.ctx.advance_data_movement_operation_epoch();
+                }
             }
             Err(err) => {
                 return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
@@ -349,6 +394,8 @@ impl ECStore {
     #[tracing::instrument(skip(self, bucktes))]
     pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
         let _start_guard = self.start_gate.lock().await;
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
 
         let decommission_running = self.is_decommission_running().await;
         {
@@ -356,12 +403,16 @@ impl ECStore {
             validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
         }
 
-        self.init_rebalance_meta(bucktes).await
+        let id = self.init_rebalance_meta(bucktes).await?;
+        self.ctx.advance_data_movement_operation_epoch();
+        Ok(id)
     }
 
     #[tracing::instrument(skip(self))]
     pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
         let _start_guard = self.start_gate.lock().await;
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
 
         {
             let rebalance_meta = self.rebalance_meta.read().await;
@@ -379,7 +430,10 @@ impl ECStore {
             }
         }
 
-        self.start_rebalance().await
+        if self.start_rebalance_inner().await? {
+            self.ctx.advance_data_movement_operation_epoch();
+        }
+        Ok(())
     }
 
     pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
@@ -548,6 +602,9 @@ impl ECStore {
 
     #[tracing::instrument(skip(self))]
     pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
+        let _start_guard = self.start_gate.lock().await;
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
             stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
@@ -560,6 +617,7 @@ impl ECStore {
                     .await,
                 "stop_rebalance",
             )?;
+            self.ctx.advance_data_movement_operation_epoch();
         }
 
         Ok(())
@@ -570,6 +628,9 @@ impl ECStore {
         expected_id: Option<&str>,
         start_error: String,
     ) -> Result<()> {
+        let _start_guard = self.start_gate.lock().await;
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
             rollback_rebalance_start_meta_snapshot_for_id(
@@ -587,6 +648,7 @@ impl ECStore {
                     .await,
                 "rollback_rebalance_start",
             )?;
+            self.ctx.advance_data_movement_operation_epoch();
         }
 
         Ok(())
@@ -597,6 +659,8 @@ impl ECStore {
             return Ok(());
         }
 
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         let encoded_error = encode_rebalance_stop_propagation_record(&record);
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -610,6 +674,7 @@ impl ECStore {
                     .await,
                 "record_rebalance_stop_propagation",
             )?;
+            self.ctx.advance_data_movement_operation_epoch();
         }
 
         Ok(())
@@ -682,7 +747,7 @@ mod tests {
             ..Default::default()
         };
 
-        merge_rebalance_status_refresh(&mut current, persisted);
+        assert!(merge_rebalance_status_refresh(&mut current, persisted));
 
         let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
         assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
@@ -721,7 +786,7 @@ mod tests {
             ..Default::default()
         };
 
-        merge_rebalance_status_refresh(&mut current, persisted);
+        assert!(!merge_rebalance_status_refresh(&mut current, persisted));
 
         assert!(
             current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),

crates/ecstore/src/services/rebalance/runtime.rs

@@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
 impl ECStore {
     #[tracing::instrument(skip_all)]
     pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
+        if self.start_rebalance_inner().await? {
+            self.ctx.advance_data_movement_operation_epoch();
+        }
+        Ok(())
+    }
+
+    pub(super) async fn start_rebalance_inner(self: &Arc<Self>) -> Result<bool> {
         info!(
             event = EVENT_REBALANCE_STATE,
             component = LOG_COMPONENT_ECSTORE,
@@ -55,6 +64,7 @@ impl ECStore {
         let cancel_tx = CancellationToken::new();
         let rx = cancel_tx.clone();
         let mut meta_to_save = None;
+        let mut movement_changed = false;
 
         {
             let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -72,14 +82,16 @@ impl ECStore {
                     reason = "already_in_progress",
                     "Skipped duplicate rebalance start"
                 );
-                return Ok(());
+                return Ok(false);
             }
             let now = OffsetDateTime::now_utc();
             if complete_rebalance_pools_at_goal(meta, now) {
                 meta_to_save = Some(meta.clone());
+                movement_changed = true;
             }
             if complete_rebalance_pools_with_empty_queue(meta, now) {
                 meta_to_save = Some(meta.clone());
+                movement_changed = true;
             }
             meta.cancel = Some(cancel_tx);
 
@@ -118,7 +130,7 @@ impl ECStore {
                 reason = "no_participants",
                 "Skipped rebalance start because no pools are participating"
             );
-            return Ok(());
+            return Ok(movement_changed);
         }
 
         let mut workers_started = 0usize;
@@ -186,7 +198,7 @@ impl ECStore {
                 reason = "no_local_participants",
                 "Skipped rebalance start because no local pools are participating"
             );
-            return Ok(());
+            return Ok(movement_changed);
         }
 
         info!(
@@ -197,7 +209,7 @@ impl ECStore {
             worker_count = workers_started,
             "Rebalance started"
         );
-        Ok(())
+        Ok(true)
     }
 
     #[tracing::instrument(skip(self, rx))]
@@ -214,53 +226,77 @@ impl ECStore {
             let mut quit = false;
 
             loop {
+                let mut terminal_state_saved = false;
                 tokio::select! {
                     result = done_rx.recv() => {
                         quit = true;
                         let now = OffsetDateTime::now_utc();
                         let terminal_event = classify_rebalance_terminal_event(result, now);
                         msg = terminal_event.message().to_string();
-                        let mut rebalance_meta = store.rebalance_meta.write().await;
-                        if let Some(meta) = rebalance_meta.as_mut() {
-                            let meta_stopped = meta.stopped_at.is_some();
-                            if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
-                                if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
-                                    && has_rebalance_cleanup_warnings(pool_stat)
-                                {
-                                    pool_stat.info.stopping = false;
-                                    pool_stat.info.status = RebalStatus::Failed;
-                                    pool_stat.info.end_time = Some(now);
-                                    pool_stat.info.last_error = Some(
-                                        pool_stat
-                                            .cleanup_warnings
-                                            .last_message
-                                            .clone()
-                                            .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
-                                    );
-                                } else if should_preserve_rebalance_stopped_state(
-                                    meta_stopped,
-                                    pool_stat.info.status,
-                                    &terminal_event,
-                                ) {
-                                    debug!(
-                                        event = EVENT_REBALANCE_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_REBALANCE,
-                                        pool_index,
-                                        state = "stopped_preserved",
-                                        "Preserved stopped rebalance status"
-                                    );
+                        let movement_gate = store.ctx.data_movement_operation_gate();
+                        let movement_guard = movement_gate.write().await;
+                        let previous_meta = store.rebalance_meta.read().await.clone();
+                        let terminal_state_present = {
+                            let mut rebalance_meta = store.rebalance_meta.write().await;
+                            if let Some(meta) = rebalance_meta.as_mut() {
+                                let meta_stopped = meta.stopped_at.is_some();
+                                if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
+                                    if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
+                                        && has_rebalance_cleanup_warnings(pool_stat)
+                                    {
+                                        pool_stat.info.stopping = false;
+                                        pool_stat.info.status = RebalStatus::Failed;
+                                        pool_stat.info.end_time = Some(now);
+                                        pool_stat.info.last_error = Some(
+                                            pool_stat
+                                                .cleanup_warnings
+                                                .last_message
+                                                .clone()
+                                                .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
+                                        );
+                                    } else if should_preserve_rebalance_stopped_state(
+                                        meta_stopped,
+                                        pool_stat.info.status,
+                                        &terminal_event,
+                                    ) {
+                                        debug!(
+                                            event = EVENT_REBALANCE_STATE,
+                                            component = LOG_COMPONENT_ECSTORE,
+                                            subsystem = LOG_SUBSYSTEM_REBALANCE,
+                                            pool_index,
+                                            state = "stopped_preserved",
+                                            "Preserved stopped rebalance status"
+                                        );
+                                    } else {
+                                        pool_stat.info.stopping = false;
+                                        apply_rebalance_terminal_event(
+                                            &mut pool_stat.info.status,
+                                            &mut pool_stat.info.end_time,
+                                            &mut pool_stat.info.last_error,
+                                            terminal_event,
+                                            now,
+                                        );
+                                    }
+                                    true
                                 } else {
-                                    pool_stat.info.stopping = false;
-                                    apply_rebalance_terminal_event(
-                                        &mut pool_stat.info.status,
-                                        &mut pool_stat.info.end_time,
-                                        &mut pool_stat.info.last_error,
-                                        terminal_event,
-                                        now,
-                                    );
+                                    false
                                 }
+                            } else {
+                                false
                             }
+                        };
+
+                        if terminal_state_present {
+                            if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await {
+                                let mut rebalance_meta = store.rebalance_meta.write().await;
+                                *rebalance_meta = previous_meta;
+                                drop(movement_guard);
+                                return Err(Error::other(format!(
+                                    "rebalance terminal state save failed for pool {pool_index}: {err}"
+                                )));
+                            }
+                            store.ctx.advance_data_movement_operation_epoch();
+                            terminal_state_saved = true;
                         }
                     }
                     _ = timer.tick() => {
@@ -269,7 +305,7 @@ impl ECStore {
                     }
                 }
 
-                if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
+                if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
                     let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
                     error!("{} err: {:?}", msg, wrapped);
                     if quit {
@@ -590,16 +626,14 @@ impl ECStore {
                     meta.percent_free_goal,
                 )
             {
-                pool_stat.info.status = RebalStatus::Completed;
-                pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
                 info!(
                     event = EVENT_REBALANCE_STATE,
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REBALANCE,
                     pool_index,
-                    state = "completed",
+                    state = "completion_ready",
                     percent_free = pfi,
-                    "Marked rebalance pool completed"
+                    "Rebalance pool reached completion goal"
                 );
                 return true;
             }
@@ -612,6 +646,12 @@ impl ECStore {
 impl ECStore {
     #[tracing::instrument(skip(self))]
     pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
+        self.save_rebalance_stats_inner(pool_idx, opt).await
+    }
+
+    pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
             let Some(meta) = rebalance_meta.as_mut() else {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -71,6 +71,9 @@ use crate::set_disk::shard_source::ShardReadCost;
 use futures::FutureExt as _;
 use futures::stream::{FuturesUnordered, StreamExt};
 use metrics::counter;
+use rustfs_io_metrics::internode_metrics::{
+    INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP,
+};
 use std::{
     collections::{HashMap, HashSet, VecDeque},
     future::Future,
@@ -140,6 +143,21 @@ struct CoalescedReadVersionRequest {
     tx: oneshot::Sender<disk::error::Result<FileInfo>>,
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct ExpectedBatchReadVersionItem {
+    path: String,
+    version_id: String,
+}
+
+impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem {
+    fn from(item: &BatchReadVersionItem) -> Self {
+        Self {
+            path: item.path.clone(),
+            version_id: item.version_id.clone(),
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
 struct ReadVersionCoalescerKey {
     disk: usize,
@@ -178,6 +196,16 @@ fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
     .increment(1);
 }
 
+fn batch_read_version_stage_timer() -> Option<Instant> {
+    rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now)
+}
+
+fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
+    if let Some(started_at) = started_at {
+        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
+    }
+}
+
 async fn read_version_via_coalescer(
     disk: DiskStore,
     org_bucket: &str,
@@ -227,8 +255,12 @@ async fn read_version_via_coalescer(
         flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
     }
 
-    rx.await
-        .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
+    let wait_started = batch_read_version_stage_timer();
+    let response = rx
+        .await
+        .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")));
+    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started);
+    response
 }
 
 async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
@@ -266,7 +298,7 @@ async fn flush_read_version_coalescer_pending(
         items.push(request.item);
     }
 
-    let expected_items = items.clone();
+    let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::<Vec<_>>();
     record_read_version_coalescer_event("attempted_batch", items.len());
     let result =
         match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
@@ -277,7 +309,9 @@ async fn flush_read_version_coalescer_pending(
         };
     match result {
         Ok(responses) => {
+            let map_started = batch_read_version_stage_timer();
             let results = map_batch_read_version_responses(&expected_items, responses);
+            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started);
             for (tx, result) in senders.into_iter().zip(results) {
                 let _ = tx.send(result);
             }
@@ -292,7 +326,7 @@ async fn flush_read_version_coalescer_pending(
 }
 
 fn map_batch_read_version_responses(
-    expected_items: &[BatchReadVersionItem],
+    expected_items: &[ExpectedBatchReadVersionItem],
     responses: Vec<BatchReadVersionResp>,
 ) -> Vec<crate::disk::error::Result<FileInfo>> {
     let mut results = (0..expected_items.len())
@@ -1412,8 +1446,11 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
 
     // Reservation won: this sighting owns the repair records for the object,
     // including the durable journal intent when the caller asked for one.
-    if let Some((kind, version_uuid)) = mrf_intent {
-        rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid);
+    if let Some((kind, version_uuid)) = mrf_intent
+        && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
+    {
+        let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
+        let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
     }
 
     let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
@@ -6878,6 +6915,7 @@ mod tests {
             },
         ];
 
+        let expected_items = expected_batch_read_version_items(&expected_items);
         let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
         let first = results
             .next()
@@ -6918,6 +6956,7 @@ mod tests {
                 version_id: "v-b".to_string(),
             },
         ];
+        let expected_items = expected_batch_read_version_items(&expected_items);
         let results = map_batch_read_version_responses(
             &expected_items,
             vec![
@@ -6957,6 +6996,7 @@ mod tests {
             path: "object-a".to_string(),
             version_id: "v-a".to_string(),
         }];
+        let expected_items = expected_batch_read_version_items(&expected_items);
         let mismatched = map_batch_read_version_responses(
             &expected_items,
             vec![BatchReadVersionResp {
@@ -7018,6 +7058,10 @@ mod tests {
         );
     }
 
+    fn expected_batch_read_version_items(items: &[BatchReadVersionItem]) -> Vec<ExpectedBatchReadVersionItem> {
+        items.iter().map(ExpectedBatchReadVersionItem::from).collect()
+    }
+
     /// Isolation guard: unobserved objects record nothing (so parallel tests do
     /// not inflate one another), and a scope clears its own counts on drop.
     #[tokio::test]

crates/ecstore/src/set_disk/mod.rs

@@ -47,6 +47,7 @@ use crate::bucket::metadata_sys;
 use crate::bucket::metadata_sys::ObjectLockConfigState;
 use crate::bucket::object_lock::objectlock_sys::{
     check_object_lock_for_deletion_with_config, check_object_lock_for_deletion_with_state, check_retention_for_modification,
+    replication_write_may_pass_worm_gate,
 };
 use crate::bucket::replication::{
     ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
@@ -3614,6 +3615,26 @@ impl SetDisks {
         &self.ctx
     }
 
+    /// Admit one short scanner cache publication under this set's instance
+    /// movement fence. The caller must hold the returned guard through its
+    /// final conditional cache write; no scan-round work belongs under it.
+    pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let operation_guard = operation_gate.read_owned().await;
+        if self.ctx.scanner_publication_state_allowed() {
+            let epoch = self.ctx.data_movement_operation_epoch();
+            return Some((operation_guard, epoch));
+        }
+
+        // The owner deliberately marks the cached state UNKNOWN after every
+        // movement epoch advance. Do not strand remote scanner writers in that
+        // state: release this guard before asking the storage owner to refresh
+        // its durable movement snapshot, since the owner uses the same gate.
+        drop(operation_guard);
+        let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?;
+        owner.scanner_data_usage_publication_admission_guard().await
+    }
+
     /// Whether both sets' namespace-lock implementations cover the same object key.
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
@@ -4736,7 +4757,15 @@ impl SetDisks {
                 achieved: 0,
             });
         }
-        let parts_metadata = vec![fi.clone(); disks.len()];
+        // Rebuilt tiered metadata starts with index zero, but shuffling validates
+        // each source slot before assigning the shuffled index below.
+        let parts_metadata: Vec<FileInfo> = (0..disks.len())
+            .map(|disk_index| {
+                let mut part = fi.clone();
+                part.erasure.index = fi.erasure.distribution[disk_index];
+                part
+            })
+            .collect();
         let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
 
         let mut errs = Vec::with_capacity(shuffle_disks.len());

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -556,6 +556,69 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::
 }
 
 impl SetDisks {
+    async fn discover_multipart_upload_paths(
+        &self,
+        orig_bucket: &str,
+        error_path: &str,
+    ) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize)> {
+        let disks = self.disks.read().await.clone();
+        if disks.is_empty() {
+            return Err(Error::ErasureReadQuorum);
+        }
+        let discovery_quorum = if self.default_parity_count == 0 {
+            disks.len()
+        } else {
+            (disks.len() / 2).max(1)
+        };
+        let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
+        let mut candidate_counts = HashMap::<String, usize>::new();
+        let mut discovery_tasks = JoinSet::new();
+        for (index, disk) in disks.iter().enumerate() {
+            let disk = disk.clone();
+            let orig_bucket = orig_bucket.to_string();
+            discovery_tasks.spawn(async move {
+                let result = match disk {
+                    Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket).await,
+                    None => Err(DiskError::DiskNotFound),
+                };
+                (index, result)
+            });
+        }
+
+        while let Some(task_result) = discovery_tasks.join_next().await {
+            let Ok((index, result)) = task_result else {
+                continue;
+            };
+            match result {
+                Ok(paths) => {
+                    discovery_errors[index] = None;
+                    for path in paths {
+                        *candidate_counts.entry(path).or_insert(0) += 1;
+                    }
+                }
+                Err(err) => discovery_errors[index] = Some(err),
+            }
+        }
+
+        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
+            return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
+        }
+
+        let mut candidate_paths = candidate_counts
+            .into_iter()
+            .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path))
+            .collect::<Vec<_>>();
+        candidate_paths.sort_unstable();
+        Ok((disks, candidate_paths, discovery_quorum))
+    }
+
+    pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
+        let (_, paths, _) = self
+            .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET)
+            .await?;
+        Ok(paths.into_iter().next())
+    }
+
     async fn acquire_multipart_upload_read_lock(
         &self,
         op: &'static str,
@@ -747,53 +810,7 @@ impl SetDisks {
         max_uploads: usize,
         expected_incarnation_id: Option<Uuid>,
     ) -> Result<ListMultipartsInfo> {
-        let disks = self.disks.read().await.clone();
-        if disks.is_empty() {
-            return Err(Error::ErasureReadQuorum);
-        }
-        let discovery_quorum = if self.default_parity_count == 0 {
-            disks.len()
-        } else {
-            (disks.len() / 2).max(1)
-        };
-        let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
-        let mut candidate_counts = HashMap::<String, usize>::new();
-        let mut discovery_tasks = JoinSet::new();
-        for (index, disk) in disks.iter().enumerate() {
-            let disk = disk.clone();
-            let bucket = bucket.to_string();
-            discovery_tasks.spawn(async move {
-                let result = match disk {
-                    Some(disk) => multipart_upload_paths_on_disk(disk, &bucket).await,
-                    None => Err(DiskError::DiskNotFound),
-                };
-                (index, result)
-            });
-        }
-
-        while let Some(task_result) = discovery_tasks.join_next().await {
-            let Ok((index, result)) = task_result else {
-                continue;
-            };
-            match result {
-                Ok(paths) => {
-                    discovery_errors[index] = None;
-                    for path in paths {
-                        *candidate_counts.entry(path).or_insert(0) += 1;
-                    }
-                }
-                Err(err) => discovery_errors[index] = Some(err),
-            }
-        }
-
-        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
-            return Err(to_object_err(err.into(), vec![bucket, prefix]));
-        }
-
-        let candidate_paths = candidate_counts
-            .into_iter()
-            .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path))
-            .collect::<Vec<_>>();
+        let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix).await?;
         let listed_uploads = stream::iter(candidate_paths)
             .map(|upload_path| {
                 let disks = &disks;
@@ -2318,6 +2335,43 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             fi.set_data_moved();
         }
 
+        // Receiver-side LWW (rustfs/backlog#1953): the multipart replication
+        // transport carries the category values at CreateMultipartUpload (in
+        // the staged upload metadata) and the source category timestamps on
+        // the complete request. Read the destination version under the held
+        // object write lock and keep any category this site modified more
+        // recently. Only an absent version has no local state to compare;
+        // other read failures must leave the upload retryable rather than
+        // committing inbound metadata without the LWW check.
+        if crate::set_disk::ops::object::replication_lww_applicable(opts)
+            && let Some(version_id) = fi.version_id
+        {
+            match self
+                .get_object_info(
+                    bucket,
+                    object,
+                    &ObjectOptions {
+                        version_id: Some(version_id.to_string()),
+                        no_lock: true,
+                        metadata_cache_safe: false,
+                        versioned: opts.versioned,
+                        version_suspended: opts.version_suspended,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                Ok(existing) => {
+                    let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
+                    crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
+                }
+                // Version absent: first replication of this version, nothing
+                // local to compare — the normal path, not a degraded one.
+                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
+                Err(err) => return Err(err),
+            }
+        }
+
         for meta in parts_metadatas.iter_mut() {
             if meta.has_valid_erasure_geometry() {
                 meta.size = fi.size;
@@ -7055,6 +7109,146 @@ mod tests {
                 .await
         }
 
+        /// Receiver-side LWW on the multipart replication transport
+        /// (rustfs/backlog#1953): a metadata-only replication of a multipart
+        /// source object rides CreateMultipartUpload (category values in the
+        /// upload metadata) + CompleteMultipartUpload (category timestamps in
+        /// the complete options). A stale inbound tagging timestamp must not
+        /// overwrite a newer locally-tagged destination version.
+        #[tokio::test]
+        #[serial]
+        async fn complete_multipart_upload_stale_replication_tags_keep_local() {
+            use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+            use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
+            use time::format_description::well_known::Rfc3339;
+
+            const T_OLD: &str = "2026-01-01T00:00:00Z";
+            const T_LOCAL: &str = "2026-02-01T00:00:00Z";
+
+            let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+            let bucket = "multipart-replication-lww-bucket";
+            let object = "object";
+            make_bucket_on_all(&disk_stores, bucket).await;
+
+            // Local destination version with newer tags.
+            let version_id = Uuid::new_v4();
+            let mut local_metadata = HashMap::new();
+            local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
+            rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
+            let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
+            set_disks
+                .put_object(
+                    bucket,
+                    object,
+                    &mut local_reader,
+                    &ObjectOptions {
+                        versioned: true,
+                        version_id: Some(version_id.to_string()),
+                        user_defined: local_metadata,
+                        // Explicit-version PUTs require the bucket Object Lock snapshot.
+                        object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
+                            crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                        ))),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("local versioned put should commit");
+
+            // Inbound replication upload carrying older tags for the same version.
+            let mut inbound_metadata = HashMap::new();
+            inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
+            rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
+            let create_opts = ObjectOptions {
+                versioned: true,
+                user_defined: inbound_metadata,
+                ..Default::default()
+            };
+            let (upload_id, parts) =
+                stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
+            rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
+
+            let complete_opts = ObjectOptions {
+                versioned: true,
+                replication_request: true,
+                replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
+                ..Default::default()
+            };
+
+            // Make the destination version unreadable on quorum while the
+            // staged upload remains intact. The commit barrier lets the old
+            // fail-open path move past the LWW read; restoring the metadata
+            // there proves it would otherwise commit the stale tags.
+            let mut damaged_metadata = Vec::new();
+            for temp_dir in temp_dirs.iter().take(3) {
+                let path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+                let original = tokio::fs::read(&path).await.expect("destination xl.meta should be readable");
+                tokio::fs::write(&path, b"not an xl.meta")
+                    .await
+                    .expect("destination xl.meta should be corruptible");
+                damaged_metadata.push((path, original));
+            }
+
+            let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeLockLost);
+            let first_set = set_disks.clone();
+            let first_upload_id = upload_id.clone();
+            let first_parts = parts.clone();
+            let first_opts = complete_opts.clone();
+            let mut first_completion = tokio::spawn(async move {
+                first_set
+                    .complete_multipart_upload(bucket, object, &first_upload_id, first_parts, &first_opts)
+                    .await
+            });
+
+            let first_result = tokio::select! {
+                result = &mut first_completion => result.expect("first completion task should finish"),
+                () = barrier.wait_until_paused() => {
+                    for (path, original) in &damaged_metadata {
+                        tokio::fs::write(path, original).await.expect("destination xl.meta should be restorable");
+                    }
+                    barrier.release();
+                    first_completion.await.expect("released completion task should finish")
+                }
+            };
+            for (path, original) in &damaged_metadata {
+                tokio::fs::write(path, original)
+                    .await
+                    .expect("destination xl.meta should be restored");
+            }
+            drop(barrier);
+
+            let first_error = first_result.expect_err("unreadable destination metadata must fail before multipart commit");
+            assert!(
+                !(is_err_object_not_found(&first_error) || is_err_version_not_found(&first_error)),
+                "corrupt destination metadata must not be treated as an absent version: {first_error}"
+            );
+
+            set_disks
+                .clone()
+                .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
+                .await
+                .expect("replication multipart completion should succeed even when a category keeps local values");
+
+            let info = set_disks
+                .get_object_info(
+                    bucket,
+                    object,
+                    &ObjectOptions {
+                        versioned: true,
+                        version_id: Some(version_id.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("completed version should be readable");
+            assert_eq!(
+                info.user_tags.as_str(),
+                "site=local",
+                "older inbound multipart tags must not overwrite newer local tags"
+            );
+            assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
+        }
+
         #[tokio::test]
         #[serial]
         async fn complete_multipart_upload_assigns_completion_version_id() {

crates/ecstore/src/set_disk/ops/object.rs

@@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{
     transition_transaction::{
         TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
         TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
-        save_transition_transaction_record,
+        load_transition_transaction_record, save_transition_transaction_record,
     },
 };
 use crate::bucket::quota::reservation;
@@ -1881,6 +1881,110 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
     transported
 }
 
+/// True when an authorized replication write carries at least one per-category
+/// source timestamp, i.e. receiver-side LWW has something to judge.
+pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
+    opts.replication_request
+        && (opts.replication_tagging_timestamp.is_some()
+            || opts.replication_retention_timestamp.is_some()
+            || opts.replication_legalhold_timestamp.is_some())
+}
+
+/// The stored per-category state of a destination version, as compared by
+/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
+/// externalizes tags into `user_tags` (stripping the metadata key), so the
+/// tag value is folded back into map form here.
+pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
+    let mut stored = (*existing.user_defined).clone();
+    if !existing.user_tags.is_empty() {
+        stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
+    }
+    stored
+}
+
+/// Receiver-side last-writer-wins for authorized replication writes
+/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
+/// whole-object transports, so in active-active topologies an inbound write
+/// carries the source's tags / retention / legal hold verbatim and would
+/// otherwise overwrite a category the destination modified more recently —
+/// both sites end up permanently diverged while reporting COMPLETED.
+///
+/// Judged per category, only when the inbound request carries that category's
+/// source timestamp (`ObjectOptions::replication_*_timestamp`):
+/// - stored timestamp newer than inbound: the local category values and
+///   timestamp are kept; the rest of the write proceeds per the inbound
+///   metadata and the object-level result stays successful (failing the write
+///   instead would loop through MRF, re-delivering the stale value forever);
+/// - otherwise the inbound category wins and its internal timestamp key is
+///   pinned to the source-authored time — the PUT path re-stamps the
+///   object-lock timestamps with the receiver's clock
+///   (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
+///   `now()` via `eval_metadata`), which would make the replica's clock the
+///   LWW authority and wedge later convergence;
+/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
+///   overwrite behavior is preserved.
+///
+/// Returns whether `inbound` was modified. Callers must hold the object write
+/// lock so the stored values compared here are the ones being replaced.
+pub(in crate::set_disk) fn merge_replication_metadata_lww(
+    inbound: &mut HashMap<String, String>,
+    existing: &HashMap<String, String>,
+    opts: &ObjectOptions,
+) -> bool {
+    use rustfs_utils::http::headers::{
+        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
+    };
+    use rustfs_utils::http::metadata_compat::{
+        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
+        remove_str,
+    };
+    use time::format_description::well_known::Rfc3339;
+
+    let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
+        (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
+        (
+            opts.replication_retention_timestamp,
+            SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
+            &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
+        ),
+        (
+            opts.replication_legalhold_timestamp,
+            SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
+            &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
+        ),
+    ];
+
+    let mut changed = false;
+    for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
+        let Some(inbound_timestamp) = inbound_timestamp else { continue };
+        let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
+        let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
+        if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
+            inbound.retain(|key, _| !is_category_value_key(key));
+            remove_str(inbound, timestamp_suffix);
+            for (key, value) in existing {
+                if is_category_value_key(key) {
+                    inbound.insert(key.clone(), value.clone());
+                }
+            }
+            // Restore the winning timestamp via insert_str, not a verbatim key
+            // copy: a MinIO-written version may carry only the
+            // x-minio-internal- key, and the dual-key invariant requires every
+            // write to produce both keys.
+            if let Some(stored_value) = get_str(existing, timestamp_suffix) {
+                rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
+            }
+            changed = true;
+        } else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
+            && get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
+        {
+            rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
+            changed = true;
+        }
+    }
+    changed
+}
+
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
         &self,
@@ -2073,6 +2177,14 @@ impl SetDisks {
                 user_defined.insert(key.clone(), value.clone());
             }
         }
+        if replication_lww_applicable(opts) {
+            // Object Lock evaluation stamps category timestamps with this
+            // receiver's clock. Pin them back to the source-authored times
+            // before the first copy of a version is committed; the existing-
+            // version branch below may still replace them with newer local
+            // state.
+            merge_replication_metadata_lww(&mut user_defined, &HashMap::new(), opts);
+        }
         if expected_restore_operation_id.is_some() {
             rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_OPERATION_ID);
         }
@@ -2559,9 +2671,35 @@ impl SetDisks {
                         let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
                             Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
                         })?;
-                        if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
+                        // The WORM gate protects the locked version from local
+                        // overwrites; an authorized replication write passes it
+                        // only when the LWW merge below will judge every
+                        // locking category (see
+                        // `replication_write_may_pass_worm_gate`, which judges
+                        // the same authoritative lock state as the gate,
+                        // bucket default retention included). Gate first so
+                        // malformed lock metadata still fails closed.
+                        if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some()
+                            && !replication_write_may_pass_worm_gate(object_lock_config.state(), &existing, opts)?
+                        {
                             return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
                         }
+                        // Receiver-side LWW (rustfs/backlog#1953): reuse this
+                        // commit-lock read of the destination version so a
+                        // category (tags / retention / legal hold) modified
+                        // more recently on this site is kept instead of being
+                        // overwritten by the inbound replication metadata.
+                        if replication_lww_applicable(opts) {
+                            let stored = stored_replication_category_metadata(&existing);
+                            let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
+                            if merge_replication_metadata_lww(&mut merged, &stored, opts) {
+                                for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
+                                    if disk.is_some() {
+                                        pfi.metadata = merged.clone();
+                                    }
+                                }
+                            }
+                        }
                     }
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
@@ -4248,7 +4386,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction,
 
 async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
     if let Some(api) = api {
-        return delete_transition_transaction_record(api.clone(), transaction_id).await;
+        let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
+            Ok(transaction) => transaction,
+            Err(Error::ConfigNotFound) => return Ok(()),
+            Err(err) => return Err(err),
+        };
+        return delete_transition_transaction_record(api.clone(), &transaction).await;
     }
     Ok(())
 }
@@ -6637,12 +6780,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
     async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
         // MRF journal intent: partial-write recovery must survive a restart
         // (HS-01); the heal request below remains the in-memory fast path.
-        rustfs_common::mrf_channel::try_send_mrf_intent(
-            rustfs_common::mrf_channel::MrfKind::PartialWrite,
-            bucket,
-            object,
-            uuid::Uuid::try_parse(version_id).ok(),
-        );
+        let version_uuid = if version_id.is_empty() {
+            Some(None)
+        } else {
+            uuid::Uuid::try_parse(version_id).ok().map(Some)
+        };
+        if let Some(version_uuid) = version_uuid
+            && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(self.pool_index), u32::try_from(self.set_index))
+        {
+            let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
+            let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
+                rustfs_common::mrf_channel::MrfKind::PartialWrite,
+                bucket,
+                object,
+                version_uuid,
+                Some(scope),
+            );
+        }
         let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
             bucket.to_string(),
             Some(object.to_string()),
@@ -8066,6 +8220,640 @@ mod replication_quota_safety_tests {
     }
 }
 
+#[cfg(test)]
+mod replication_lww_tests {
+    //! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
+    //! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
+    //! (tags / retention / legal hold) is older than the destination version's
+    //! stored timestamp must keep the local category values instead of
+    //! overwriting them; categories are judged independently and the write
+    //! itself still succeeds.
+
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
+    use super::*;
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
+    use rustfs_utils::http::headers::{
+        AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
+    };
+    use rustfs_utils::http::{
+        SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
+        insert_str,
+    };
+    use time::format_description::well_known::Rfc3339;
+
+    const T_OLD: &str = "2026-01-01T00:00:00Z";
+    const T_LOCAL: &str = "2026-02-01T00:00:00Z";
+    const T_NEW: &str = "2026-03-01T00:00:00Z";
+
+    fn parse_ts(value: &str) -> OffsetDateTime {
+        OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
+    }
+
+    async fn make_bucket(disks: &[DiskStore], bucket: &str) {
+        for disk in disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+    }
+
+    async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        set_disks
+            .put_object(bucket, object, &mut reader, opts)
+            .await
+            .expect("versioned put should commit");
+        assert_eq!(opts.version_id.as_deref(), Some(version_id));
+    }
+
+    fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
+        ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.to_string()),
+            user_defined,
+            // Explicit-version PUTs require the bucket Object Lock snapshot.
+            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+            ))),
+            ..Default::default()
+        }
+    }
+
+    /// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
+    async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
+        let mut user_defined = HashMap::new();
+        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
+        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
+        put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
+    }
+
+    fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
+        let mut user_defined = HashMap::new();
+        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
+        insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
+        ObjectOptions {
+            replication_request: true,
+            replication_tagging_timestamp: Some(parse_ts(timestamp)),
+            ..versioned_opts(version_id, user_defined)
+        }
+    }
+
+    async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
+        set_disks
+            .get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
+            .await
+            .expect("version should be readable")
+    }
+
+    #[tokio::test]
+    async fn inbound_stale_tagging_keeps_newer_local_tags() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-tagging-stale";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_tags.as_str(),
+            "site=local",
+            "older inbound tags must not overwrite newer local tags"
+        );
+        assert_eq!(
+            get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
+            Some(T_LOCAL),
+            "the winning local tagging timestamp must be preserved"
+        );
+    }
+
+    #[tokio::test]
+    async fn inbound_newer_tagging_overwrites_local_tags() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-tagging-newer";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_tagging_opts(&version_id, "site=remote", T_NEW),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_tags.as_str(),
+            "site=remote",
+            "newer inbound tags must overwrite older local tags"
+        );
+        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
+    }
+
+    #[tokio::test]
+    async fn inbound_wins_when_local_has_no_tagging_timestamp() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-tagging-no-local-ts";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        // Pre-P1-6 data: local tags without a stored tagging timestamp.
+        let mut user_defined = HashMap::new();
+        user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_tags.as_str(),
+            "site=remote",
+            "without a local timestamp the inbound category must win (pre-LWW data compatibility)"
+        );
+    }
+
+    #[tokio::test]
+    async fn categories_are_judged_independently() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-category-independent";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+
+        // Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
+        // timestamp key only, the shape a replicated retention clear stores.
+        // (An active local retention would already block the overwrite at the
+        // WORM gate; the LWW-reachable retention states are cleared/expired.)
+        let mut local = HashMap::new();
+        local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
+        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
+        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        // Inbound: older tags (T_OLD), newer retention (T_NEW).
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
+        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
+            replication_retention_timestamp: Some(parse_ts(T_NEW)),
+            ..versioned_opts(&version_id, inbound)
+        };
+        put_version(&set_disks, bucket, object, &version_id, &opts).await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
+        assert_eq!(
+            info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
+            Some("COMPLIANCE"),
+            "the newer retention category must be applied in the same write"
+        );
+        assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
+    }
+
+    #[tokio::test]
+    async fn inbound_stale_legal_hold_keeps_local_value() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-legalhold-stale";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+
+        // Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
+        // still ON already blocks the overwrite at the WORM gate; the
+        // LWW-reachable divergence is a stale inbound ON resurrecting a hold
+        // that was released more recently on this site.)
+        let mut local = HashMap::new();
+        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
+        insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
+            ..versioned_opts(&version_id, inbound)
+        };
+        put_version(&set_disks, bucket, object, &version_id, &opts).await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
+            Some("OFF"),
+            "a stale inbound legal hold must not resurrect a hold released more recently"
+        );
+        assert_eq!(
+            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
+            Some(T_LOCAL)
+        );
+    }
+
+    /// Dual-key invariant under LWW: a MinIO-written destination version may
+    /// carry only the x-minio-internal timestamp key; when the local category
+    /// wins, the restored map must still hold BOTH compatibility keys.
+    #[test]
+    fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
+        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
+        let existing = HashMap::from([
+            (AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
+            ("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
+        ]);
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_tagging_timestamp: Some(parse_ts(T_OLD)),
+            ..Default::default()
+        };
+
+        assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
+        assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
+        assert_eq!(
+            inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
+            Some(T_LOCAL),
+            "the RustFS twin key must be materialized even when the source version only had the MinIO key"
+        );
+        assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
+    }
+
+    /// When the inbound category wins, the stored timestamp must be the
+    /// source-authored one: the PUT path's eval_metadata stamps the
+    /// object-lock timestamps with the receiver's clock
+    /// (`parse_object_lock_retention`), which would otherwise make this
+    /// replica's clock the LWW authority and wedge later convergence.
+    #[tokio::test]
+    async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-retention-ts-pinned";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+
+        // Local cleared retention at T_OLD.
+        let mut local = HashMap::new();
+        insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        // Inbound newer retention: the source authored T_LOCAL, but the PUT
+        // path's eval_metadata stomped the metadata key with receiver-now
+        // (simulated by T_NEW here).
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
+            ..versioned_opts(&version_id, inbound)
+        };
+        put_version(&set_disks, bucket, object, &version_id, &opts).await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
+            Some(T_LOCAL),
+            "the stored category timestamp must be the source-authored time, not the receiver's clock"
+        );
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
+    }
+
+    #[tokio::test]
+    async fn first_inbound_version_pins_source_authored_timestamp() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-first-version-ts-pinned";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
+        let mut evaluated = inbound.clone();
+        insert_str(&mut evaluated, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_NEW.to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
+            eval_metadata: Some(evaluated),
+            ..versioned_opts(&version_id, inbound)
+        };
+
+        put_version(&set_disks, bucket, object, &version_id, &opts).await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
+            Some(T_OLD),
+            "the first copy must store the source timestamp, not the receiver evaluation time"
+        );
+    }
+
+    #[tokio::test]
+    async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-tagging-deleted";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        // Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
+        let mut local = HashMap::new();
+        insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_tagging_opts(&version_id, "site=remote", T_OLD),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert!(
+            info.user_tags.is_empty(),
+            "a newer local tag deletion must not be resurrected by older inbound tags"
+        );
+        assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
+    }
+
+    /// Destination version under an active legal hold at `hold_timestamp`,
+    /// plus an active COMPLIANCE retention (no retention timestamp).
+    async fn seed_locked_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, hold_timestamp: &str) {
+        let mut local = HashMap::new();
+        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
+        insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, hold_timestamp.to_string());
+        local.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
+        local.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
+        put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, local)).await;
+    }
+
+    /// Inbound legal-hold release from a source that also carries the (same)
+    /// COMPLIANCE retention; the sender stamps a source timestamp for every
+    /// category the source version has.
+    fn inbound_legal_hold_release_opts(version_id: &str, timestamp: &str) -> ObjectOptions {
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, timestamp.to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
+        ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: Some(parse_ts(timestamp)),
+            replication_retention_timestamp: Some(parse_ts(T_OLD)),
+            ..versioned_opts(version_id, inbound)
+        }
+    }
+
+    /// The source's lock state governs the replica: a legal-hold release (or a
+    /// retention change) can only reach this site through the authorized
+    /// replication write, so the commit-time WORM gate must not reject it
+    /// because the destination version is currently locked.
+    #[tokio::test]
+    async fn inbound_newer_legal_hold_release_updates_locked_version() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-release-newer";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_legal_hold_release_opts(&version_id, T_NEW),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
+            Some("OFF"),
+            "a newer source-side legal hold release must be applied to the locked replica"
+        );
+        assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(), Some(T_NEW));
+        assert_eq!(
+            info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
+            Some("COMPLIANCE"),
+            "the untouched retention category must survive the write"
+        );
+    }
+
+    /// Skipping the WORM gate for replication writes must not weaken LWW: a
+    /// stale inbound release still loses to a hold applied more recently here.
+    #[tokio::test]
+    async fn inbound_stale_legal_hold_release_keeps_newer_local_hold() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-release-stale";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_locked_version(&set_disks, bucket, object, &version_id, T_LOCAL).await;
+
+        put_version(
+            &set_disks,
+            bucket,
+            object,
+            &version_id,
+            &inbound_legal_hold_release_opts(&version_id, T_OLD),
+        )
+        .await;
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(
+            info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
+            Some("ON"),
+            "a stale inbound release must not lift a hold applied more recently on this site"
+        );
+        assert_eq!(
+            get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
+            Some(T_LOCAL)
+        );
+    }
+
+    /// A replication write that carries no source decision for a locking
+    /// category (here: tags changed at a source that never held the object)
+    /// must not lift the destination's hold by replacing the metadata
+    /// unjudged; it stays WORM-rejected like a local overwrite.
+    #[tokio::test]
+    async fn inbound_without_legal_hold_timestamp_stays_rejected_on_held_version() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-unjudged-category";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
+
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "k=v".to_string());
+        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_NEW.to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_tagging_timestamp: Some(parse_ts(T_NEW)),
+            replication_retention_timestamp: Some(parse_ts(T_NEW)),
+            replication_legalhold_timestamp: None,
+            ..versioned_opts(&version_id, inbound)
+        };
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        let err = set_disks
+            .put_object(bucket, object, &mut reader, &opts)
+            .await
+            .expect_err("a replication write without the legal-hold source timestamp must stay rejected");
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
+    }
+
+    fn default_retention_snapshot(mode: &'static str) -> Arc<ObjectLockConfigSnapshot> {
+        Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::Configured {
+            config: s3s::dto::ObjectLockConfiguration {
+                object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)),
+                rule: Some(s3s::dto::ObjectLockRule {
+                    default_retention: Some(s3s::dto::DefaultRetention {
+                        mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(mode)),
+                        days: Some(1),
+                        years: None,
+                    }),
+                }),
+            },
+            updated_at: OffsetDateTime::now_utc(),
+        }))
+    }
+
+    /// The bucket default retention locks a version that carries no explicit
+    /// retention keys. A tagging-only authorized replication write carries no
+    /// source retention decision, so it must stay WORM-rejected exactly like
+    /// it does for an explicitly retained version; with the retention source
+    /// timestamp the write passes and LWW judges it.
+    #[tokio::test]
+    async fn inbound_without_retention_timestamp_stays_rejected_under_bucket_default_retention() {
+        for mode in [
+            s3s::dto::ObjectLockRetentionMode::COMPLIANCE,
+            s3s::dto::ObjectLockRetentionMode::GOVERNANCE,
+        ] {
+            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+            let bucket = "lww-locked-default-retention";
+            let object = "object";
+            let version_id = Uuid::new_v4().to_string();
+            make_bucket(&disk_stores, bucket).await;
+            seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
+            let seeded = version_info(&set_disks, bucket, object, &version_id).await;
+            assert!(
+                !seeded.user_defined.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER),
+                "the seeded version must be protected by the bucket default only"
+            );
+
+            let tagging_only = ObjectOptions {
+                object_lock_config_snapshot: Some(default_retention_snapshot(mode)),
+                ..inbound_tagging_opts(&version_id, "site=remote", T_NEW)
+            };
+            let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+            let err = set_disks
+                .put_object(bucket, object, &mut reader, &tagging_only)
+                .await
+                .expect_err("{mode}: a tagging-only replication write must not pass the bucket default retention lock");
+            assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "{mode}: unexpected error: {err}");
+            let info = version_info(&set_disks, bucket, object, &version_id).await;
+            assert_eq!(
+                info.user_tags.as_str(),
+                "site=local",
+                "{mode}: the default-protected version must be untouched"
+            );
+
+            let with_retention_decision = ObjectOptions {
+                replication_retention_timestamp: Some(parse_ts(T_NEW)),
+                ..tagging_only
+            };
+            put_version(&set_disks, bucket, object, &version_id, &with_retention_decision).await;
+            let info = version_info(&set_disks, bucket, object, &version_id).await;
+            assert_eq!(
+                info.user_tags.as_str(),
+                "site=remote",
+                "{mode}: with the retention source timestamp the newer inbound tags win"
+            );
+        }
+    }
+
+    /// The gate runs before the replication bypass, so malformed persisted
+    /// lock metadata still fails closed for an authorized replication write.
+    #[tokio::test]
+    async fn replication_write_on_malformed_lock_metadata_still_fails_closed() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-malformed";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        let mut local = HashMap::new();
+        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "MAYBE".to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        let err = set_disks
+            .put_object(bucket, object, &mut reader, &inbound_legal_hold_release_opts(&version_id, T_NEW))
+            .await
+            .expect_err("malformed persisted lock metadata must fail the replication write closed");
+        assert!(!matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("MAYBE"));
+    }
+
+    /// The bypass is scoped to authorized replication writes: the same
+    /// explicit-version PUT without `replication_request` stays WORM-rejected.
+    #[tokio::test]
+    async fn non_replication_overwrite_of_locked_version_is_still_rejected() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-plain-put";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
+
+        let opts = ObjectOptions {
+            replication_request: false,
+            ..inbound_legal_hold_release_opts(&version_id, T_NEW)
+        };
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        let err = set_disks
+            .put_object(bucket, object, &mut reader, &opts)
+            .await
+            .expect_err("a non-replication overwrite of a locked version must stay rejected");
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
+    }
+}
+
 #[cfg(test)]
 mod inline_put_commit_path_tests {
     use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
@@ -14053,6 +14841,72 @@ mod put_object_tmp_cleanup_tests {
         assert_eq!(body, original_body);
     }
 
+    /// A local PutObjectRetention / PutObjectLegalHold clear persists empty
+    /// lock keys (`parse_object_lock_retention`). The commit-time WORM gate
+    /// must read that as unlocked: an explicit-version PUT (the inbound
+    /// replication transport) and a version delete both have to succeed
+    /// (rustfs/backlog#1953).
+    #[tokio::test]
+    async fn explicit_version_overwrite_and_delete_succeed_after_local_lock_clear() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "put-explicit-version-cleared-lock";
+        let object = "object";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut initial_reader = PutObjReader::from_vec(b"original".to_vec());
+        let initial = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut initial_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("initial version should be written");
+        let version_id = initial
+            .version_id
+            .expect("versioned PUT should return a version ID")
+            .to_string();
+        let version_opts = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.clone()),
+            delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
+            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+            ..Default::default()
+        };
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    eval_metadata: Some(HashMap::from([
+                        (X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), String::new()),
+                        (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), String::new()),
+                        (X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), String::new()),
+                    ])),
+                    ..version_opts.clone()
+                },
+            )
+            .await
+            .expect("cleared lock metadata should be written");
+
+        let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
+        set_disks
+            .put_object(bucket, object, &mut replacement, &version_opts)
+            .await
+            .expect("explicit-version PUT must not be wedged by cleared lock metadata");
+
+        set_disks
+            .delete_object(bucket, object, version_opts)
+            .await
+            .expect("version delete must not be wedged by cleared lock metadata");
+    }
+
     #[tokio::test]
     async fn version_only_copy_checks_the_destination_version_object_lock() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -27,12 +27,12 @@ pub(crate) mod internode {
         NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
         NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
         NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
-        NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN,
-        PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1,
-        PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
-        PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
-        SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
-        WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
+        NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
+        PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
+        PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
+        PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
+        SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
+        WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
     };
 }
 

crates/ecstore/src/store/bucket.rs

@@ -327,7 +327,7 @@ impl ECStore {
     async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
         run_bucket_usage_cleanup(guard, bucket, async {
             crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
-            crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
+            crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
         })
         .await
     }

crates/ecstore/src/store/heal.rs

@@ -109,7 +109,8 @@ impl ECStore {
         }
 
         let mut pool_meta = PoolMeta::default();
-        pool_meta.load_no_lock(metadata_pool.clone()).await?;
+        let replica_state = pool_meta.load_no_lock_from_replicas(self.pools.clone()).await?;
+        replica_state.ensure_write_safe("heal format fence failed")?;
         if pool_meta.pools.len() != self.pools.len()
             || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
                 pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line

crates/ecstore/src/store/mod.rs

@@ -44,7 +44,7 @@ use crate::error::{
 use crate::runtime::global::DISK_RESERVE_FRACTION;
 use crate::runtime::instance::InstanceContext;
 use crate::runtime::sources as runtime_sources;
-use crate::services::rebalance::RebalanceMeta;
+use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
 use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
     /// remain suspended until an operator clears or retries them, so they are
     /// a publication barrier even after the worker has stopped.
     pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
-        if self.scanner_data_movement_active().await {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let _operation_guard = operation_gate.read_owned().await;
+        self.scanner_data_usage_publication_snapshot_blocked().await
+    }
+
+    async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
+        if self.ctx.data_movement_operation_epoch_exhausted() {
+            self.ctx.set_scanner_publication_state(true);
             return true;
         }
-
+        let decommission_cancelers = self.decommission_cancelers.read().await;
+        let decommission_active = decommission_cancelers
+            .iter()
+            .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
         let pool_meta = self.pool_meta.read().await;
-        pool_meta.pools.iter().any(|pool| {
+        let decommission_active = decommission_active
+            || pool_meta.pools.iter().any(|pool| {
+                pool.decommission
+                    .as_ref()
+                    .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
+            });
+        let decommission_terminal = pool_meta.pools.iter().any(|pool| {
             pool.decommission
                 .as_ref()
                 .is_some_and(|info| !info.queued && (info.failed || info.canceled))
-        })
+        });
+        drop(pool_meta);
+
+        let rebalance_active = self
+            .rebalance_meta
+            .read()
+            .await
+            .as_ref()
+            .is_some_and(is_rebalance_conflicting_with_decommission);
+
+        let blocked = decommission_active || decommission_terminal || rebalance_active;
+        self.ctx.set_scanner_publication_state(blocked);
+        blocked
+    }
+
+    /// Admit one short data-usage publication commit under the same
+    /// per-instance gate used by decommission side effects and transitions.
+    /// The epoch is sampled while the read guard is held, so a transition
+    /// cannot cross this admission without waiting for the commit to finish.
+    pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let operation_guard = operation_gate.read_owned().await;
+        let epoch = self.ctx.data_movement_operation_epoch();
+        (operation_guard, epoch)
+    }
+
+    /// Acquire the movement gate and inspect the movement owner once. The
+    /// state inspection is performed after acquiring the read guard so a
+    /// transition cannot update its durable state between the check and the
+    /// publication commit.
+    pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let operation_guard = operation_gate.read_owned().await;
+        if self.ctx.data_movement_operation_epoch_exhausted() {
+            return None;
+        }
+        if self.scanner_data_usage_publication_snapshot_blocked().await {
+            return None;
+        }
+
+        Some((operation_guard, self.ctx.data_movement_operation_epoch()))
+    }
+
+    /// Capture the current publication epoch without holding the movement
+    /// gate across backend I/O. Callers must re-admit the same epoch before a
+    /// mutation commits.
+    pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
+        let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        drop(operation_guard);
+        Some(epoch)
     }
 }
 
@@ -995,6 +1060,85 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let operation_gate = store.ctx.data_movement_operation_gate();
+        let movement_guard = operation_gate.write().await;
+        let pending = {
+            let store = store.clone();
+            tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
+        };
+        tokio::task::yield_now().await;
+        assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
+        drop(movement_guard);
+
+        let (_, epoch) = pending
+            .await
+            .expect("publication admission task should not panic")
+            .expect("idle store should admit publication");
+        assert_eq!(epoch, 0);
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
+        let (_, next_epoch) = store
+            .scanner_data_usage_publication_admission_guard()
+            .await
+            .expect("idle store should admit the next publication");
+        assert_eq!(next_epoch, 1);
+    }
+
+    #[tokio::test]
+    async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let epoch = store
+            .scanner_data_usage_publication_epoch()
+            .await
+            .expect("idle store should expose a publication epoch");
+        assert_eq!(epoch, 0);
+
+        let operation_gate = store.ctx.data_movement_operation_gate();
+        let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
+            .await
+            .expect("epoch capture must not hold the movement gate across backend I/O");
+    }
+
+    #[tokio::test]
+    async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *store.rebalance_meta.write().await = Some(RebalanceMeta {
+            pool_stats: vec![crate::services::rebalance::RebalanceStats {
+                participating: true,
+                info: crate::services::rebalance::RebalanceInfo {
+                    status: crate::services::rebalance::RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+
+        assert!(
+            store.scanner_data_usage_publication_admission_guard().await.is_none(),
+            "active rebalance must fail closed at the storage-owned admission boundary"
+        );
+    }
+
+    #[tokio::test]
+    async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
+
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
+        assert!(store.ctx.data_movement_operation_epoch_exhausted());
+        assert!(
+            store.scanner_data_usage_publication_admission_guard().await.is_none(),
+            "publication must fail closed at the reserved terminal epoch"
+        );
+
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
+        assert!(store.ctx.data_movement_operation_epoch_exhausted());
+        assert!(store.scanner_data_usage_publication_blocked().await);
+    }
+
     // The object graph is the isolation carrier: two ECStore instances holding
     // distinct contexts report independent erasure state through their real
     // `&self` accessors — no cross-contamination.

crates/ecstore/src/store/multipart.rs

@@ -196,6 +196,25 @@ async fn list_pool_multipart_uploads_for_incarnation(
 }
 
 impl ECStore {
+    async fn existing_multipart_pool_order(&self) -> Vec<usize> {
+        // A draining source must not hide a valid UploadID in an active target,
+        // while physical order within each phase preserves fail-closed errors.
+        let mut active = Vec::with_capacity(self.pools.len());
+        let mut draining = Vec::new();
+        for (idx, pool) in self.pools.iter().enumerate() {
+            if self.is_pool_rebalancing(pool.pool_idx).await {
+                continue;
+            }
+            if self.is_suspended(pool.pool_idx).await {
+                draining.push(idx);
+            } else {
+                active.push(idx);
+            }
+        }
+        active.extend(draining);
+        active
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub async fn list_multipart_uploads_for_bucket_incarnation(
         &self,
@@ -290,10 +309,8 @@ impl ECStore {
                 .await;
         }
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
             return match pool
                 .list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts)
                 .await
@@ -353,10 +370,8 @@ impl ECStore {
         let mut common_prefixes = HashSet::new();
         let mut source_truncated = false;
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
             let res = list_pool_multipart_uploads_for_incarnation(
                 pool,
                 bucket,
@@ -523,10 +538,8 @@ impl ECStore {
                 .await;
         }
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
             let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
                 Ok(res) => return Ok(res),
                 Err(err) => {
@@ -586,10 +599,8 @@ impl ECStore {
             return self.pools[0].get_multipart_info(bucket, object, upload_id, opts).await;
         }
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
 
             return match pool.get_multipart_info(bucket, object, upload_id, opts).await {
                 Ok(res) => Ok(res),
@@ -624,10 +635,8 @@ impl ECStore {
             return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
         }
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
 
             let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
                 Ok(_) => return Ok(()),
@@ -685,10 +694,8 @@ impl ECStore {
                 .await;
         }
 
-        for pool in self.pools.iter() {
-            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
-                continue;
-            }
+        for pool_idx in self.existing_multipart_pool_order().await {
+            let pool = &self.pools[pool_idx];
 
             let pool = pool.clone();
             let err = match pool

crates/ecstore/src/store/object.rs

@@ -1203,7 +1203,8 @@ fn inject_batch_delete_pool_errors(
     bucket: &str,
     pool_idx: usize,
     object_names: &[String],
-    result: &mut (Vec<DeletedObject>, Vec<Option<Error>>),
+    deleted: &[DeletedObject],
+    errors: &mut [Option<Error>],
 ) {
     let state = BATCH_DELETE_POOL_ERROR_INJECTION
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -1220,8 +1221,8 @@ fn inject_batch_delete_pool_errors(
         let Some(error) = state.errors.get(object_name) else {
             continue;
         };
-        if result.1[idx].is_none() && result.0[idx].found {
-            result.1[idx] = Some(error.clone());
+        if errors[idx].is_none() && deleted[idx].found {
+            errors[idx] = Some(error.clone());
             state.observed.fetch_add(1, Ordering::AcqRel);
         }
     }
@@ -3194,7 +3195,7 @@ impl ECStore {
 
         // Default return value
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
-        let accounting = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
 
         let mut del_errs = Vec::with_capacity(objects.len());
         for _ in 0..objects.len() {
@@ -3333,11 +3334,12 @@ impl ECStore {
                     .iter()
                     .map(|object| object.object_name.clone())
                     .collect::<Vec<_>>();
-                let result = pool.delete_objects(bucket, pool_objects, pool_opts).await;
+                let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
                 #[cfg(test)]
                 let result = {
                     let mut result = result;
-                    inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result);
+                    let (deleted, errors, _) = &mut result;
+                    inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
                     result
                 };
                 (object_indices, result)
@@ -3347,7 +3349,7 @@ impl ECStore {
         let results = join_all(futures).await;
 
         for idx in 0..del_objects.len() {
-            let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| {
+            let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
                 let pool_object_idx = object_indices.binary_search(&idx).ok()?;
                 Some((&dels[pool_object_idx], &errs[pool_object_idx]))
             });
@@ -3367,6 +3369,12 @@ impl ECStore {
             }
         }
 
+        for (object_indices, (_, _, pool_accounting)) in &results {
+            for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
+                accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
+            }
+        }
+
         #[cfg(test)]
         for (idx, object) in objects.iter().enumerate() {
             if del_errs[idx].is_none() && del_objects[idx].delete_marker {

crates/ecstore/src/store/rebalance.rs

@@ -14,10 +14,15 @@
 
 use super::*;
 use crate::config::storageclass;
+use crate::core::pools::merge_pool_status_refresh;
 use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
 use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _};
 pub(in crate::store) mod support;
+
+const LOG_COMPONENT_ECSTORE: &str = "ecstore";
+const LOG_SUBSYSTEM_POOLS: &str = "pools";
+const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload";
 use support::{
     LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
     rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates, resolve_rebalance_delete_from_all_pools_result,
@@ -684,17 +689,61 @@ impl ECStore {
         )
     }
 
-    pub async fn reload_pool_meta(&self) -> Result<()> {
-        let mut meta = PoolMeta::default();
+    /// Peer reload entry: refreshes in-memory pool metadata from the shared
+    /// persisted snapshot. Returns whether newer state was actually merged so
+    /// callers only trigger missing-worker recovery after a real state change;
+    /// delayed snapshots are merged monotonically and never blind-assigned.
+    pub async fn reload_pool_meta(&self) -> Result<bool> {
+        // Serialize the durable reload with local movement transitions. Loading
+        // before acquiring this gate would allow a stale disk snapshot to
+        // overwrite a newer local transition after the writer commits.
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
+        let mut reloaded = PoolMeta::default();
         resolve_store_rebalance_pool_meta_reload_result(
-            meta.load(self.pools[0].clone(), self.pools.clone()).await,
+            reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
             "reload_pool_meta",
         )?;
 
+        // Lock order: release the decommission_cancelers guard before taking
+        // the pool_meta write guard; neither is held without the movement gate.
+        let active_workers = {
+            let cancelers = self.decommission_cancelers.read().await;
+            cancelers
+                .iter()
+                .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
+                .collect::<Vec<_>>()
+        };
+
+        let incoming_has_pools = !reloaded.pools.is_empty();
         let mut pool_meta = self.pool_meta.write().await;
-        *pool_meta = meta;
-        // *self.pool_meta.write().expect("operation should succeed") = meta;
-        Ok(())
+        let movement_before = pool_meta.clone();
+        let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
+        if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
+            self.ctx.advance_data_movement_operation_epoch();
+        }
+
+        if !merged_newer && !incoming_has_pools {
+            warn!(
+                event = EVENT_POOL_META_RELOAD,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_POOLS,
+                result = "ignored",
+                reason = "missing_metadata",
+                "Peer pool meta reload ignored because persisted metadata is missing"
+            );
+        } else if !merged_newer {
+            debug!(
+                event = EVENT_POOL_META_RELOAD,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_POOLS,
+                result = "ignored",
+                reason = "stale_snapshot",
+                "Peer pool meta reload ignored as a stale snapshot"
+            );
+        }
+
+        Ok(merged_newer)
     }
 
     /// Disk information deduplication function
@@ -861,6 +910,7 @@ mod tests {
     use super::*;
     use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
     use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
+    use crate::core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolStatus};
     use crate::disk::error::DiskError;
     use crate::layout::endpoint::Endpoint;
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
@@ -871,6 +921,7 @@ mod tests {
     use rustfs_config::server_config::KVS;
     use rustfs_filemeta::FileInfo;
     use std::sync::Arc;
+    use time::{Duration as TimeDuration, OffsetDateTime};
     use tokio_util::sync::CancellationToken;
 
     async fn setup_multi_pool_test_store(
@@ -1587,10 +1638,6 @@ mod tests {
     fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
         let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
 
-        let mut data_dir = base.clone();
-        data_dir.data_dir = Some(Uuid::from_u128(2));
-        assert_equal_time_identity_conflict(base.clone(), data_dir);
-
         let mut size = base.clone();
         size.size = 1;
         assert_equal_time_identity_conflict(base.clone(), size);
@@ -1620,6 +1667,58 @@ mod tests {
         );
     }
 
+    #[test]
+    fn resolve_latest_object_info_candidates_accepts_data_movement_rewrites() {
+        let mut source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        source.data_dir = Some(Uuid::from_u128(1));
+
+        let mut target = source.clone();
+        target.data_dir = Some(Uuid::from_u128(2));
+        rustfs_utils::http::insert_str(
+            Arc::make_mut(&mut target.user_defined),
+            rustfs_utils::http::SUFFIX_DATA_MOVED,
+            "true".to_string(),
+        );
+        rustfs_utils::http::insert_str(
+            Arc::make_mut(&mut target.user_defined),
+            rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
+            "0".to_string(),
+        );
+
+        let (info, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(source),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(target.clone()),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("a committed data-movement target must remain readable before source cleanup");
+
+        assert_eq!(idx, 1);
+        assert_eq!(info.data_dir, target.data_dir);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_unmarked_data_dir_conflict() {
+        let mut left = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        left.data_dir = Some(Uuid::from_u128(1));
+
+        let mut right = left.clone();
+        right.data_dir = Some(Uuid::from_u128(2));
+
+        assert_equal_time_identity_conflict(left, right);
+    }
+
     #[test]
     fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
         let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
@@ -2097,4 +2196,282 @@ mod tests {
                 .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3")
         );
     }
+
+    fn reload_test_pool_status(decommission: Option<PoolDecommissionInfo>, last_update: time::OffsetDateTime) -> PoolStatus {
+        PoolStatus {
+            id: 0,
+            cmd_line: "pool-0".to_string(),
+            last_update,
+            decommission,
+        }
+    }
+
+    fn reload_test_pool_meta(pool: PoolStatus) -> PoolMeta {
+        PoolMeta {
+            version: POOL_META_VERSION,
+            pools: vec![pool],
+            dont_save: false,
+        }
+    }
+
+    async fn persist_reload_snapshot(store: &ECStore, snapshot: &PoolMeta) {
+        snapshot
+            .save(store.pools.clone())
+            .await
+            .expect("pool meta snapshot should persist to every pool");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() {
+        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-stale", &[2]).await;
+
+        let stale_time = OffsetDateTime::now_utc();
+        let newer_time = stale_time + TimeDuration::seconds(30);
+        let progressed_states: [(&str, PoolDecommissionInfo); 4] = [
+            (
+                "queued",
+                PoolDecommissionInfo {
+                    queued: true,
+                    start_time: Some(stale_time),
+                    ..Default::default()
+                },
+            ),
+            (
+                "canceled",
+                PoolDecommissionInfo {
+                    canceled: true,
+                    start_time: Some(stale_time),
+                    ..Default::default()
+                },
+            ),
+            (
+                "failed",
+                PoolDecommissionInfo {
+                    failed: true,
+                    start_time: Some(stale_time),
+                    ..Default::default()
+                },
+            ),
+            (
+                "complete",
+                PoolDecommissionInfo {
+                    complete: true,
+                    start_time: Some(stale_time),
+                    ..Default::default()
+                },
+            ),
+        ];
+
+        for (state_label, local_state) in progressed_states {
+            {
+                let mut pool_meta = store.pool_meta.write().await;
+                *pool_meta = reload_test_pool_meta(reload_test_pool_status(Some(local_state.clone()), newer_time));
+            }
+
+            // A delayed peer message carries a snapshot that predates the local progression.
+            let stale_snapshot = reload_test_pool_meta(reload_test_pool_status(
+                Some(PoolDecommissionInfo {
+                    start_time: Some(stale_time),
+                    ..Default::default()
+                }),
+                stale_time,
+            ));
+            persist_reload_snapshot(&store, &stale_snapshot).await;
+
+            let merged_newer = store.reload_pool_meta().await.expect("stale reload should succeed");
+            assert!(
+                !merged_newer,
+                "a delayed reload must not report merged newer state for the {state_label} progression"
+            );
+
+            let pool_meta = store.pool_meta.read().await;
+            let info = pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .expect("local decommission state should survive a stale reload");
+            assert_eq!(info.queued, local_state.queued, "{state_label} queued flag must not roll back");
+            assert_eq!(info.canceled, local_state.canceled, "{state_label} canceled flag must not roll back");
+            assert_eq!(info.failed, local_state.failed, "{state_label} failed flag must not roll back");
+            assert_eq!(info.complete, local_state.complete, "{state_label} complete flag must not roll back");
+            assert_eq!(
+                pool_meta.pools[0].last_update, newer_time,
+                "{state_label} progress timestamp must be kept"
+            );
+        }
+
+        shutdown.cancel();
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn peer_pool_meta_reload_merges_newer_state_and_is_idempotent_on_duplicate_delivery() {
+        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-duplicate", &[2]).await;
+
+        let older_time = OffsetDateTime::now_utc();
+        let newer_time = older_time + TimeDuration::seconds(30);
+
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
+                Some(PoolDecommissionInfo {
+                    items_decommissioned: 1,
+                    ..Default::default()
+                }),
+                older_time,
+            ));
+        }
+        let newer_snapshot = reload_test_pool_meta(reload_test_pool_status(
+            Some(PoolDecommissionInfo {
+                complete: true,
+                items_decommissioned: 10,
+                ..Default::default()
+            }),
+            newer_time,
+        ));
+        persist_reload_snapshot(&store, &newer_snapshot).await;
+
+        let merged_newer = store.reload_pool_meta().await.expect("first reload should succeed");
+        assert!(merged_newer, "a strictly newer persisted snapshot must merge");
+
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
+            assert!(info.complete);
+            assert_eq!(info.items_decommissioned, 10);
+            assert_eq!(pool_meta.pools[0].last_update, newer_time);
+        }
+
+        // Redelivering the same generation must be a no-op.
+        let duplicate_merged = store.reload_pool_meta().await.expect("duplicate reload should succeed");
+        assert!(!duplicate_merged, "a duplicate delivery must not re-apply merged state");
+
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
+            assert!(info.complete);
+            assert_eq!(info.items_decommissioned, 10);
+            assert_eq!(pool_meta.pools[0].last_update, newer_time);
+        }
+
+        shutdown.cancel();
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() {
+        let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await;
+        *store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test(
+            CancellationToken::new(),
+        ))];
+
+        let worker_time = OffsetDateTime::now_utc();
+        let newer_time = worker_time + TimeDuration::seconds(30);
+
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
+                Some(PoolDecommissionInfo {
+                    start_time: Some(worker_time),
+                    items_decommissioned: 10,
+                    bytes_done: 1_024,
+                    ..Default::default()
+                }),
+                worker_time,
+            ));
+        }
+        // Even a strictly newer terminal snapshot must not override a live worker.
+        let newer_terminal_snapshot = reload_test_pool_meta(reload_test_pool_status(
+            Some(PoolDecommissionInfo {
+                complete: true,
+                ..Default::default()
+            }),
+            newer_time,
+        ));
+        persist_reload_snapshot(&store, &newer_terminal_snapshot).await;
+
+        let merged_newer = store
+            .reload_pool_meta()
+            .await
+            .expect("reload under an active worker should succeed");
+        assert!(!merged_newer, "an active local worker must block snapshot replacement");
+
+        let pool_meta = store.pool_meta.read().await;
+        let info = pool_meta.pools[0]
+            .decommission
+            .as_ref()
+            .expect("worker progress should remain");
+        assert!(!info.complete);
+        assert_eq!(info.items_decommissioned, 10);
+        assert_eq!(info.bytes_done, 1_024);
+        assert_eq!(pool_meta.pools[0].last_update, worker_time);
+
+        shutdown.cancel();
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn peer_pool_meta_reload_fails_closed_when_persisted_metadata_is_missing() {
+        let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await;
+
+        let kept_time = OffsetDateTime::now_utc();
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            *pool_meta = reload_test_pool_meta(reload_test_pool_status(
+                Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+                kept_time,
+            ));
+        }
+        // Persist first so the test controls exactly what exists on disk.
+        persist_reload_snapshot(
+            &store,
+            &reload_test_pool_meta(reload_test_pool_status(
+                Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+                kept_time,
+            )),
+        )
+        .await;
+
+        let mut deleted_any = false;
+        for disk_index in 0..2 {
+            let pool_bin_dir = temp_dir
+                .path()
+                .join(format!("pool0-disk{disk_index}"))
+                .join(crate::disk::RUSTFS_META_BUCKET)
+                .join(crate::core::pools::POOL_META_NAME);
+            if pool_bin_dir.exists() {
+                tokio::fs::remove_dir_all(&pool_bin_dir)
+                    .await
+                    .expect("persisted pool metadata object dir should be removable");
+                deleted_any = true;
+            }
+        }
+        // The meta-bucket layout may nest objects per pool; fall back to removing
+        // every pool.bin object directory below the temp root.
+        if !deleted_any {
+            panic!("no pool.bin found under {:?}", temp_dir.path());
+        }
+
+        let merged_newer = store
+            .reload_pool_meta()
+            .await
+            .expect("reload with missing metadata should fail closed, not error");
+        assert!(!merged_newer, "missing persisted metadata must not count as merged state");
+
+        let pool_meta = store.pool_meta.read().await;
+        let info = pool_meta.pools[0]
+            .decommission
+            .as_ref()
+            .expect("missing persisted metadata must not default local state away");
+        assert!(info.complete);
+        assert_eq!(pool_meta.pools[0].last_update, kept_time);
+
+        shutdown.cancel();
+    }
 }

crates/ecstore/src/store/rebalance/support.rs

@@ -214,47 +214,67 @@ fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
     }
 }
 
-/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
-/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
-/// differ for the same object version while the selected winner still carries
-/// the chosen pool's layout. `put_object_reader` is also intentionally
-/// excluded because it is a transient request handle that `ObjectInfo::clone`
-/// drops. Every other ObjectInfo field is part of the production-visible
-/// identity and must agree before the pool index can provide a deterministic
-/// tie-break.
+/// Pool-specific erasure geometry and data-movement rewrites are intentionally
+/// excluded. The selected winner still carries the chosen pool's layout, while
+/// the remaining read-visible fields must agree before the pool index can
+/// provide a deterministic tie-break.
 fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
-    left.bucket == right.bucket
+    let same_read_surface = left.bucket == right.bucket
         && left.name == right.name
-        && left.storage_class == right.storage_class
+        && left.is_dir == right.is_dir
+        && left.restore_ongoing == right.restore_ongoing
+        && left.restore_expires == right.restore_expires
+        && left.is_latest == right.is_latest
+        && left.content_type == right.content_type
+        && left.content_encoding == right.content_encoding
+        && left.num_versions == right.num_versions
+        && left.successor_mod_time == right.successor_mod_time
+        && left.inlined == right.inlined
+        && left.metadata_only == right.metadata_only
+        && left.version_only == right.version_only
+        && left.replication_decision == right.replication_decision;
+
+    if !same_read_surface {
+        return false;
+    }
+
+    let exact_identity = left.storage_class == right.storage_class
         && left.mod_time == right.mod_time
         && left.size == right.size
         && left.actual_size == right.actual_size
-        && left.is_dir == right.is_dir
         && same_user_defined_identity(left, right)
         && left.user_tags == right.user_tags
         && left.version_id == right.version_id
         && left.data_dir == right.data_dir
         && left.delete_marker == right.delete_marker
         && same_transition_identity(left, right)
-        && left.restore_ongoing == right.restore_ongoing
-        && left.restore_expires == right.restore_expires
         && left.parts == right.parts
-        && left.is_latest == right.is_latest
-        && left.content_type == right.content_type
-        && left.content_encoding == right.content_encoding
         && left.expires == right.expires
-        && left.num_versions == right.num_versions
-        && left.successor_mod_time == right.successor_mod_time
         && left.etag == right.etag
-        && left.inlined == right.inlined
-        && left.metadata_only == right.metadata_only
-        && left.version_only == right.version_only
         && left.replication_status_internal == right.replication_status_internal
         && left.replication_status == right.replication_status
         && left.version_purge_status_internal == right.version_purge_status_internal
         && left.version_purge_status == right.version_purge_status
-        && left.replication_decision == right.replication_decision
-        && left.checksum == right.checksum
+        && left.checksum == right.checksum;
+
+    if exact_identity {
+        return true;
+    }
+
+    let left_moved =
+        rustfs_utils::http::get_consistent_str(&left.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED) == Some("true");
+    let right_moved =
+        rustfs_utils::http::get_consistent_str(&right.user_defined, rustfs_utils::http::SUFFIX_DATA_MOVED) == Some("true");
+
+    match (left_moved, right_moved) {
+        (false, false) => false,
+        (false, true) => crate::data_movement::is_equivalent_data_movement_object_identity(left, right, true, true),
+        (true, false) => crate::data_movement::is_equivalent_data_movement_object_identity(right, left, true, true),
+        (true, true) => {
+            crate::data_movement::is_equivalent_data_movement_object_identity(left, right, true, true)
+                || crate::data_movement::is_equivalent_data_movement_object_identity(right, left, true, true)
+        }
+    }
 }
 
 pub(super) fn resolve_latest_object_info_candidates(

crates/filemeta/src/metacache.rs

@@ -21,6 +21,7 @@ use arc_swap::ArcSwapOption;
 use rmp::Marker;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
+use std::collections::{HashMap, HashSet};
 use std::str::from_utf8;
 use std::{
     fmt::Debug,
@@ -37,8 +38,13 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use tokio::spawn;
 use tokio::sync::Mutex;
 use tracing::{debug, warn};
+use uuid::Uuid;
 
 const SLASH_SEPARATOR: &str = "/";
+pub const MAX_META_CACHE_HEAL_CANDIDATES: usize = 1024;
+/// Keep truncation continuations bounded while still giving the scanner a
+/// safe object-level retry for versions that did not fit in the candidate set.
+pub const MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS: usize = 64;
 
 #[derive(Clone, Debug, Default)]
 pub struct MetadataResolutionParams {
@@ -66,6 +72,50 @@ pub struct MetaCacheEntry {
     pub reusable: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum MetaCacheHealCandidateKind {
+    Object,
+    DeleteMarker,
+    UnversionedObject,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct MetaCacheHealCandidate {
+    pub object: String,
+    pub version_id: Option<Uuid>,
+    pub kind: MetaCacheHealCandidateKind,
+    /// Number of raw disk entries that carried this validated version.
+    pub replica_count: usize,
+}
+
+impl MetaCacheHealCandidate {
+    pub fn validated_version(&self) -> Option<Uuid> {
+        match self.kind {
+            MetaCacheHealCandidateKind::Object | MetaCacheHealCandidateKind::DeleteMarker => self.version_id,
+            MetaCacheHealCandidateKind::UnversionedObject => None,
+        }
+    }
+
+    pub fn is_unversioned(&self) -> bool {
+        self.kind == MetaCacheHealCandidateKind::UnversionedObject
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct MetaCacheHealDiscovery {
+    pub candidates: Vec<MetaCacheHealCandidate>,
+    pub unverified_count: usize,
+    pub truncated: bool,
+    /// Object names whose validated version set exceeded the candidate cap.
+    /// The scanner retries these names without a version and with destructive
+    /// healing disabled; this is an explicit bounded continuation, not a
+    /// version claim.
+    pub truncated_objects: Vec<String>,
+    /// Validated candidates beyond the main cap, retained with exact version
+    /// identities so callers never fall back to a latest-version request.
+    pub truncated_candidates: Vec<MetaCacheHealCandidate>,
+}
+
 impl MetaCacheEntry {
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
         let mut wr = Vec::new();
@@ -370,6 +420,185 @@ impl MetaCacheEntries {
         })
     }
 
+    /// Discover validated object/delete-marker versions and safe unversioned
+    /// inspection candidates in the raw entries without applying read quorum.
+    /// This is intentionally separate from [`Self::resolve`]: a sub-quorum
+    /// version is a valid heal target even though it must not participate in
+    /// normal reads or writes.
+    ///
+    /// The validated list is bounded and deduplicated by object, version id,
+    /// and metadata kind; each candidate retains the number of raw disk
+    /// entries that carried it so callers can classify sub-quorum versions.
+    /// Entries whose xl.meta cannot be decoded are counted separately for
+    /// discovery accounting; they never become versionless destructive heal
+    /// requests and do not consume the validated quota. An
+    /// [`MetaCacheHealCandidateKind::UnversionedObject`] is always consumed by
+    /// a non-destructive scanner request.
+    pub fn discover_heal_candidates(&self, bucket: &str, max_candidates: usize) -> MetaCacheHealDiscovery {
+        let limit = max_candidates.min(MAX_META_CACHE_HEAL_CANDIDATES);
+        if limit == 0 || bucket.is_empty() {
+            return MetaCacheHealDiscovery::default();
+        }
+
+        let mut discovery = MetaCacheHealDiscovery {
+            candidates: Vec::<MetaCacheHealCandidate>::with_capacity(limit.min(self.0.len())),
+            unverified_count: 0,
+            truncated: false,
+            truncated_objects: Vec::with_capacity(MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS.min(limit)),
+            truncated_candidates: Vec::new(),
+        };
+        let mut seen: HashMap<(String, Option<Uuid>, MetaCacheHealCandidateKind), usize> =
+            HashMap::with_capacity(limit.min(self.0.len()));
+
+        for entry in self.0.iter().flatten() {
+            if !valid_heal_candidate_name(bucket, entry) {
+                continue;
+            }
+
+            let meta = match FileMeta::load(&entry.metadata) {
+                Ok(meta) => meta,
+                Err(_) => {
+                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                    continue;
+                }
+            };
+            let mut entry_seen = HashSet::new();
+
+            for shallow in meta.versions {
+                let version = match shallow.parse_version_meta() {
+                    Ok(version) if version.valid() => version,
+                    Ok(_) | Err(_) => {
+                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                        continue;
+                    }
+                };
+                if version.free_version() {
+                    continue;
+                }
+
+                let payload_header = version.header();
+                if normalize_version_id(shallow.header.version_id) != normalize_version_id(payload_header.version_id)
+                    || shallow.header.version_type != payload_header.version_type
+                {
+                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                    continue;
+                }
+
+                let (kind, version_id) = match version.version_type {
+                    VersionType::Object
+                        if version.object.is_some() && version.delete_marker.is_none() && version.legacy_object.is_none() =>
+                    {
+                        match version.object.as_ref().and_then(|object| object.version_id) {
+                            Some(id) if !id.is_nil() => (MetaCacheHealCandidateKind::Object, Some(id)),
+                            Some(_) | None => (MetaCacheHealCandidateKind::UnversionedObject, None),
+                        }
+                    }
+                    VersionType::Delete
+                        if version.delete_marker.is_some() && version.object.is_none() && version.legacy_object.is_none() =>
+                    {
+                        let Some(id) = version.delete_marker.as_ref().and_then(|marker| marker.version_id) else {
+                            discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                            continue;
+                        };
+                        if id.is_nil() {
+                            discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                            continue;
+                        }
+                        (MetaCacheHealCandidateKind::DeleteMarker, Some(id))
+                    }
+                    VersionType::Legacy
+                        if version.legacy_object.is_some() && version.object.is_none() && version.delete_marker.is_none() =>
+                    {
+                        let Some(legacy) = version.legacy_object.as_ref() else {
+                            continue;
+                        };
+                        if legacy.version_id.is_empty() {
+                            (MetaCacheHealCandidateKind::UnversionedObject, None)
+                        } else {
+                            let Ok(id) = Uuid::parse_str(&legacy.version_id) else {
+                                discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                                continue;
+                            };
+                            if id.is_nil() {
+                                discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                                continue;
+                            }
+                            (MetaCacheHealCandidateKind::Object, Some(id))
+                        }
+                    }
+                    _ => {
+                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                        continue;
+                    }
+                };
+
+                if normalize_version_id(payload_header.version_id) != version_id {
+                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                    continue;
+                }
+
+                // `all_parts=true` is the trust-boundary check for versioned
+                // candidates. A null/legacy object may still need the old
+                // non-destructive inspection fallback when its part arrays
+                // are parseable but incomplete; never use that fallback for
+                // a candidate carrying a real version id.
+                let file_info = match version.clone().into_fileinfo(bucket, &entry.name, true) {
+                    Ok(file_info) => file_info,
+                    Err(_) if version_id.is_none() && matches!(kind, MetaCacheHealCandidateKind::UnversionedObject) => {
+                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                        match version.into_fileinfo(bucket, &entry.name, false) {
+                            Ok(file_info) => file_info,
+                            Err(_) => continue,
+                        }
+                    }
+                    Err(_) => {
+                        discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                        continue;
+                    }
+                };
+                if file_info.volume != bucket || file_info.name != entry.name {
+                    discovery.unverified_count = discovery.unverified_count.saturating_add(1);
+                    continue;
+                }
+
+                let candidate = MetaCacheHealCandidate {
+                    object: entry.name.clone(),
+                    version_id,
+                    kind,
+                    replica_count: 1,
+                };
+                let key = (candidate.object.clone(), candidate.version_id, candidate.kind.clone());
+                if entry_seen.contains(&key) {
+                    continue;
+                }
+                if let Some(index) = seen.get(&key).copied() {
+                    entry_seen.insert(key);
+                    discovery.candidates[index].replica_count = discovery.candidates[index].replica_count.saturating_add(1);
+                } else if discovery.candidates.len() >= limit {
+                    // Keep the validated candidate list bounded, but retain a
+                    // bounded object-level continuation so the scanner cannot
+                    // silently lose every version of a busy object.
+                    discovery.truncated = true;
+                    if discovery.truncated_objects.len() < MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS
+                        && !discovery.truncated_objects.iter().any(|object| object == &candidate.object)
+                    {
+                        discovery.truncated_objects.push(candidate.object.clone());
+                    }
+                    if discovery.truncated_objects.iter().any(|object| object == &candidate.object) {
+                        discovery.truncated_candidates.push(candidate);
+                    }
+                    continue;
+                } else {
+                    entry_seen.insert(key.clone());
+                    seen.insert(key, discovery.candidates.len());
+                    discovery.candidates.push(candidate);
+                }
+            }
+        }
+
+        discovery
+    }
+
     fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
         if self.0.is_empty() {
             debug!(
@@ -546,6 +775,33 @@ impl MetaCacheEntries {
     }
 }
 
+fn valid_heal_candidate_name(bucket: &str, entry: &MetaCacheEntry) -> bool {
+    if bucket.is_empty()
+        || entry.name.is_empty()
+        || entry.is_dir()
+        || (cfg!(windows) && entry.name.contains('\\'))
+        || entry.name.chars().any(char::is_control)
+    {
+        return false;
+    }
+
+    // Validate raw key components without normalizing them. The scanner maps
+    // accepted keys to filesystem paths later, so dot components and empty
+    // internal components must be rejected before that boundary. A final
+    // empty component is retained for valid keys ending in '/'.
+    let mut components = entry.name.split('/').peekable();
+    while let Some(component) = components.next() {
+        if component == "." || component == ".." || (component.is_empty() && components.peek().is_some()) {
+            return false;
+        }
+    }
+    true
+}
+
+fn normalize_version_id(version_id: Option<Uuid>) -> Option<Uuid> {
+    version_id.filter(|id| !id.is_nil())
+}
+
 #[derive(Debug, Default)]
 pub struct MetaCacheEntriesSortedResult {
     pub entries: Option<MetaCacheEntriesSorted>,
@@ -638,6 +894,7 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
     }
 
     pub async fn close(&mut self) -> Result<()> {
+        self.init().await?;
         rmp::encode::write_bool(&mut self.buf, false).map_err(|e| Error::other(format!("{e:?}")))?;
         self.flush().await?;
         Ok(())
@@ -991,7 +1248,7 @@ impl<T: Clone + Debug + Send + Sync + 'static> Cache<T> {
 mod tests {
     use super::*;
     use crate::test_data::create_real_xlmeta;
-    use crate::{FileMetaVersion, MetaDeleteMarker, TRANSITION_COMPLETE};
+    use crate::{FileMetaVersion, MetaDeleteMarker, MetaObjectV1, MetaObjectV1Erasure, MetaObjectV1Stat, TRANSITION_COMPLETE};
     use std::collections::HashMap;
     use std::io::Cursor;
     use std::sync::{
@@ -1029,6 +1286,16 @@ mod tests {
         assert_eq!(objs, nobjs);
     }
 
+    #[tokio::test]
+    async fn empty_writer_emits_a_valid_stream() {
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.close().await.expect("empty stream should close");
+
+        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
+        assert!(reader.read_all().await.expect("empty stream should decode").is_empty());
+    }
+
     fn corrupt_stream_with_metadata_len(len_marker: u8, len: u32) -> Vec<u8> {
         let mut data = Vec::new();
         rmp::encode::write_u8(&mut data, METACACHE_STREAM_VERSION).unwrap();
@@ -1592,6 +1859,381 @@ mod tests {
         );
     }
 
+    #[test]
+    fn discover_heal_candidates_keeps_sub_quorum_versions_and_deduplicates() {
+        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entries = MetaCacheEntries(vec![
+            Some(metacache_entry_single_version(1, now, "one")),
+            Some(metacache_entry_single_version(2, now, "two")),
+            Some(metacache_entry_single_version(2, now, "two")),
+            Some(metacache_entry_single_version(3, now, "three")),
+        ]);
+
+        let discovery = entries.discover_heal_candidates("bucket", 16);
+        let ids: std::collections::HashSet<Uuid> = discovery
+            .candidates
+            .iter()
+            .filter_map(|candidate| candidate.version_id)
+            .collect();
+        assert_eq!(
+            ids,
+            [Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)]
+                .into_iter()
+                .collect()
+        );
+        assert_eq!(discovery.candidates.len(), 3, "duplicate tied versions must be emitted once");
+        assert_eq!(
+            discovery
+                .candidates
+                .iter()
+                .find(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
+                .expect("duplicate version should be discovered")
+                .replica_count,
+            2
+        );
+    }
+
+    #[test]
+    fn discover_heal_candidates_does_not_count_duplicate_versions_within_one_entry() {
+        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut meta = FileMeta::load(&metacache_entry_single_version(1, now, "duplicate").metadata)
+            .expect("duplicate fixture should decode");
+        meta.versions.push(meta.versions[0].clone());
+        let entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: meta.marshal_msg().expect("duplicate metadata should marshal"),
+            cached: Some(meta),
+            reusable: false,
+        };
+
+        let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 16);
+        let candidate = discovery
+            .candidates
+            .iter()
+            .find(|candidate| candidate.version_id == Some(Uuid::from_u128(1)))
+            .expect("duplicate fixture should be discovered");
+        assert_eq!(candidate.replica_count, 1);
+    }
+
+    #[test]
+    fn discover_heal_candidates_covers_divergent_quorum_boundaries_n2_n4_n6() {
+        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+
+        for (disk_count, quorum) in [(2usize, 1usize), (4, 2), (6, 3)] {
+            let target_id = Uuid::from_u128(0x1000 + disk_count as u128);
+            for target_replicas in [quorum.saturating_sub(1), quorum, quorum + 1] {
+                let entries = (0..disk_count)
+                    .map(|disk| {
+                        let version_id = if disk < target_replicas {
+                            target_id
+                        } else {
+                            Uuid::from_u128(0x2000 + disk as u128)
+                        };
+                        Some(metacache_entry_single_version(version_id.as_u128(), now, "divergent"))
+                    })
+                    .collect();
+                let discovery = MetaCacheEntries(entries).discover_heal_candidates("bucket", 32);
+                let target = discovery
+                    .candidates
+                    .iter()
+                    .find(|candidate| candidate.version_id == Some(target_id));
+                assert_eq!(target.is_some(), target_replicas > 0, "N={disk_count}, replicas={target_replicas}");
+                if let Some(target) = target {
+                    assert_eq!(target.replica_count, target_replicas);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn discover_heal_candidates_separates_delete_markers_and_preserves_unversioned_objects() {
+        let mut marker_meta = FileMeta::new();
+        marker_meta
+            .add_version(FileInfo {
+                volume: "bucket".to_string(),
+                name: "object".to_string(),
+                version_id: Some(Uuid::from_u128(99)),
+                deleted: true,
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
+                ..Default::default()
+            })
+            .expect("delete marker should be added");
+        let marker = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: marker_meta.marshal_msg().expect("delete marker metadata should marshal"),
+            cached: Some(marker_meta),
+            reusable: false,
+        };
+
+        let unversioned_entry = metacache_entry_with_mod_time(
+            OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"),
+            "unversioned",
+        );
+        let discovery = MetaCacheEntries(vec![Some(marker), Some(unversioned_entry)]).discover_heal_candidates("bucket", 16);
+        assert!(discovery.candidates.iter().any(|candidate| {
+            candidate.kind == MetaCacheHealCandidateKind::DeleteMarker && candidate.version_id == Some(Uuid::from_u128(99))
+        }));
+        assert!(discovery.candidates.iter().any(|candidate| {
+            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
+        }));
+    }
+
+    #[test]
+    fn discover_heal_candidates_rejects_delete_markers_without_ids() {
+        let mut marker_meta = FileMeta::new();
+        marker_meta
+            .add_version(FileInfo {
+                volume: "bucket".to_string(),
+                name: "object".to_string(),
+                deleted: true,
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
+                ..Default::default()
+            })
+            .expect("nil delete marker should be added");
+        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: marker_meta.marshal_msg().expect("nil marker metadata should marshal"),
+            cached: Some(marker_meta),
+            reusable: false,
+        })])
+        .discover_heal_candidates("bucket", 16);
+
+        assert!(
+            !discovery
+                .candidates
+                .iter()
+                .any(|candidate| candidate.kind == MetaCacheHealCandidateKind::DeleteMarker)
+        );
+        assert!(discovery.unverified_count >= 1);
+    }
+
+    #[test]
+    fn discover_heal_candidates_skips_free_versions() {
+        let object_id = Uuid::from_u128(100);
+        let free_id = Uuid::from_u128(101);
+        let mut meta = FileMeta::new();
+        meta.add_version(FileInfo {
+            volume: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(object_id),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_version_id: Some(Uuid::from_u128(102)),
+            transition_tier: "WARM".to_string(),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        })
+        .expect("transitioned object should be added");
+        let mut delete = FileInfo {
+            volume: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(object_id),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        delete.set_tier_free_version_id(&free_id.to_string());
+        meta.delete_version(&delete).expect("free version should be persisted");
+
+        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: meta.marshal_msg().expect("free version metadata should marshal"),
+            cached: Some(meta),
+            reusable: false,
+        })])
+        .discover_heal_candidates("bucket", 16);
+        assert!(discovery.candidates.is_empty());
+    }
+
+    #[test]
+    fn discover_heal_candidates_preserves_unversioned_legacy_object() {
+        let legacy = MetaObjectV1 {
+            version: "1.0.1".to_string(),
+            format: "xl".to_string(),
+            stat: MetaObjectV1Stat {
+                size: 1,
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")),
+                name: "object".to_string(),
+                ..Default::default()
+            },
+            erasure: MetaObjectV1Erasure {
+                data_blocks: 4,
+                parity_blocks: 2,
+                index: 1,
+                distribution: vec![1, 2, 3, 4, 5, 6],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let version = FileMetaVersion {
+            version_type: VersionType::Legacy,
+            legacy_object: Some(legacy),
+            ..Default::default()
+        };
+        let mut meta = FileMeta::new();
+        meta.versions
+            .push(FileMetaShallowVersion::try_from(version).expect("legacy metadata should marshal"));
+        let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: meta.marshal_msg().expect("legacy metadata should marshal"),
+            cached: Some(meta),
+            reusable: false,
+        })])
+        .discover_heal_candidates("bucket", 16);
+        assert!(discovery.candidates.iter().any(|candidate| {
+            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
+        }));
+    }
+
+    #[test]
+    fn discover_heal_candidates_rejects_nil_and_malformed_metadata_and_is_bounded() {
+        let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut nil = metacache_entry_single_version(1, now, "nil");
+        let mut nil_meta = FileMeta::load(&nil.metadata).expect("nil fixture should decode");
+        let mut nil_version = nil_meta.versions[0]
+            .parse_version_meta()
+            .expect("nil fixture version should decode");
+        nil_version.object.as_mut().expect("object fixture").version_id = Some(Uuid::nil());
+        nil_meta.versions[0] = FileMetaShallowVersion::try_from(nil_version).expect("nil fixture should marshal");
+        nil.metadata = nil_meta.marshal_msg().expect("nil fixture metadata should marshal");
+
+        let mut mismatched = metacache_entry_single_version(2, now, "mismatched");
+        let mut mismatched_meta = FileMeta::load(&mismatched.metadata).expect("mismatched fixture should decode");
+        mismatched_meta.versions[0].header.version_id = Some(Uuid::from_u128(200));
+        mismatched.metadata = mismatched_meta.marshal_msg().expect("mismatched metadata should marshal");
+
+        let mut short_parts = metacache_entry_single_version(3, now, "short-parts");
+        let mut short_parts_meta = FileMeta::load(&short_parts.metadata).expect("short-parts fixture should decode");
+        let mut short_parts_version = short_parts_meta.versions[0]
+            .parse_version_meta()
+            .expect("short-parts fixture version should decode");
+        let object = short_parts_version.object.as_mut().expect("object fixture");
+        object.part_numbers = vec![1];
+        object.part_actual_sizes = vec![1];
+        object.part_sizes.clear();
+        short_parts_meta.versions[0] = FileMetaShallowVersion::try_from(short_parts_version).expect("short-parts should marshal");
+        short_parts.metadata = short_parts_meta.marshal_msg().expect("short-parts metadata should marshal");
+
+        let mut short_unversioned = metacache_entry_with_mod_time(now, "short-unversioned");
+        let mut short_unversioned_meta =
+            FileMeta::load(&short_unversioned.metadata).expect("short-unversioned fixture should decode");
+        let mut short_unversioned_version = short_unversioned_meta.versions[0]
+            .parse_version_meta()
+            .expect("short-unversioned version should decode");
+        let unversioned_object = short_unversioned_version.object.as_mut().expect("unversioned object fixture");
+        unversioned_object.part_numbers = vec![1];
+        unversioned_object.part_actual_sizes = vec![1];
+        unversioned_object.part_sizes.clear();
+        short_unversioned_meta.versions[0] =
+            FileMetaShallowVersion::try_from(short_unversioned_version).expect("short-unversioned should marshal");
+        short_unversioned.metadata = short_unversioned_meta
+            .marshal_msg()
+            .expect("short-unversioned metadata should marshal");
+
+        let mut malformed = nil.clone();
+        malformed.name = "malformed".to_string();
+        malformed.metadata = vec![1, 2, 3];
+
+        let entries = MetaCacheEntries(
+            std::iter::once(Some(nil))
+                .chain(std::iter::once(Some(mismatched)))
+                .chain(std::iter::once(Some(short_parts)))
+                .chain(std::iter::once(Some(short_unversioned)))
+                .chain(std::iter::once(Some(malformed)))
+                .chain((0..32).map(|id| Some(metacache_entry_single_version(id + 10, now, "bounded"))))
+                .collect(),
+        );
+        let discovery = entries.discover_heal_candidates("bucket", 5);
+        assert!(discovery.candidates.len() <= 5);
+        assert!(discovery.truncated, "bounded discovery must expose dropped candidates");
+        assert!(
+            discovery
+                .truncated_candidates
+                .iter()
+                .all(|candidate| candidate.version_id.is_some()),
+            "overflow candidates must retain exact version identities"
+        );
+        assert!(
+            discovery.truncated_objects.iter().any(|object| object == "object"),
+            "bounded discovery must expose an object-level safe continuation"
+        );
+        assert!(
+            !discovery
+                .candidates
+                .iter()
+                .any(|candidate| candidate.version_id == Some(Uuid::nil()))
+        );
+        assert!(
+            !discovery
+                .candidates
+                .iter()
+                .any(|candidate| candidate.version_id == Some(Uuid::from_u128(2)))
+        );
+        assert!(
+            !discovery
+                .candidates
+                .iter()
+                .any(|candidate| candidate.version_id == Some(Uuid::from_u128(3)))
+        );
+        assert!(discovery.candidates.iter().any(|candidate| {
+            candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none()
+        }));
+        assert!(
+            discovery.unverified_count >= 1,
+            "malformed and rejected metadata must remain observable during discovery"
+        );
+
+        for invalid_name in [
+            "../object",
+            "./object",
+            "object/../other",
+            "object//name",
+            "object\u{0001}name",
+            "object\0name",
+        ] {
+            let mut entry = metacache_entry_single_version(400, now, invalid_name);
+            entry.name = invalid_name.to_string();
+            let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
+            assert!(
+                discovery.candidates.is_empty(),
+                "invalid key should not become a heal candidate: {invalid_name:?}"
+            );
+        }
+
+        #[cfg(windows)]
+        {
+            let mut entry = metacache_entry_single_version(400, now, "object\\name");
+            entry.name = "object\\name".to_string();
+            assert!(
+                MetaCacheEntries(vec![Some(entry)])
+                    .discover_heal_candidates("bucket", 5)
+                    .candidates
+                    .is_empty(),
+                "backslash is a path separator on Windows"
+            );
+        }
+
+        #[cfg(not(windows))]
+        {
+            let mut entry = metacache_entry_single_version(400, now, "object\\name");
+            entry.name = "object\\name".to_string();
+            assert_eq!(
+                MetaCacheEntries(vec![Some(entry)])
+                    .discover_heal_candidates("bucket", 5)
+                    .candidates
+                    .len(),
+                1,
+                "backslash is object-key data on Unix"
+            );
+        }
+
+        for valid_name in ["trailing/", "prefix/object"] {
+            let mut entry = metacache_entry_single_version(401, now, valid_name);
+            entry.name = valid_name.to_string();
+            let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5);
+            assert_eq!(discovery.candidates.len(), 1, "raw S3 key should remain opaque: {valid_name:?}");
+        }
+    }
+
     #[test]
     fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() {
         let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");

crates/heal/Cargo.toml

@@ -91,6 +91,7 @@ metrics = { workspace = true }
 base64 = { workspace = true }
 bytes = { workspace = true }
 crc-fast = { workspace = true }
+sha2 = { workspace = true }
 
 [dev-dependencies]
 serde_json = { workspace = true, features = ["raw_value"] }

crates/heal/src/heal/erasure_healer.rs

@@ -13,10 +13,10 @@
 // limitations under the License.
 
 use crate::heal::{
-    progress::HealProgress,
+    progress::{HealProgress, add_bytes, increment_counter},
     resume::{
-        CheckpointManager, ReplacementTargetIdentity, ResumeManager, ResumeUtils, compose_key,
-        replacement_target_identities_match,
+        CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
+        ResumeUtils, compose_key, replacement_target_identities_match,
     },
     storage::{HealStorageAPI, next_heal_listing_token},
     task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample},
@@ -373,6 +373,11 @@ impl ErasureSetHealer {
         set_disk_id: &str,
         buckets: &[String],
     ) -> Result<(ResumeManager, CheckpointManager)> {
+        if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await {
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Resume task {task_id} has a blocked checkpoint"),
+            });
+        }
         // check if resume state exists
         let has_resume_state = if self.replacement_task_id.is_some() {
             ResumeManager::has_replacement_intent(&self.disk, task_id).await
@@ -410,6 +415,9 @@ impl ErasureSetHealer {
                 && state.successful_objects == 0
                 && state.failed_objects == 0
                 && state.skipped_objects == 0
+                && state.skipped_new_versions == 0
+                && state.skipped_ilm_expired == 0
+                && state.processed_bytes == 0
             {
                 // schedule_retry persists the authoritative resume reset before
                 // resetting the checkpoint. Reapply the checkpoint reset after
@@ -474,6 +482,23 @@ impl ErasureSetHealer {
 
         // 2. initialize progress
         self.initialize_progress(buckets, &state).await;
+        let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
+            let baseline = self.progress.read().await;
+            (
+                baseline.baseline_known,
+                baseline.objects_total_count,
+                baseline.objects_total_size,
+                baseline.baseline_generation,
+            )
+        };
+        if baseline_known {
+            resume_manager
+                .set_progress_baseline(baseline_count, baseline_size, baseline_generation)
+                .await?;
+            checkpoint_manager
+                .set_progress_baseline(baseline_count, baseline_size, baseline_generation)
+                .await?;
+        }
 
         // 3. continue from checkpoint
         let current_bucket_index = checkpoint.current_bucket_index;
@@ -483,12 +508,66 @@ impl ErasureSetHealer {
         let mut successful_objects = state.successful_objects;
         let mut failed_objects = state.failed_objects;
         let mut skipped_objects = state.skipped_objects;
+        let checkpoint_has_progress = checkpoint.baseline_known
+            || checkpoint.successful_objects > 0
+            || checkpoint.failed_object_count > 0
+            || checkpoint.skipped_object_count > 0
+            || checkpoint.skipped_new_versions > 0
+            || checkpoint.skipped_ilm_expired > 0
+            || checkpoint.processed_bytes > 0
+            || checkpoint.total_objects > 0
+            || checkpoint.total_bytes > 0
+            || checkpoint.baseline_generation.is_some()
+            || checkpoint.counter_unknown;
+        let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
+        let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
+        if checkpoint_has_progress {
+            successful_objects = checkpoint.successful_objects;
+            failed_objects = checkpoint.failed_object_count;
+            skipped_objects = checkpoint.skipped_object_count;
+            let restored_processed_objects = successful_objects
+                .checked_add(failed_objects)
+                .and_then(|value| value.checked_add(skipped_objects))
+                .and_then(|value| value.checked_add(checkpoint.skipped_new_versions))
+                .and_then(|value| value.checked_add(checkpoint.skipped_ilm_expired));
+            let checkpoint_counter_overflow = restored_processed_objects.is_none();
+            restored_counter_unknown |= checkpoint_counter_overflow;
+            processed_objects = restored_processed_objects.unwrap_or(u64::MAX);
+            let mut progress = self.progress.write().await;
+            progress.objects_scanned = processed_objects;
+            progress.objects_healed = successful_objects;
+            progress.objects_failed = failed_objects;
+            progress.skipped_objects = skipped_objects;
+            progress.skipped_new_versions = checkpoint.skipped_new_versions;
+            progress.skipped_ilm_expired = checkpoint.skipped_ilm_expired;
+            if checkpoint.baseline_known && !checkpoint_generation_mismatch {
+                progress.objects_total_count = checkpoint.total_objects;
+                progress.objects_total_size = checkpoint.total_bytes;
+                progress.baseline_generation = checkpoint.baseline_generation;
+                progress.baseline_known = true;
+            }
+            progress.bytes_processed = checkpoint.processed_bytes;
+            progress.counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
+            progress.refresh_progress_percentage();
+            if checkpoint_generation_mismatch || checkpoint_counter_overflow || progress.counter_unknown {
+                progress.mark_unknown();
+            }
+        }
+        if checkpoint_generation_mismatch {
+            restored_counter_unknown = true;
+        }
+        if restored_counter_unknown {
+            checkpoint_manager.mark_counter_unknown().await?;
+            resume_manager.mark_counter_unknown().await?;
+        }
         let mut failed_buckets = 0u64;
 
         // 4. process remaining buckets
         for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) {
             // check if completed
             if state.completed_buckets.contains(bucket) {
+                checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
+                current_object_index = 0;
                 continue;
             }
 
@@ -516,13 +595,42 @@ impl ErasureSetHealer {
                 return bucket_result;
             }
 
-            // update checkpoint position
-            checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
-
             // update progress
-            resume_manager
-                .update_progress(processed_objects, successful_objects, failed_objects, skipped_objects)
+            let progress_snapshot = self.progress.read().await;
+            let bytes_processed = progress_snapshot.bytes_processed;
+            let skipped_new_versions = progress_snapshot.skipped_new_versions;
+            let skipped_ilm_expired = progress_snapshot.skipped_ilm_expired;
+            let counter_unknown = progress_snapshot.counter_unknown;
+            drop(progress_snapshot);
+            // The checkpoint is the recovery authority for object progress.
+            // Publish its counters and fence before the resume summary so a
+            // crash between the two stores cannot make recovery select newer
+            // summary bytes with an older checkpoint ledger.
+            if counter_unknown {
+                checkpoint_manager.mark_counter_unknown().await?;
+            }
+            checkpoint_manager
+                .update_progress(successful_objects, failed_objects, skipped_objects, bytes_processed)
                 .await?;
+            checkpoint_manager
+                .set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
+                .await?;
+            checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
+            resume_manager
+                .update_progress_with_bytes(
+                    processed_objects,
+                    successful_objects,
+                    failed_objects,
+                    skipped_objects,
+                    bytes_processed,
+                )
+                .await?;
+            resume_manager
+                .set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
+                .await?;
+            if counter_unknown {
+                resume_manager.mark_counter_unknown().await?;
+            }
 
             // check cancel status
             if self.cancel_token.is_cancelled() {
@@ -542,6 +650,7 @@ impl ErasureSetHealer {
             match bucket_result {
                 Ok(_) => {
                     resume_manager.complete_bucket(bucket).await?;
+                    checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
                     debug!(
                         target: "rustfs::heal::erasure_healer",
                         event = EVENT_HEAL_ERASURE_BUCKET_STATE,
@@ -567,7 +676,9 @@ impl ErasureSetHealer {
                         error = %e,
                         "Erasure set bucket heal failed"
                     );
-                    // continue to next bucket, do not interrupt the whole process
+                    // A single durable cursor and ledger cannot safely preserve
+                    // this bucket while processing a later one.
+                    break;
                 }
             }
 
@@ -775,20 +886,49 @@ impl ErasureSetHealer {
 
                 // Per-version dedup identity — the single canonical key.
                 let key = compose_key(&item.name, item.version_id.as_deref());
-                if checkpoint.processed_objects.contains(&key) || checkpoint.skipped_objects.contains(&key) {
+                if checkpoint.processed_objects.contains(&key)
+                    || checkpoint.failed_objects.contains(&key)
+                    || checkpoint.skipped_objects.contains(&key)
+                {
                     continue;
                 }
 
                 if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
-                    checkpoint_manager.add_processed_object(key).await?;
-                    *processed_objects = processed_objects.saturating_add(1);
+                    let counter_ok = increment_counter(processed_objects);
                     completed_in_page = completed_in_page.saturating_add(1);
                     counter!("rustfs_heal_skipped_new_versions_total").increment(1);
-                    {
+                    let (outcome_record, counter_unknown) = {
                         let mut progress = self.progress.write().await;
                         progress.record_skipped_new_version();
                         progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
-                        progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
+                        progress.update_object_progress(
+                            *processed_objects,
+                            *successful_objects,
+                            *failed_objects,
+                            *skipped_objects,
+                            bytes_processed,
+                        );
+                        if !counter_ok {
+                            progress.mark_unknown();
+                        }
+                        (
+                            CheckpointObjectOutcomeRecord {
+                                object: key,
+                                outcome: CheckpointObjectOutcome::Processed,
+                                successful: progress.objects_healed,
+                                failed: progress.objects_failed,
+                                skipped: progress.skipped_objects,
+                                bytes: progress.bytes_processed,
+                                skipped_new_versions: progress.skipped_new_versions,
+                                skipped_ilm_expired: progress.skipped_ilm_expired,
+                                counter_unknown: progress.counter_unknown,
+                            },
+                            progress.counter_unknown,
+                        )
+                    };
+                    checkpoint_manager.record_object_outcome(outcome_record).await?;
+                    if counter_unknown {
+                        resume_manager.mark_counter_unknown().await?;
                     }
                     debug!(
                         target: "rustfs::heal::erasure_healer",
@@ -820,15 +960,41 @@ impl ErasureSetHealer {
                         )
                         .await?
                 {
-                    checkpoint_manager.add_processed_object(key).await?;
-                    *processed_objects = processed_objects.saturating_add(1);
+                    let counter_ok = increment_counter(processed_objects);
                     completed_in_page = completed_in_page.saturating_add(1);
                     counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
-                    {
+                    let (outcome_record, counter_unknown) = {
                         let mut progress = self.progress.write().await;
                         progress.record_skipped_ilm_expired();
                         progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
-                        progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
+                        progress.update_object_progress(
+                            *processed_objects,
+                            *successful_objects,
+                            *failed_objects,
+                            *skipped_objects,
+                            bytes_processed,
+                        );
+                        if !counter_ok {
+                            progress.mark_unknown();
+                        }
+                        (
+                            CheckpointObjectOutcomeRecord {
+                                object: key,
+                                outcome: CheckpointObjectOutcome::Processed,
+                                successful: progress.objects_healed,
+                                failed: progress.objects_failed,
+                                skipped: progress.skipped_objects,
+                                bytes: progress.bytes_processed,
+                                skipped_new_versions: progress.skipped_new_versions,
+                                skipped_ilm_expired: progress.skipped_ilm_expired,
+                                counter_unknown: progress.counter_unknown,
+                            },
+                            progress.counter_unknown,
+                        )
+                    };
+                    checkpoint_manager.record_object_outcome(outcome_record).await?;
+                    if counter_unknown {
+                        resume_manager.mark_counter_unknown().await?;
                     }
                     debug!(
                         target: "rustfs::heal::erasure_healer",
@@ -954,11 +1120,11 @@ impl ErasureSetHealer {
 
             while let Some((key, object, version_id, result)) = page_tasks.next().await {
                 let (object_size, result) = result;
-                match result {
+                let mut telemetry_unknown = false;
+                let checkpoint_outcome = match result {
                     Ok(true) => {
-                        *successful_objects += 1;
-                        bytes_processed = bytes_processed.saturating_add(object_size);
-                        checkpoint_manager.add_processed_object(key).await?;
+                        telemetry_unknown |= !increment_counter(successful_objects);
+                        telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
                         debug!(
                             target: "rustfs::heal::erasure_healer",
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -971,11 +1137,11 @@ impl ErasureSetHealer {
                             state = "healed",
                             "Erasure set object healed"
                         );
+                        CheckpointObjectOutcome::Processed
                     }
                     Ok(false) => {
-                        checkpoint_manager.add_processed_object(key).await?;
-                        *successful_objects += 1;
-                        bytes_processed = bytes_processed.saturating_add(object_size);
+                        telemetry_unknown |= !increment_counter(successful_objects);
+                        telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
                         debug!(
                             target: "rustfs::heal::erasure_healer",
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -988,12 +1154,12 @@ impl ErasureSetHealer {
                             state = "missing_treated_as_ok",
                             "Erasure set missing object treated as ok"
                         );
+                        CheckpointObjectOutcome::Processed
                     }
                     Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
                     Err(Error::TransientSkip { message }) => {
-                        *skipped_objects += 1;
-                        bytes_processed = bytes_processed.saturating_add(object_size);
-                        checkpoint_manager.add_skipped_object(key).await?;
+                        telemetry_unknown |= !increment_counter(skipped_objects);
+                        telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
                         demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
                             component = LOG_COMPONENT_HEAL,
@@ -1006,11 +1172,11 @@ impl ErasureSetHealer {
                             error = %message,
                             "Erasure set object heal skipped due to transient error"
                         });
+                        CheckpointObjectOutcome::Skipped
                     }
                     Err(err) => {
-                        *failed_objects += 1;
-                        bytes_processed = bytes_processed.saturating_add(object_size);
-                        checkpoint_manager.add_failed_object(key).await?;
+                        telemetry_unknown |= !increment_counter(failed_objects);
+                        telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
                         demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
                             event = EVENT_HEAL_ERASURE_OBJECT_STATE,
                             component = LOG_COMPONENT_HEAL,
@@ -1023,15 +1189,43 @@ impl ErasureSetHealer {
                             error = %err,
                             "Erasure set object heal failed"
                         });
+                        CheckpointObjectOutcome::Failed
                     }
-                }
+                };
 
-                *processed_objects += 1;
+                telemetry_unknown |= !increment_counter(processed_objects);
                 completed_in_page += 1;
-                {
+                let (outcome_record, counter_unknown) = {
                     let mut progress = self.progress.write().await;
                     progress.set_current_object(Some(format!("{bucket}/{object}")));
-                    progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
+                    progress.update_object_progress(
+                        *processed_objects,
+                        *successful_objects,
+                        *failed_objects,
+                        *skipped_objects,
+                        bytes_processed,
+                    );
+                    if telemetry_unknown {
+                        progress.mark_unknown();
+                    }
+                    (
+                        CheckpointObjectOutcomeRecord {
+                            object: key,
+                            outcome: checkpoint_outcome,
+                            successful: progress.objects_healed,
+                            failed: progress.objects_failed,
+                            skipped: progress.skipped_objects,
+                            bytes: progress.bytes_processed,
+                            skipped_new_versions: progress.skipped_new_versions,
+                            skipped_ilm_expired: progress.skipped_ilm_expired,
+                            counter_unknown: progress.counter_unknown,
+                        },
+                        progress.counter_unknown,
+                    )
+                };
+                checkpoint_manager.record_object_outcome(outcome_record).await?;
+                if counter_unknown {
+                    resume_manager.mark_counter_unknown().await?;
                 }
 
                 if completed_in_page.is_multiple_of(100) {
@@ -1041,16 +1235,22 @@ impl ErasureSetHealer {
 
             *current_object_index = global_obj_idx;
 
-            // Persist the authoritative cursor FIRST (points at the next page
-            // boundary), then prune the per-version dedup sets. Both are
-            // idempotent under crash: heal_object re-heals safely.
-            let next_cursor = if is_truncated { next_token.clone() } else { None };
-            resume_manager.set_resume_cursor(next_cursor.clone()).await?;
-            checkpoint_manager.complete_page(bucket_index, *current_object_index).await?;
+            // Persist the checkpoint ledger and page position before exposing
+            // the next resume cursor. A crash before cursor publication keeps
+            // the page identities available for exact-once replay.
+            checkpoint_manager.advance_page(bucket_index, *current_object_index).await?;
             // Check if there are more pages
             if !is_truncated {
                 break;
             }
+            continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
+            if continuation_token.is_none() {
+                // A truncated page without a continuation token is terminal.
+                // Retain its ledger until bucket completion is durable.
+                break;
+            }
+            resume_manager.set_resume_cursor(continuation_token.clone()).await?;
+            checkpoint_manager.prune_completed_page().await?;
 
             // Anti-loop guard: an empty page reported as truncated cannot advance
             // the cursor (there is no last identity to move past), so treat it as a
@@ -1069,12 +1269,6 @@ impl ErasureSetHealer {
                 )));
             }
             previous_page_last = page_last;
-
-            continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
-            if continuation_token.is_none() {
-                // Truncated but no continuation token: treat as end of listing.
-                break;
-            }
         }
 
         Ok(())
@@ -1083,10 +1277,66 @@ impl ErasureSetHealer {
     /// initialize progress tracking
     async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) {
         let mut progress = self.progress.write().await;
-        progress.objects_scanned = state.total_objects;
+        let existing_baseline = (
+            progress.objects_total_count,
+            progress.objects_total_size,
+            progress.baseline_generation,
+            progress.progress_state,
+            progress.baseline_known,
+        );
+        let baseline_generation_mismatch =
+            state.baseline_known && existing_baseline.4 && state.baseline_generation != existing_baseline.2;
+        let use_persisted_baseline = state.baseline_known && !baseline_generation_mismatch;
+        progress.objects_scanned = state.processed_objects;
         progress.objects_healed = state.successful_objects;
         progress.objects_failed = state.failed_objects;
-        progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
+        progress.skipped_objects = state.skipped_objects;
+        progress.skipped_new_versions = state.skipped_new_versions;
+        progress.skipped_ilm_expired = state.skipped_ilm_expired;
+        progress.bytes_processed = state.processed_bytes;
+        progress.counter_unknown = state.counter_unknown;
+        if use_persisted_baseline
+            || existing_baseline.0 > 0
+            || existing_baseline.1 > 0
+            || existing_baseline.2.is_some()
+            || existing_baseline.4
+        {
+            progress.objects_total_count = if use_persisted_baseline {
+                state.total_objects
+            } else {
+                existing_baseline.0
+            };
+            progress.objects_total_size = if use_persisted_baseline {
+                state.total_bytes
+            } else {
+                existing_baseline.1
+            };
+            progress.baseline_generation = if use_persisted_baseline {
+                state.baseline_generation
+            } else {
+                existing_baseline.2
+            };
+            progress.baseline_known = use_persisted_baseline
+                || existing_baseline.0 > 0
+                || existing_baseline.1 > 0
+                || existing_baseline.2.is_some()
+                || existing_baseline.4;
+        }
+        progress.progress_state = if use_persisted_baseline
+            || existing_baseline.0 > 0
+            || existing_baseline.1 > 0
+            || existing_baseline.2.is_some()
+            || existing_baseline.4
+        {
+            crate::heal::progress::HealProgressState::Running
+        } else {
+            crate::heal::progress::HealProgressState::Indeterminate
+        };
+        if baseline_generation_mismatch || state.counter_unknown {
+            progress.mark_unknown();
+        }
+        progress.ledger_complete = false;
+        progress.refresh_progress_percentage();
         progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
         progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
         progress.set_current_object(state.current_object.clone());
@@ -1264,8 +1514,8 @@ mod resume_loop_tests {
     };
     use crate::heal::progress::HealProgress;
     use crate::heal::resume::{
-        CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
-        compose_key,
+        CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
+        ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key,
     };
     use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
     use crate::heal::storage_api::status::BucketInfo;
@@ -1405,6 +1655,7 @@ mod resume_loop_tests {
         list_include_lifecycle_object_info: Mutex<Vec<bool>>,
         replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
         fail_listing: AtomicBool,
+        fail_listing_buckets: Mutex<HashSet<String>>,
     }
 
     impl FakeStorage {
@@ -1441,6 +1692,9 @@ mod resume_loop_tests {
         fn fail_listing(&self) {
             self.fail_listing.store(true, Ordering::SeqCst);
         }
+        fn fail_bucket_listing(&self, bucket: &str) {
+            self.fail_listing_buckets.lock().unwrap().insert(bucket.to_string());
+        }
     }
 
     #[async_trait::async_trait]
@@ -1531,7 +1785,7 @@ mod resume_loop_tests {
         }
         async fn list_objects_for_heal_page(
             &self,
-            _bucket: &str,
+            bucket: &str,
             _prefix: &str,
             continuation_token: Option<&str>,
             include_lifecycle_object_info: bool,
@@ -1540,7 +1794,7 @@ mod resume_loop_tests {
                 .lock()
                 .unwrap()
                 .push(include_lifecycle_object_info);
-            if self.fail_listing.load(Ordering::SeqCst) {
+            if self.fail_listing.load(Ordering::SeqCst) || self.fail_listing_buckets.lock().unwrap().contains(bucket) {
                 return Err(Error::other("injected listing failure"));
             }
             let key = continuation_token.map(str::to_string);
@@ -1918,6 +2172,49 @@ mod resume_loop_tests {
         assert!(state.completed_buckets.is_empty(), "the failed bucket must remain resumable");
     }
 
+    #[tokio::test]
+    async fn bucket_failure_stops_before_a_later_bucket_checkpoint() {
+        let env = make_env().await;
+        let task_id = ResumeUtils::generate_task_id();
+        let buckets = vec!["a".to_string(), "b".to_string()];
+        let resume = ResumeManager::new(
+            env.healer.disk.clone(),
+            task_id.clone(),
+            "erasure_set".to_string(),
+            "pool_0_set_0".to_string(),
+            buckets.clone(),
+        )
+        .await
+        .unwrap();
+        let checkpoint = CheckpointManager::new(env.healer.disk.clone(), task_id.clone())
+            .await
+            .unwrap();
+        env.storage.fail_bucket_listing("a");
+        for _ in 0..3 {
+            assert!(resume.schedule_retry().await.unwrap());
+        }
+
+        env.healer
+            .execute_heal_with_resume(&buckets, "pool_0_set_0", &resume, &checkpoint)
+            .await
+            .expect_err("the first bucket failure must keep the pass incomplete");
+        let persisted = checkpoint.get_checkpoint().await;
+        assert_eq!(persisted.current_bucket_index, 0);
+        assert!(resume.get_state().await.completed_buckets.is_empty());
+
+        let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &task_id)
+            .await
+            .unwrap();
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &task_id)
+            .await
+            .unwrap();
+        env.healer
+            .execute_heal_with_resume(&buckets, "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect_err("recovery must retry the earlier failed bucket");
+        assert!(!resumed.get_state().await.completed);
+    }
+
     #[tokio::test]
     async fn completed_resume_state_is_not_selected_for_a_new_heal() {
         let env = make_env().await;
@@ -2107,8 +2404,175 @@ mod resume_loop_tests {
         let mut names: Vec<String> = env.storage.calls().into_iter().map(|(n, _)| n).collect();
         names.sort();
         assert_eq!(names, vec!["a", "b", "c", "d"], "every object exactly once, none dropped/doubled");
-        // Final page not truncated => cursor cleared.
-        assert_eq!(env.resume.resume_cursor().await, None);
+        // Keep the final page cursor until the outer loop durably completes the
+        // bucket, so a crash can replay only this page against its identities.
+        assert_eq!(env.resume.resume_cursor().await, Some("t1".to_string()));
+    }
+
+    #[tokio::test]
+    async fn persisted_failure_waits_for_the_bounded_retry_after_page_replay() {
+        let env = make_env().await;
+        env.storage.set_page(
+            None,
+            Page {
+                items: vec![item("object", Some("v1"), false)],
+                next: None,
+                truncated: false,
+            },
+        );
+        env.checkpoint
+            .record_object_outcome(CheckpointObjectOutcomeRecord {
+                object: compose_key("object", Some("v1")),
+                outcome: CheckpointObjectOutcome::Failed,
+                successful: 0,
+                failed: 1,
+                skipped: 0,
+                bytes: 0,
+                skipped_new_versions: 0,
+                skipped_ilm_expired: 0,
+                counter_unknown: false,
+            })
+            .await
+            .unwrap();
+        env.checkpoint.advance_page(0, 1).await.unwrap();
+
+        let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+
+        env.healer
+            .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect_err("the persisted failure must schedule a bounded retry");
+        assert!(
+            env.storage.calls().is_empty(),
+            "the failed identity must not be repeated in the same pass"
+        );
+
+        env.healer
+            .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect("the bounded retry must heal the object");
+        assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
+        let state = resumed.get_state().await;
+        assert_eq!(state.successful_objects, 1);
+        assert_eq!(state.failed_objects, 0);
+    }
+
+    #[tokio::test]
+    async fn final_page_crash_replays_only_the_retained_page_identities() {
+        let env = make_env().await;
+        env.storage.set_page(
+            None,
+            Page {
+                items: vec![item("first", Some("v1"), false)],
+                next: Some("final-page".to_string()),
+                truncated: true,
+            },
+        );
+        env.storage.set_page(
+            Some("final-page"),
+            Page {
+                items: vec![item("last", Some("v1"), false)],
+                next: None,
+                truncated: false,
+            },
+        );
+
+        let (processed, successful, failed, skipped, result) = run(&env).await;
+        result.expect("the bucket pass must finish before the simulated crash");
+        assert_eq!((processed, successful, failed, skipped), (2, 2, 0, 0));
+
+        let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        env.healer
+            .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect("the retained final-page ledger must make recovery exact");
+
+        assert_eq!(
+            env.storage.calls(),
+            vec![
+                ("first".to_string(), Some("v1".to_string())),
+                ("last".to_string(), Some("v1".to_string()))
+            ]
+        );
+        let state = resumed.get_state().await;
+        assert_eq!(state.successful_objects, 2);
+        assert_eq!(state.processed_objects, 2);
+    }
+
+    #[tokio::test]
+    async fn truncated_page_without_token_retains_its_replay_ledger() {
+        let env = make_env().await;
+        env.storage.set_page(
+            None,
+            Page {
+                items: vec![item("object", Some("v1"), false)],
+                next: None,
+                truncated: true,
+            },
+        );
+
+        let (processed, successful, failed, skipped, result) = run(&env).await;
+        result.expect("the tokenless truncated page is a terminal page");
+        assert_eq!((processed, successful, failed, skipped), (1, 1, 0, 0));
+
+        let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        env.healer
+            .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect("terminal-page recovery must not replay a durable identity");
+
+        assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
+        let state = resumed.get_state().await;
+        assert_eq!(state.successful_objects, 1);
+        assert_eq!(state.processed_objects, 1);
+    }
+
+    #[tokio::test]
+    async fn completed_bucket_reconciles_its_final_page_checkpoint_after_crash() {
+        let env = make_env().await;
+        env.storage.set_page(
+            None,
+            Page {
+                items: vec![item("object", Some("v1"), false)],
+                next: None,
+                truncated: false,
+            },
+        );
+
+        let (_, _, _, _, result) = run(&env).await;
+        result.expect("the bucket pass must finish before the simulated crash");
+        env.resume.complete_bucket("b").await.unwrap();
+
+        let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .unwrap();
+        env.healer
+            .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
+            .await
+            .expect("recovery must finish the checkpoint transition without replaying the bucket");
+
+        assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
+        let checkpoint = checkpoint.get_checkpoint().await;
+        assert_eq!(checkpoint.current_bucket_index, 1);
+        assert!(checkpoint.processed_objects.is_empty());
     }
 
     #[tokio::test]

crates/heal/src/heal/manager.rs

@@ -14,7 +14,7 @@
 
 use crate::heal::{
     progress::{HealProgress, HealStatistics},
-    resume::{ReplacementPhase, ResumeManager, ResumeState, ResumeUtils},
+    resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
     storage::HealStorageAPI,
     task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
 };
@@ -53,9 +53,11 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
 const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
 const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
 const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
+const EVENT_HEAL_RESUME_GC: &str = "heal_resume_gc";
 const LEGACY_ROOT_HEAL_PATH: &str = ".";
 const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
 const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
+const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60);
 
 // Admission/scheduler outcomes for per-object requests (Object/Metadata/
 // ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
@@ -115,10 +117,13 @@ async fn pause_duplicate_admission_after_active_lock(request_id: &str) {
 type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-struct MrfRepairNoticeTarget {
-    bucket: Arc<str>,
-    object: Arc<str>,
-    version_id: Option<[u8; 16]>,
+pub(super) struct MrfRepairNoticeTarget {
+    pub(super) bucket: Arc<str>,
+    pub(super) object: Arc<str>,
+    pub(super) version_id: Option<[u8; 16]>,
+    pub(super) kind: rustfs_common::mrf_channel::MrfKind,
+    pub(super) scope: Option<rustfs_common::mrf_channel::MrfScope>,
+    pub(super) lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
 }
 
 #[derive(Debug, Clone)]
@@ -889,7 +894,19 @@ impl HealManager {
     }
 
     fn remove_mrf_repair_notice_targets_for_task(&self, task_id: &str) {
-        lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
+        let targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
+        if let Some(targets) = targets {
+            for target in targets {
+                rustfs_common::mrf_channel::release_mrf_identity(
+                    target.kind,
+                    &target.bucket,
+                    &target.object,
+                    target.version_id,
+                    target.scope,
+                    target.lease,
+                );
+            }
+        }
     }
 
     fn insert_mrf_repair_notice_target(
@@ -1150,6 +1167,49 @@ impl HealManager {
         Ok(())
     }
 
+    /// Start the bounded resume-state inspector.  Destructive GC remains
+    /// disabled until the durable owner/CAS contract from backlog#1927 is
+    /// available; this task therefore cannot remove an active or stale file.
+    async fn start_resume_gc(&self) {
+        let cancel = self.cancel_token.clone();
+        tokio::spawn(async move {
+            let mut gc_by_disk = HashMap::<String, ResumeGc>::new();
+            let mut ticker = interval(RESUME_GC_INTERVAL);
+            loop {
+                tokio::select! {
+                    _ = cancel.cancelled() => break,
+                    _ = ticker.tick() => {
+                        let disks = {
+                            let local_disk_map = local_disk_map_read().await;
+                            local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
+                        };
+                        for disk in disks {
+                            let disk_key = disk.endpoint().to_string();
+                            let gc = gc_by_disk.entry(disk_key).or_default();
+                            tokio::select! {
+                                _ = cancel.cancelled() => return,
+                                result = gc.inspect_disk(&disk) => {
+                                    if let Err(error) = result {
+                                        warn!(
+                                            target: "rustfs::heal::manager",
+                                            event = EVENT_HEAL_RESUME_GC,
+                                            component = LOG_COMPONENT_HEAL,
+                                            subsystem = LOG_SUBSYSTEM_MANAGER,
+                                            state = "inspect_failed",
+                                            endpoint = %disk.endpoint(),
+                                            error = %error,
+                                            "Heal resume GC inspection failed"
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        });
+    }
+
     /// Create new HealManager
     pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
         Self::new_with_workload_provider(storage, config, None)
@@ -1215,6 +1275,9 @@ impl HealManager {
         // competing task for the same set.
         self.process_unclean_shutdown().await;
 
+        // Inspect resume artifacts in a bounded, fail-closed background task.
+        self.start_resume_gc().await;
+
         // start auto disk scanner to heal unformatted disks
         if self.config.read().await.enable_auto_heal {
             self.start_auto_disk_scanner().await?;
@@ -1279,7 +1342,20 @@ impl HealManager {
         }
         self.task_aliases.lock().await.clear();
         self.retrying_heals.lock().await.clear();
-        lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
+        let mrf_targets = {
+            let mut registry = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
+            registry.drain().flat_map(|(_, targets)| targets).collect::<Vec<_>>()
+        };
+        for target in mrf_targets {
+            rustfs_common::mrf_channel::release_mrf_identity(
+                target.kind,
+                &target.bucket,
+                &target.object,
+                target.version_id,
+                target.scope,
+                target.lease,
+            );
+        }
         crate::set_heal_queue_length(0);
 
         // update state
@@ -1311,6 +1387,7 @@ impl HealManager {
             .await
     }
 
+    #[cfg(test)]
     pub(crate) async fn submit_mrf_heal_request_with_receipt(
         &self,
         request: HealRequest,
@@ -1318,18 +1395,34 @@ impl HealManager {
         object: Arc<str>,
         version_id: Option<[u8; 16]>,
     ) -> Result<HealAdmissionReceipt> {
-        self.submit_heal_request_with_receipt_alias_and_mrf_notice(
+        let kind = match &request.heal_type {
+            HealType::Metadata { .. } => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
+            HealType::ECDecode { .. } => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
+            _ => rustfs_common::mrf_channel::MrfKind::PartialWrite,
+        };
+        self.submit_mrf_heal_request_with_receipt_and_identity(
             request,
-            true,
-            Some(MrfRepairNoticeTarget {
+            MrfRepairNoticeTarget {
                 bucket,
                 object,
                 version_id,
-            }),
+                kind,
+                scope: None,
+                lease: None,
+            },
         )
         .await
     }
 
+    pub(super) async fn submit_mrf_heal_request_with_receipt_and_identity(
+        &self,
+        request: HealRequest,
+        mrf_notice_target: MrfRepairNoticeTarget,
+    ) -> Result<HealAdmissionReceipt> {
+        self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, Some(mrf_notice_target))
+            .await
+    }
+
     async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
         &self,
         request: HealRequest,
@@ -1539,7 +1632,7 @@ impl HealManager {
             Self::insert_mrf_repair_notice_target(&mut targets, &task_id, target);
         }
         if let Some(displaced_task_id) = &displaced_task_id {
-            lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(displaced_task_id);
+            self.remove_mrf_repair_notice_targets_for_task(displaced_task_id);
         }
         drop(retrying_heals);
         drop(queue);
@@ -2011,34 +2104,11 @@ impl HealManager {
             return None;
         }
 
-        let mut snapshot = HealProgress::default();
+        let mut progresses = Vec::with_capacity(active_tasks.len());
         for task in active_tasks {
-            let progress = task.get_progress().await;
-            snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
-            snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
-            snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
-            snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
-            snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
-            snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
-            snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
-            snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
-            snapshot.start_time = match (snapshot.start_time, progress.start_time) {
-                (Some(current), Some(next)) => Some(current.min(next)),
-                (None, next) => next,
-                (current, None) => current,
-            };
-            snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
-                (Some(current), Some(next)) => Some(current.max(next)),
-                (None, next) => next,
-                (current, None) => current,
-            };
-            if progress.current_object.is_some() {
-                snapshot.current_object = progress.current_object;
-            }
+            progresses.push(task.get_progress().await);
         }
-        snapshot.refresh_progress_percentage();
-        snapshot.refresh_estimated_completion_time();
-        Some(snapshot)
+        crate::heal::progress::aggregate_heal_progress(progresses)
     }
 }
 

crates/heal/src/heal/manager/auto_scan.rs

@@ -506,7 +506,18 @@ impl HealManager {
                                     &displaced_terminal,
                                 )
                                 .await;
-                                lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
+                                if let Some(targets) = lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id) {
+                                    for target in targets {
+                                        rustfs_common::mrf_channel::release_mrf_identity(
+                                            target.kind,
+                                            &target.bucket,
+                                            &target.object,
+                                            target.version_id,
+                                            target.scope,
+                                            target.lease,
+                                        );
+                                    }
+                                }
                             }
                             if matches!(admission, HealAdmissionResult::Accepted) {
                                 if should_notify {

crates/heal/src/heal/manager/queue.rs

@@ -299,7 +299,11 @@ impl PriorityHealQueue {
 
     /// Create a deduplication key from a heal request
     pub(super) fn make_dedup_key(request: &HealRequest) -> String {
-        Self::make_dedup_key_for_type(&request.heal_type)
+        let base = Self::make_dedup_key_for_type(&request.heal_type);
+        match (&request.heal_type, request.options.set_key()) {
+            (HealType::Object { .. } | HealType::ECDecode { .. }, Some(scope)) => format!("{base}:scope:{scope}"),
+            _ => base,
+        }
     }
 
     pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {

crates/heal/src/heal/manager/scheduler.rs

@@ -349,6 +349,8 @@ impl HealManager {
                             let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
                             if successful_completion {
                                 emit_mrf_repaired_events(notice_targets);
+                            } else {
+                                release_mrf_repair_notice_targets(notice_targets);
                             }
                             task_aliases_clone
                                 .lock()
@@ -638,7 +640,19 @@ pub(super) fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTas
 }
 
 fn remove_mrf_repair_notice_targets(registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>, task_id: &str) {
-    lock_mrf_repair_notice_targets(registry).remove(task_id);
+    let targets = lock_mrf_repair_notice_targets(registry).remove(task_id);
+    if let Some(targets) = targets {
+        for target in targets {
+            rustfs_common::mrf_channel::release_mrf_identity(
+                target.kind,
+                &target.bucket,
+                &target.object,
+                target.version_id,
+                target.scope,
+                target.lease,
+            );
+        }
+    }
 }
 
 fn take_mrf_repair_notice_targets(
@@ -671,6 +685,27 @@ fn move_mrf_repair_notice_targets(
 fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
     for target in targets {
         rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
+        rustfs_common::mrf_channel::release_mrf_identity(
+            target.kind,
+            &target.bucket,
+            &target.object,
+            target.version_id,
+            target.scope,
+            target.lease,
+        );
+    }
+}
+
+fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
+    for target in targets {
+        rustfs_common::mrf_channel::release_mrf_identity(
+            target.kind,
+            &target.bucket,
+            &target.object,
+            target.version_id,
+            target.scope,
+            target.lease,
+        );
     }
 }
 

crates/heal/src/heal/mrf_queue.rs

@@ -33,11 +33,11 @@
 //! than waiting for the failed-object TTL to re-scan the path.
 
 use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
-use crate::heal::manager::HealManager;
+use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
 use metrics::{counter, gauge};
 use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
 use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
-use std::collections::VecDeque;
+use std::collections::{HashSet, VecDeque};
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc;
@@ -48,15 +48,27 @@ use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
 /// Journal location inside the metadata bucket, following the resume-state
 /// layout.
 pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
+/// The scoped path is the authoritative snapshot for new readers and carries
+/// both v1 and v2 records. The legacy path is only a v1 compatibility mirror;
+/// older readers ignore the authoritative path, while new readers never merge
+/// the two files. This prevents a partial two-file flush from fabricating a
+/// mixed epoch.
+pub(crate) const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
 
 /// Record format tag.
 const MRF_JOURNAL_FORMAT: u8 = 1;
 /// Record layout version.
 const MRF_JOURNAL_VERSION: u8 = 1;
+const MRF_JOURNAL_VERSION_SCOPED: u8 = 2;
 
 /// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
 /// has_version flag.
 const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
+const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
+
+fn metric_f64(value: usize) -> f64 {
+    f64::from(u32::try_from(value).unwrap_or(u32::MAX))
+}
 
 #[derive(Debug, Clone)]
 pub(crate) struct MrfConsumerConfig {
@@ -101,40 +113,90 @@ impl Default for MrfConsumerConfig {
 /// incoming intent (never a resident one) and counts the loss.
 pub(crate) struct MrfQueue {
     pending: VecDeque<MrfIntent>,
+    pending_keys: HashSet<MrfQueueKey>,
     bytes: usize,
     capacity: usize,
     byte_budget: usize,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct MrfQueueKey {
+    kind: rustfs_common::mrf_channel::MrfKind,
+    bucket: Arc<str>,
+    object: Arc<str>,
+    version_id: Option<[u8; 16]>,
+    scope: Option<rustfs_common::mrf_channel::MrfScope>,
+}
+
+fn queue_key(intent: &MrfIntent) -> MrfQueueKey {
+    let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
+    let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
+        .then_some(intent.scope)
+        .flatten();
+    MrfQueueKey {
+        kind: intent.kind,
+        bucket: intent.bucket.clone(),
+        object: intent.object.clone(),
+        version_id,
+        scope,
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum MrfQueuePushResult {
+    Enqueued,
+    Coalesced,
+    Rejected,
+}
+
 impl MrfQueue {
     pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
         Self {
             pending: VecDeque::new(),
+            pending_keys: HashSet::new(),
             bytes: 0,
             capacity,
             byte_budget,
         }
     }
 
-    /// Returns `false` (after counting) when either ceiling would be crossed.
-    pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
+    pub(crate) fn try_push_typed(&mut self, intent: MrfIntent) -> MrfQueuePushResult {
+        if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
+            counter!("rustfs_heal_mrf_dropped_total", "reason" => "identity_oversized").increment(1);
+            return MrfQueuePushResult::Rejected;
+        }
+        let key = queue_key(&intent);
+        if self.pending_keys.contains(&key) {
+            counter!("rustfs_heal_mrf_coalesced_total", "layer" => "queue").increment(1);
+            return MrfQueuePushResult::Coalesced;
+        }
         let cost = intent.estimated_bytes();
         if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
             counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
-            return false;
+            return MrfQueuePushResult::Rejected;
         }
         self.bytes += cost;
+        self.pending_keys.insert(key);
         self.pending.push_back(intent);
-        true
+        MrfQueuePushResult::Enqueued
+    }
+
+    /// Bool compatibility adapter: only a newly executable queue item is
+    /// reported as accepted; a coalesced duplicate is not durable admission.
+    #[cfg(test)]
+    pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
+        matches!(self.try_push_typed(intent), MrfQueuePushResult::Enqueued)
     }
 
     pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
         let intent = self.pending.pop_front()?;
+        self.pending_keys.remove(&queue_key(&intent));
         self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
         Some(intent)
     }
 
     pub(crate) fn push_back(&mut self, intent: MrfIntent) {
+        self.pending_keys.insert(queue_key(&intent));
         self.bytes += intent.estimated_bytes();
         self.pending.push_back(intent);
     }
@@ -157,10 +219,24 @@ impl MrfQueue {
 // ---------------------------------------------------------------------------
 
 /// Append one encoded record to `out`.
-pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
+pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
+    let Ok(bucket_len) = u32::try_from(intent.bucket.len()) else {
+        return false;
+    };
+    let Ok(object_len) = u32::try_from(intent.object.len()) else {
+        return false;
+    };
+    let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
+        .then_some(intent.scope)
+        .flatten();
+    let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
     let start = out.len();
     out.push(MRF_JOURNAL_FORMAT);
-    out.push(MRF_JOURNAL_VERSION);
+    out.push(if scope.is_some() {
+        MRF_JOURNAL_VERSION_SCOPED
+    } else {
+        MRF_JOURNAL_VERSION
+    });
     out.push(match intent.kind {
         rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
         rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
@@ -168,27 +244,36 @@ pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
     });
     out.push(intent.attempts);
     out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
-    match intent.version_id {
+    match version_id {
         Some(bytes) => {
             out.push(1);
             out.extend_from_slice(&bytes);
         }
         None => out.push(0),
     }
-    out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
-    out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
+    if let Some(scope) = scope {
+        out.extend_from_slice(&scope.pool_index.to_le_bytes());
+        out.extend_from_slice(&scope.set_index.to_le_bytes());
+    }
+    out.extend_from_slice(&bucket_len.to_le_bytes());
+    out.extend_from_slice(&object_len.to_le_bytes());
     out.extend_from_slice(intent.bucket.as_bytes());
     out.extend_from_slice(intent.object.as_bytes());
     let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
     hasher.update(&out[start..]);
-    out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
+    let Ok(checksum) = u32::try_from(hasher.finalize()) else {
+        out.truncate(start);
+        return false;
+    };
+    out.extend_from_slice(&checksum.to_le_bytes());
+    true
 }
 
 fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
     if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
         return None;
     }
-    if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
+    if data[0] != MRF_JOURNAL_FORMAT || !matches!(data[1], MRF_JOURNAL_VERSION | MRF_JOURNAL_VERSION_SCOPED) {
         return None;
     }
     let kind = match data[2] {
@@ -198,24 +283,38 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
         _ => return None,
     };
     let attempts = data[3];
-    let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
+    let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
     let has_version = data[12] != 0;
     let mut cursor = MRF_RECORD_FIXED_HEAD;
     let version_id = if has_version {
         if data.len() < cursor + 16 {
             return None;
         }
-        let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
+        let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().ok()?;
         cursor += 16;
         Some(bytes)
     } else {
         None
     };
+    let scope = if data[1] == MRF_JOURNAL_VERSION_SCOPED {
+        if data.len() < cursor + 8 {
+            return None;
+        }
+        let pool_index = u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?);
+        let set_index = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?);
+        cursor += 8;
+        Some(rustfs_common::mrf_channel::MrfScope { pool_index, set_index })
+    } else {
+        None
+    };
     if data.len() < cursor + 8 {
         return None;
     }
-    let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
-    let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
+    let bucket_len = usize::try_from(u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?)).ok()?;
+    let object_len = usize::try_from(u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?)).ok()?;
+    if bucket_len > MRF_MAX_IDENTITY_COMPONENT || object_len > MRF_MAX_IDENTITY_COMPONENT {
+        return None;
+    }
     cursor += 8;
     let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
     let record_end = body_end.checked_add(4)?;
@@ -224,7 +323,7 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
     }
     let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
     hasher.update(&data[..body_end]);
-    if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
+    if u32::try_from(hasher.finalize()).ok()? != u32::from_le_bytes(data[body_end..record_end].try_into().ok()?) {
         return None;
     }
     let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
@@ -235,6 +334,12 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
             object,
             version_id,
             kind,
+            scope: if matches!(kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) {
+                None
+            } else {
+                scope
+            },
+            lease: None,
             enqueued_at_ms,
             attempts,
         },
@@ -269,9 +374,9 @@ async fn journal_disks() -> Vec<DiskStore> {
     map.values().flatten().cloned().collect()
 }
 
-async fn read_journal() -> Option<Vec<u8>> {
+async fn read_journal(path: &str) -> Option<Vec<u8>> {
     for disk in journal_disks().await {
-        match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
+        match disk.read_all(super::RUSTFS_META_BUCKET, path).await {
             Ok(bytes) => return Some(bytes.to_vec()),
             Err(_) => continue,
         }
@@ -282,35 +387,51 @@ async fn read_journal() -> Option<Vec<u8>> {
 /// Write the snapshot to every local disk; returns true when at least one
 /// disk accepted it, so a total write failure keeps the runtime dirty and
 /// the next tick retries the persist.
-async fn write_journal(data: &[u8]) -> bool {
+async fn write_journal(path: &str, data: &[u8]) -> bool {
     let payload = bytes::Bytes::copy_from_slice(data);
     let mut any_persisted = false;
     for disk in journal_disks().await {
-        match disk
-            .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
-            .await
-        {
+        match disk.write_all(super::RUSTFS_META_BUCKET, path, payload.clone()).await {
             Ok(()) => any_persisted = true,
             Err(err) => warn_mrf_journal_write(&err),
         }
     }
-    if !data.is_empty() {
-        counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
-    }
-    gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
     any_persisted
 }
 
-async fn delete_journal() {
-    for disk in journal_disks().await {
-        let _ = disk
+async fn delete_journal(path: &str) -> bool {
+    let disks = journal_disks().await;
+    if disks.is_empty() {
+        counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
+        return false;
+    }
+    let mut all_deleted = true;
+    for disk in disks {
+        let result = disk
             .delete(
                 super::RUSTFS_META_BUCKET,
-                MRF_JOURNAL_PATH,
+                path,
                 crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
             )
             .await;
+        if let Err(err) = result {
+            // Delete is idempotent: a compatibility mirror that was never
+            // written (or was already removed) is clean, not a retry state.
+            if !matches!(err, super::DiskError::FileNotFound | super::DiskError::VolumeNotFound) {
+                all_deleted = false;
+            }
+        }
     }
+    if !all_deleted {
+        counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
+    }
+    all_deleted
+}
+
+async fn delete_journals() -> bool {
+    let authoritative_deleted = delete_journal(MRF_SCOPED_JOURNAL_PATH).await;
+    let legacy_deleted = delete_journal(MRF_JOURNAL_PATH).await;
+    authoritative_deleted && legacy_deleted
 }
 
 fn warn_mrf_journal_write(err: &super::DiskError) {
@@ -331,7 +452,10 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
 pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
     let bucket = intent.bucket.to_string();
     let object = intent.object.to_string();
-    let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
+    let version_id = intent
+        .version_id
+        .filter(|bytes| *bytes != [0; 16])
+        .map(|bytes| Uuid::from_bytes(bytes).to_string());
     let (heal_type, priority) = match intent.kind {
         rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
             HealType::ECDecode {
@@ -351,18 +475,30 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
             HealPriority::Normal,
         ),
     };
-    let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
+    let mut options = HealOptions::default();
+    if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
+        && let Some(scope) = intent.scope
+    {
+        options.pool_index = usize::try_from(scope.pool_index).ok();
+        options.set_index = usize::try_from(scope.set_index).ok();
+    }
+    let mut request = HealRequest::new(heal_type, options, priority);
     request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
     request
 }
 
 async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
     let receipt = manager
-        .submit_mrf_heal_request_with_receipt(
+        .submit_mrf_heal_request_with_receipt_and_identity(
             build_heal_request(intent),
-            intent.bucket.clone(),
-            intent.object.clone(),
-            intent.version_id,
+            MrfRepairNoticeTarget {
+                bucket: intent.bucket.clone(),
+                object: intent.object.clone(),
+                version_id: intent.version_id,
+                kind: intent.kind,
+                scope: intent.scope,
+                lease: intent.lease,
+            },
         )
         .await?;
     Ok(receipt.result)
@@ -387,16 +523,38 @@ struct MrfRuntime {
 }
 
 impl MrfRuntime {
-    fn snapshot(&self) -> Vec<u8> {
-        let mut buf = Vec::new();
+    fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
+        let mut authoritative = Vec::new();
+        let mut legacy = Vec::new();
         for intent in self.queue.intents() {
-            encode_intent(intent, &mut buf);
+            let scoped_identity =
+                !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
+            if !encode_intent(intent, &mut authoritative) {
+                counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
+            }
+            if !scoped_identity && !encode_intent(intent, &mut legacy) {
+                counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
+            }
         }
-        buf
+        (authoritative, legacy)
     }
 
     async fn flush(&mut self) {
-        let persisted = write_journal(&self.snapshot()).await;
+        let (authoritative, legacy) = self.snapshot();
+        let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
+        if !authoritative.is_empty() {
+            counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
+        }
+        gauge!("rustfs_heal_mrf_journal_bytes").set(metric_f64(authoritative.len()));
+        // Publish the compatibility mirror only after the authoritative
+        // snapshot has reached at least one disk. This ordering prevents an
+        // old reader from observing a newer epoch that a new reader cannot
+        // see when the canonical write is unavailable.
+        let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
+        // Keep dirty until both the authoritative snapshot and its
+        // compatibility mirror have been accepted; otherwise a one-sided
+        // failure would never retry the missing file.
+        let persisted = authoritative_persisted && legacy_persisted;
         self.new_since_flush = 0;
         // Keep the dirty flag when every disk write failed: a clean backlog
         // would otherwise never rewrite, losing the periodic persist retry a
@@ -404,7 +562,7 @@ impl MrfRuntime {
         if persisted {
             self.dirty = false;
         }
-        self.journal_on_disk = true;
+        self.journal_on_disk |= authoritative_persisted || legacy_persisted;
     }
 
     /// Drain pending intents into the heal manager until it is full, the
@@ -430,6 +588,7 @@ impl MrfRuntime {
                     intent.attempts = intent.attempts.saturating_add(1);
                     if intent.attempts >= MRF_MAX_ATTEMPTS {
                         counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
+                        rustfs_common::mrf_channel::release_mrf_intent(&intent);
                         continue;
                     }
                     self.queue.push_back(intent);
@@ -438,11 +597,13 @@ impl MrfRuntime {
                 }
                 Ok(HealAdmissionResult::Dropped(_)) => {
                     counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
+                    rustfs_common::mrf_channel::release_mrf_intent(&intent);
                 }
                 Err(_) => {
                     intent.attempts = intent.attempts.saturating_add(1);
                     if intent.attempts >= MRF_MAX_ATTEMPTS {
                         counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
+                        rustfs_common::mrf_channel::release_mrf_intent(&intent);
                         continue;
                     }
                     self.queue.push_back(intent);
@@ -451,8 +612,8 @@ impl MrfRuntime {
                 }
             }
         }
-        gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
-        gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
+        gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
+        gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
     }
 }
 
@@ -496,7 +657,12 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
     let config = MrfConsumerConfig::default();
     let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
     let mut backoff_until: Option<tokio::time::Instant> = None;
-    replay_into(manager, &mut queue, &mut backoff_until).await
+    replay_into(manager, &mut queue, &mut backoff_until).await.replayed
+}
+
+struct ReplayOutcome {
+    replayed: usize,
+    journal_on_disk: bool,
 }
 
 /// Shared replay core: read + decode + re-arm + delete, then drain what fits.
@@ -504,11 +670,25 @@ async fn replay_into(
     manager: &Arc<HealManager>,
     queue: &mut MrfQueue,
     backoff_until: &mut Option<tokio::time::Instant>,
-) -> usize {
-    let Some(data) = read_journal().await else {
-        return 0;
+) -> ReplayOutcome {
+    // The scoped file is a complete authoritative snapshot. Fall back to the
+    // legacy mirror only when the authoritative path is unavailable; merging
+    // both files could combine records from different flush epochs.
+    let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
+        Some(data) => data,
+        None => match read_journal(MRF_JOURNAL_PATH).await {
+            Some(data) => data,
+            None => {
+                return ReplayOutcome {
+                    replayed: 0,
+                    journal_on_disk: false,
+                };
+            }
+        },
     };
-    let (intents, truncated) = decode_journal(&data);
+    let mut intents = Vec::new();
+    let (decoded, truncated) = decode_journal(&data);
+    intents.extend(decoded);
     if truncated > 0 {
         tracing::warn!(
             target: "rustfs::heal::mrf",
@@ -516,12 +696,15 @@ async fn replay_into(
             "MRF journal had a torn tail; truncated records were discarded"
         );
     }
-    counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
+    counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX));
     let replayed = intents.len();
     for intent in intents {
-        queue.try_push(intent);
+        let result = queue.try_push_typed(intent.clone());
+        if !matches!(result, MrfQueuePushResult::Enqueued) {
+            rustfs_common::mrf_channel::release_mrf_intent(&intent);
+        }
     }
-    delete_journal().await;
+    let journal_on_disk = !delete_journals().await;
 
     // Drain the replayed intents immediately; whatever the manager refuses
     // stays armed in `queue` for the consumer's retry loop.
@@ -541,7 +724,10 @@ async fn replay_into(
             }
         }
     }
-    replayed
+    ReplayOutcome {
+        replayed,
+        journal_on_disk,
+    }
 }
 
 /// Replay the journal, then keep draining the channel into the heal manager
@@ -559,7 +745,8 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
 
     // Replay: read the journal, re-arm intents (duplicates are merged by the
     // manager's dedup key), then drop the file so the next flush starts clean.
-    replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
+    let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
+    runtime.journal_on_disk = replay.journal_on_disk;
     // The replay deleted the journal file; anything still pending (e.g. the
     // manager was full and backoff armed) must be re-persisted by the next
     // flush or a crash before it would lose those intents.
@@ -587,9 +774,14 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
                     return;
                 }
                 for intent in batch.drain(..) {
-                    if runtime.queue.try_push(intent) {
-                        runtime.new_since_flush += 1;
-                        runtime.dirty = true;
+                    match runtime.queue.try_push_typed(intent.clone()) {
+                        MrfQueuePushResult::Enqueued => {
+                            runtime.new_since_flush += 1;
+                            runtime.dirty = true;
+                        }
+                        MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
+                            rustfs_common::mrf_channel::release_mrf_intent(&intent);
+                        }
                     }
                 }
                 runtime.dispatch(manager.as_ref()).await;
@@ -613,13 +805,14 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
                     TickAction::DeleteJournal => {
                         // All intents consumed: remove the journal so a restart
                         // replays nothing (mirrors MinIO's post-replay unlink).
-                        delete_journal().await;
-                        runtime.journal_on_disk = false;
-                        gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
+                        if delete_journals().await {
+                            runtime.journal_on_disk = false;
+                            gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
+                        }
                     }
                     TickAction::Idle => {}
                 }
-                gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
+                gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(runtime.queue.depth()));
             }
         }
     }
@@ -664,6 +857,8 @@ mod tests {
             object: StdArc::from(object),
             version_id: Some([7u8; 16]),
             kind: MrfKind::DecodeFailure,
+            scope: None,
+            lease: None,
             enqueued_at_ms: 1_700_000_000_000,
             attempts,
         }
@@ -694,17 +889,101 @@ mod tests {
     fn queue_enforces_count_and_byte_ceilings() {
         let mut queue = MrfQueue::new(2, usize::MAX);
         assert!(queue.try_push(intent("b", "o", 0)));
-        assert!(queue.try_push(intent("b", "o", 0)));
-        assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
+        assert!(queue.try_push(intent("b", "o2", 0)));
+        assert!(!queue.try_push(intent("b", "o3", 0)), "count ceiling must drop");
 
         let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
         assert!(tiny.try_push(intent("bucket", "object", 0)));
         assert!(
-            !tiny.try_push(intent("bucket", "object", 0)),
+            !tiny.try_push(intent("bucket", "object2", 0)),
             "byte budget must drop before the second intent fits"
         );
     }
 
+    #[test]
+    fn duplicate_mrf_intents_coalesce_to_one_execution() {
+        let mut queue = MrfQueue::new(1000, usize::MAX);
+        let mut enqueued = 0;
+        let mut coalesced = 0;
+        assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+        enqueued += 1;
+        for _ in 0..999 {
+            match queue.try_push_typed(intent("bucket", "object", 0)) {
+                MrfQueuePushResult::Coalesced => coalesced += 1,
+                other => panic!("duplicate intent was not coalesced: {other:?}"),
+            }
+        }
+        assert_eq!(enqueued, 1);
+        assert_eq!(coalesced, 999);
+        assert_eq!(queue.depth(), 1);
+    }
+
+    #[test]
+    fn mrf_dedupe_does_not_merge_adjacent_version_pool_or_kind() {
+        let mut queue = MrfQueue::new(8, usize::MAX);
+        let mut first = intent("bucket", "object", 0);
+        first.kind = MrfKind::PartialWrite;
+        first.scope = Some(rustfs_common::mrf_channel::MrfScope {
+            pool_index: 1,
+            set_index: 1,
+        });
+        assert!(queue.try_push(first.clone()));
+        first.version_id = Some([8u8; 16]);
+        assert!(queue.try_push(first));
+        let mut other_scope = intent("bucket", "object", 0);
+        other_scope.kind = MrfKind::PartialWrite;
+        other_scope.scope = Some(rustfs_common::mrf_channel::MrfScope {
+            pool_index: 2,
+            set_index: 1,
+        });
+        assert!(queue.try_push(other_scope));
+        let mut other_kind = intent("bucket", "object", 0);
+        other_kind.kind = MrfKind::DecodeFailure;
+        other_kind.scope = None;
+        assert!(queue.try_push(other_kind));
+        assert_eq!(queue.depth(), 4);
+    }
+
+    #[test]
+    fn mrf_dedupe_full_returns_rejected_with_durable_pending() {
+        let mut queue = MrfQueue::new(1, usize::MAX);
+        assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Rejected);
+        assert_eq!(queue.depth(), 1);
+        let mut snapshot = Vec::new();
+        assert!(encode_intent(queue.intents().next().expect("resident intent"), &mut snapshot));
+        assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection");
+    }
+
+    #[test]
+    fn mrf_dedupe_failure_releases_key_for_retry() {
+        let mut queue = MrfQueue::new(1, usize::MAX);
+        assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+        let _failed = queue.pop_front().expect("queued intent");
+        assert_eq!(queue.try_push_typed(intent("bucket", "object", 1)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.depth(), 1);
+    }
+
+    #[test]
+    fn mrf_dedupe_key_and_map_are_bounded() {
+        let mut queue = MrfQueue::new(2, usize::MAX);
+        assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(queue.pending_keys.len(), 2);
+        assert_eq!(queue.try_push_typed(intent("bucket", "third", 0)), MrfQueuePushResult::Rejected);
+        assert_eq!(queue.depth(), 2);
+    }
+
+    #[test]
+    fn cross_node_duplicate_execution_remains_idempotent() {
+        // Node-local ingress maps intentionally do not merge across nodes;
+        // the manager's existing identity key absorbs the duplicate later.
+        let mut node_a = MrfQueue::new(8, usize::MAX);
+        let mut node_b = MrfQueue::new(8, usize::MAX);
+        assert_eq!(node_a.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+        assert_eq!(node_b.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
+    }
+
     #[test]
     fn journal_roundtrip_preserves_intents() {
         let intents = vec![
@@ -715,6 +994,8 @@ mod tests {
                 object: StdArc::from("object/c"),
                 version_id: None,
                 kind: MrfKind::MetadataCorruption,
+                scope: None,
+                lease: None,
                 enqueued_at_ms: 5,
                 attempts: 1,
             },
@@ -766,6 +1047,8 @@ mod tests {
             object: StdArc::from("o"),
             version_id: None,
             kind: MrfKind::MetadataCorruption,
+            scope: None,
+            lease: None,
             enqueued_at_ms: 0,
             attempts: 0,
         });
@@ -777,6 +1060,8 @@ mod tests {
             object: StdArc::from("o"),
             version_id: None,
             kind: MrfKind::PartialWrite,
+            scope: None,
+            lease: None,
             enqueued_at_ms: 0,
             attempts: 0,
         });

crates/heal/src/heal/progress.rs

@@ -15,15 +15,91 @@
 use serde::{Deserialize, Serialize};
 use std::time::{Duration, SystemTime};
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub(crate) fn stable_generation(parts: &[&[u8]]) -> u64 {
+    let mut hash = 0xcbf29ce484222325u64;
+    for part in parts {
+        for byte in (part.len() as u64).to_be_bytes().into_iter().chain(part.iter().copied()) {
+            hash ^= u64::from(byte);
+            hash = hash.wrapping_mul(0x100000001b3);
+        }
+    }
+    hash
+}
+
+#[cfg(test)]
+mod stable_generation_tests {
+    use super::stable_generation;
+
+    #[test]
+    fn stable_generation_has_a_fixed_vector() {
+        assert_eq!(stable_generation(&[b"rustfs", b"heal", b"42"]), 11_007_672_338_488_385_056);
+    }
+}
+
+pub(crate) fn increment_counter(counter: &mut u64) -> bool {
+    match counter.checked_add(1) {
+        Some(next) => {
+            *counter = next;
+            true
+        }
+        None => {
+            *counter = u64::MAX;
+            false
+        }
+    }
+}
+
+pub(crate) fn add_bytes(total: &mut u64, amount: u64) -> bool {
+    match total.checked_add(amount) {
+        Some(next) => {
+            *total = next;
+            true
+        }
+        None => {
+            *total = u64::MAX;
+            false
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
+pub enum HealProgressKind {
+    #[default]
+    Unknown,
+    Stage,
+    ObjectSweep,
+}
+
+/// Whether the object ledger can produce a meaningful percentage.
+///
+/// A zero-valued baseline is not a completed scan: it means that no complete
+/// usage snapshot was available.  Keep this state explicit so callers do not
+/// mistake the legacy `0.0` wire value for a measured zero-percent result.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum HealProgressState {
+    #[default]
+    Unknown,
+    Indeterminate,
+    Running,
+    Completed,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
 pub struct HealProgress {
+    #[serde(default)]
+    pub kind: HealProgressKind,
     /// Objects scanned
     pub objects_scanned: u64,
     /// Objects healed
     pub objects_healed: u64,
     /// Objects failed
     pub objects_failed: u64,
+    /// Versions deferred for a later retry pass.
+    #[serde(default)]
+    pub skipped_objects: u64,
     /// Versions skipped because they were written after this heal started
     pub skipped_new_versions: u64,
     /// Versions skipped because lifecycle already selected them for expiry
@@ -44,11 +120,38 @@ pub struct HealProgress {
     pub last_update_time: Option<SystemTime>,
     /// Estimated completion time
     pub estimated_completion_time: Option<SystemTime>,
+    /// Current stage number. Stage updates are intentionally independent from
+    /// the object ledger below.
+    #[serde(default)]
+    pub stage_current: u64,
+    /// Number of stages in the current task.
+    #[serde(default)]
+    pub stage_total: u64,
+    /// Explicitly distinguishes a missing usage baseline from measured 0%.
+    #[serde(default)]
+    pub progress_state: HealProgressState,
+    /// True only after the task's durable completion ledger was committed.
+    #[serde(default)]
+    pub ledger_complete: bool,
+    /// Generation of the usage snapshot used for the baseline, if available.
+    #[serde(default)]
+    pub baseline_generation: Option<u64>,
+    /// Whether the baseline was explicitly observed.  This is separate from
+    /// the counters so a known empty scope (0 objects, 0 bytes) is not
+    /// confused with a legacy snapshot that omitted the baseline fields.
+    #[serde(default)]
+    pub baseline_known: bool,
+    /// Internal telemetry fence set when an aggregate counter overflows or
+    /// becomes inconsistent.  It prevents a later refresh from fabricating a
+    /// percentage from the poisoned values.
+    #[serde(default)]
+    pub counter_unknown: bool,
 }
 
 impl HealProgress {
     pub fn new() -> Self {
         Self {
+            kind: HealProgressKind::Unknown,
             start_time: Some(SystemTime::now()),
             last_update_time: Some(SystemTime::now()),
             ..Default::default()
@@ -56,12 +159,87 @@ impl HealProgress {
     }
 
     pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
+        self.update_object_sweep_progress(scanned, healed, failed, bytes);
+    }
+
+    pub fn update_object_sweep_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
+        self.kind = HealProgressKind::ObjectSweep;
         self.objects_scanned = scanned;
         self.objects_healed = healed;
         self.objects_failed = failed;
         self.bytes_processed = bytes;
         self.last_update_time = Some(SystemTime::now());
 
+        let explicit_skipped = match self.skipped_new_versions.checked_add(self.skipped_ilm_expired) {
+            Some(value) => value,
+            None => {
+                self.mark_unknown();
+                0
+            }
+        };
+        let skipped = healed
+            .checked_add(failed)
+            .and_then(|value| value.checked_add(explicit_skipped))
+            .and_then(|value| scanned.checked_sub(value))
+            .unwrap_or(0);
+        self.update_object_progress(scanned, healed, failed, skipped, bytes);
+    }
+
+    /// Update task stage progress without modifying object counters.
+    pub fn update_stage(&mut self, current: u64, total: u64) {
+        let object_sweep_active = matches!(self.kind, HealProgressKind::ObjectSweep);
+        if !object_sweep_active {
+            self.kind = HealProgressKind::Stage;
+        }
+        self.ledger_complete = false;
+        self.stage_current = current.min(total);
+        self.stage_total = total;
+        if object_sweep_active {
+            self.last_update_time = Some(SystemTime::now());
+            self.refresh_progress_percentage();
+            return;
+        }
+        self.progress_state = if total == 0 {
+            HealProgressState::Indeterminate
+        } else {
+            HealProgressState::Running
+        };
+        self.progress_percentage = if total == 0 {
+            0.0
+        } else {
+            (current as f64 / total as f64 * 100.0).min(100.0)
+        };
+        self.last_update_time = Some(SystemTime::now());
+    }
+
+    /// Update the disjoint object ledger. `scanned` is the number of terminal
+    /// object outcomes and must equal healed + failed + deferred skipped plus
+    /// the two terminal skip classes. Overflow is a corrupt/unknown counter
+    /// state, not a reason to abort a completed heal.
+    pub fn update_object_progress(&mut self, scanned: u64, healed: u64, failed: u64, skipped: u64, bytes: u64) {
+        self.kind = HealProgressKind::ObjectSweep;
+        // `skipped` is the transient/deferred class.  The two explicit skip
+        // counters are terminal classifications too, so include them in the
+        // same ledger without making callers maintain a second aggregate.
+        let outcomes = healed
+            .checked_add(failed)
+            .and_then(|value| value.checked_add(skipped))
+            .and_then(|value| value.checked_add(self.skipped_new_versions))
+            .and_then(|value| value.checked_add(self.skipped_ilm_expired));
+        self.objects_scanned = scanned;
+        self.objects_healed = healed;
+        self.objects_failed = failed;
+        self.skipped_objects = skipped;
+        self.bytes_processed = bytes;
+        self.last_update_time = Some(SystemTime::now());
+        self.ledger_complete = false;
+        if outcomes != Some(scanned) {
+            // Telemetry corruption must not abort a heal.  Preserve the
+            // counters for diagnostics, but do not derive a percentage from a
+            // double-counted or overflowing ledger.
+            self.mark_unknown();
+            return;
+        }
         self.refresh_progress_percentage();
         self.refresh_estimated_completion_time();
     }
@@ -69,50 +247,88 @@ impl HealProgress {
     pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
         self.objects_total_count = objects_total_count;
         self.objects_total_size = objects_total_size;
+        self.baseline_known = true;
         self.last_update_time = Some(SystemTime::now());
         self.refresh_progress_percentage();
         self.refresh_estimated_completion_time();
     }
 
+    pub fn set_total_baseline_with_generation(&mut self, objects_total_count: u64, objects_total_size: u64, generation: u64) {
+        self.baseline_generation = Some(generation);
+        self.set_total_baseline(objects_total_count, objects_total_size);
+    }
+
     pub fn record_skipped_new_version(&mut self) {
-        self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
+        let Some(next) = self.skipped_new_versions.checked_add(1) else {
+            self.mark_unknown();
+            return;
+        };
+        self.skipped_new_versions = next;
         self.last_update_time = Some(SystemTime::now());
         self.refresh_progress_percentage();
         self.refresh_estimated_completion_time();
     }
 
     pub fn record_skipped_ilm_expired(&mut self) {
-        self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
+        let Some(next) = self.skipped_ilm_expired.checked_add(1) else {
+            self.mark_unknown();
+            return;
+        };
+        self.skipped_ilm_expired = next;
         self.last_update_time = Some(SystemTime::now());
         self.refresh_progress_percentage();
         self.refresh_estimated_completion_time();
     }
 
-    fn completed_for_baseline(&self) -> u64 {
+    fn completed_for_baseline(&self) -> Option<u64> {
         self.objects_healed
-            .saturating_add(self.objects_failed)
-            .saturating_add(self.skipped_new_versions)
-            .saturating_add(self.skipped_ilm_expired)
+            .checked_add(self.objects_failed)?
+            .checked_add(self.skipped_objects)?
+            .checked_add(self.skipped_new_versions)?
+            .checked_add(self.skipped_ilm_expired)
     }
 
     pub(crate) fn refresh_progress_percentage(&mut self) {
+        if self.ledger_complete {
+            self.progress_state = HealProgressState::Completed;
+            self.progress_percentage = 100.0;
+            return;
+        }
+        if self.counter_unknown {
+            self.progress_state = HealProgressState::Unknown;
+            self.progress_percentage = 0.0;
+            return;
+        }
+        if !self.baseline_known {
+            self.progress_state = HealProgressState::Indeterminate;
+            self.progress_percentage = 0.0;
+            self.estimated_completion_time = None;
+            return;
+        }
         if self.objects_total_size > 0 {
             self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
+            self.progress_percentage = self.progress_percentage.min(99.999);
+            self.progress_state = HealProgressState::Running;
             return;
         }
         if self.objects_total_count > 0 {
-            let completed = self.completed_for_baseline();
+            let Some(completed) = self.completed_for_baseline() else {
+                self.progress_state = HealProgressState::Unknown;
+                self.progress_percentage = 0.0;
+                return;
+            };
             self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
+            self.progress_percentage = self.progress_percentage.min(99.999);
+            self.progress_state = HealProgressState::Running;
             return;
         }
-
-        let total = self
-            .objects_scanned
-            .saturating_add(self.objects_healed)
-            .saturating_add(self.objects_failed);
-        if total > 0 {
-            self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
+        if self.baseline_known {
+            self.progress_state = HealProgressState::Running;
+            self.progress_percentage = 0.0;
+            return;
         }
+        self.progress_state = HealProgressState::Indeterminate;
+        self.progress_percentage = 0.0;
     }
 
     pub fn set_current_object(&mut self, object: Option<String>) {
@@ -125,7 +341,11 @@ impl HealProgress {
             self.estimated_completion_time = None;
             return;
         };
-        if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
+        if self.is_completed()
+            || self.progress_percentage <= 0.0
+            || self.progress_percentage >= 100.0
+            || self.bytes_processed == 0
+        {
             self.estimated_completion_time = None;
             return;
         }
@@ -142,18 +362,39 @@ impl HealProgress {
     }
 
     pub fn is_completed(&self) -> bool {
-        if self.progress_percentage >= 100.0 {
-            return true;
-        }
-        if self.objects_total_count > 0 || self.objects_total_size > 0 {
-            return false;
-        }
+        self.ledger_complete
+    }
 
-        self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
+    /// Mark telemetry unknown while allowing the underlying heal operation to
+    /// continue.  This is used for corrupt/overflowing counters at the
+    /// observability boundary; it must never turn a successful heal into an
+    /// execution error.
+    pub fn mark_unknown(&mut self) {
+        self.counter_unknown = true;
+        self.progress_state = HealProgressState::Unknown;
+        self.ledger_complete = false;
+        self.progress_percentage = 0.0;
+        self.estimated_completion_time = None;
+        self.last_update_time = Some(SystemTime::now());
+    }
+
+    /// Mark the object ledger terminal only after the enclosing task has
+    /// committed all durable resume state and cleanup fences.
+    pub fn mark_completed(&mut self) {
+        let telemetry_unknown = self.counter_unknown || self.progress_state == HealProgressState::Unknown;
+        self.ledger_complete = true;
+        if !telemetry_unknown {
+            self.progress_state = HealProgressState::Completed;
+        }
+        self.progress_percentage = 100.0;
+        self.last_update_time = Some(SystemTime::now());
+        self.estimated_completion_time = None;
     }
 
     pub fn get_success_rate(&self) -> f64 {
-        let total = self.objects_healed + self.objects_failed;
+        let Some(total) = self.objects_healed.checked_add(self.objects_failed) else {
+            return 0.0;
+        };
         if total > 0 {
             (self.objects_healed as f64 / total as f64) * 100.0
         } else {
@@ -162,6 +403,101 @@ impl HealProgress {
     }
 }
 
+pub fn aggregate_heal_progress(progresses: impl IntoIterator<Item = HealProgress>) -> Option<HealProgress> {
+    let mut snapshot = HealProgress::default();
+    let mut found = false;
+    let mut has_object_sweep = false;
+    let mut all_object_baselines_known = true;
+    let mut baseline_generation = None;
+    let mut baseline_generation_consistent = true;
+    let mut all_ledgers_complete = true;
+    let mut counter_overflow = false;
+
+    for progress in progresses {
+        found = true;
+        let object_sweep = matches!(progress.kind, HealProgressKind::ObjectSweep);
+        has_object_sweep |= object_sweep;
+        all_ledgers_complete &= progress.ledger_complete;
+        if object_sweep {
+            all_object_baselines_known &= progress.baseline_known;
+            match baseline_generation {
+                None => baseline_generation = Some(progress.baseline_generation),
+                Some(generation) => baseline_generation_consistent &= generation == progress.baseline_generation,
+            }
+        }
+        counter_overflow |= progress.counter_unknown || matches!(progress.progress_state, HealProgressState::Unknown);
+        for (target, value) in [
+            (&mut snapshot.objects_scanned, progress.objects_scanned),
+            (&mut snapshot.objects_healed, progress.objects_healed),
+            (&mut snapshot.objects_failed, progress.objects_failed),
+            (&mut snapshot.skipped_objects, progress.skipped_objects),
+            (&mut snapshot.skipped_new_versions, progress.skipped_new_versions),
+            (&mut snapshot.skipped_ilm_expired, progress.skipped_ilm_expired),
+            (&mut snapshot.objects_total_count, progress.objects_total_count),
+            (&mut snapshot.objects_total_size, progress.objects_total_size),
+            (&mut snapshot.bytes_processed, progress.bytes_processed),
+            (&mut snapshot.stage_current, progress.stage_current),
+            (&mut snapshot.stage_total, progress.stage_total),
+        ] {
+            match target.checked_add(value) {
+                Some(sum) => *target = sum,
+                None => {
+                    *target = u64::MAX;
+                    counter_overflow = true;
+                }
+            }
+        }
+        snapshot.start_time = match (snapshot.start_time, progress.start_time) {
+            (Some(current), Some(next)) => Some(current.min(next)),
+            (None, next) => next,
+            (current, None) => current,
+        };
+        snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
+            (Some(current), Some(next)) => Some(current.max(next)),
+            (None, next) => next,
+            (current, None) => current,
+        };
+        if progress.current_object.is_some() {
+            snapshot.current_object = progress.current_object;
+        }
+    }
+
+    if !found {
+        return None;
+    }
+
+    snapshot.kind = if has_object_sweep {
+        HealProgressKind::ObjectSweep
+    } else {
+        HealProgressKind::Stage
+    };
+    snapshot.baseline_known = has_object_sweep && all_object_baselines_known && baseline_generation_consistent;
+    snapshot.baseline_generation = if snapshot.baseline_known && baseline_generation_consistent {
+        baseline_generation.flatten()
+    } else {
+        None
+    };
+    snapshot.ledger_complete = all_ledgers_complete;
+    snapshot.counter_unknown = counter_overflow;
+    if counter_overflow {
+        snapshot.progress_state = HealProgressState::Unknown;
+        snapshot.progress_percentage = if snapshot.ledger_complete { 100.0 } else { 0.0 };
+    } else if snapshot.ledger_complete {
+        snapshot.progress_state = HealProgressState::Completed;
+        snapshot.progress_percentage = 100.0;
+    } else if has_object_sweep {
+        snapshot.refresh_progress_percentage();
+    } else if snapshot.stage_total == 0 {
+        snapshot.progress_state = HealProgressState::Indeterminate;
+        snapshot.progress_percentage = 0.0;
+    } else {
+        snapshot.progress_state = HealProgressState::Running;
+        snapshot.progress_percentage = ((snapshot.stage_current as f64 / snapshot.stage_total as f64) * 100.0).min(99.999);
+    }
+    snapshot.refresh_estimated_completion_time();
+    Some(snapshot)
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct HealStatistics {
     /// Total heal tasks
@@ -230,6 +566,7 @@ mod tests {
         assert_eq!(progress.objects_scanned, 0);
         assert_eq!(progress.objects_healed, 0);
         assert_eq!(progress.objects_failed, 0);
+        assert_eq!(progress.skipped_objects, 0);
         assert_eq!(progress.skipped_new_versions, 0);
         assert_eq!(progress.skipped_ilm_expired, 0);
         assert_eq!(progress.objects_total_count, 0);
@@ -250,10 +587,8 @@ mod tests {
         assert_eq!(progress.objects_healed, 8);
         assert_eq!(progress.objects_failed, 2);
         assert_eq!(progress.bytes_processed, 1024);
-        // Progress percentage should be calculated based on healed/total
-        // total = scanned + healed + failed = 10 + 8 + 2 = 20
-        // healed/total = 8/20 = 0.4 = 40%
-        assert!((progress.progress_percentage - 40.0).abs() < 0.001);
+        assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
+        assert_eq!(progress.progress_percentage, 0.0);
         assert!(progress.last_update_time.is_some());
     }
 
@@ -262,7 +597,8 @@ mod tests {
         let mut progress = HealProgress::new();
         progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
 
-        progress.update_progress(100, 25, 0, 4096);
+        progress.set_total_baseline(100, 16384);
+        progress.update_progress(25, 25, 0, 4096);
 
         let eta = progress
             .estimated_completion_time
@@ -275,7 +611,7 @@ mod tests {
         let mut progress = HealProgress::new();
         progress.set_total_baseline(10, 8192);
 
-        progress.update_progress(100, 25, 0, 4096);
+        progress.update_progress(25, 25, 0, 4096);
 
         assert!((progress.progress_percentage - 50.0).abs() < 0.001);
     }
@@ -285,7 +621,7 @@ mod tests {
         let mut progress = HealProgress::new();
         progress.set_total_baseline(10, 0);
 
-        progress.update_progress(100, 3, 2, 0);
+        progress.update_progress(5, 3, 2, 0);
 
         assert!((progress.progress_percentage - 50.0).abs() < 0.001);
     }
@@ -295,7 +631,7 @@ mod tests {
         let mut progress = HealProgress::new();
         progress.set_total_baseline(10, 0);
 
-        progress.update_progress(100, 3, 2, 0);
+        progress.update_progress(5, 3, 2, 0);
         progress.record_skipped_new_version();
 
         assert_eq!(progress.skipped_new_versions, 1);
@@ -336,7 +672,8 @@ mod tests {
     fn test_heal_progress_update_progress_all_healed() {
         let mut progress = HealProgress::new();
         // When scanned=0, healed=10, failed=0: total=10, progress = 10/10 = 100%
-        progress.update_progress(0, 10, 0, 2048);
+        progress.update_progress(10, 10, 0, 2048);
+        progress.mark_completed();
 
         // All healed, should be 100%
         assert!((progress.progress_percentage - 100.0).abs() < 0.001);
@@ -394,6 +731,7 @@ mod tests {
         assert_eq!(json["objectsScanned"], 10);
         assert_eq!(json["objectsHealed"], 8);
         assert_eq!(json["objectsFailed"], 2);
+        assert_eq!(json["skippedObjects"], 0);
         assert_eq!(json["skippedNewVersions"], 0);
         assert_eq!(json["skippedIlmExpired"], 0);
         assert_eq!(json["bytesProcessed"], 1024);
@@ -405,6 +743,7 @@ mod tests {
     fn test_heal_progress_is_completed_by_percentage() {
         let mut progress = HealProgress::new();
         progress.update_progress(10, 10, 0, 1024);
+        progress.mark_completed();
 
         assert!(progress.is_completed());
     }
@@ -415,7 +754,7 @@ mod tests {
         progress.objects_scanned = 10;
         progress.objects_healed = 8;
         progress.objects_failed = 2;
-        // healed + failed = 8 + 2 = 10 >= scanned = 10
+        progress.mark_completed();
         assert!(progress.is_completed());
     }
 
@@ -455,6 +794,108 @@ mod tests {
         assert!((progress.get_success_rate() - 100.0).abs() < 0.001);
     }
 
+    #[test]
+    fn single_object_progress_reaches_terminal_100() {
+        let mut progress = HealProgress::new();
+        progress.update_object_progress(1, 1, 0, 0, 128);
+        assert!(!progress.is_completed());
+        progress.mark_completed();
+        assert!(progress.is_completed());
+        assert_eq!(progress.progress_percentage, 100.0);
+    }
+
+    #[test]
+    fn progress_without_baseline_is_indeterminate() {
+        let mut progress = HealProgress::new();
+        progress.update_object_progress(1, 1, 0, 0, 128);
+        assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
+        assert_eq!(progress.progress_percentage, 0.0);
+        assert!(progress.estimated_completion_time.is_none());
+    }
+
+    #[test]
+    fn progress_retry_is_exactly_once() {
+        let mut progress = HealProgress::new();
+        progress.set_total_baseline(1, 128);
+        progress.update_object_progress(1, 1, 0, 0, 128);
+        progress.update_object_progress(1, 1, 0, 0, 128);
+        assert_eq!(progress.objects_scanned, 1);
+        assert_eq!(progress.objects_healed, 1);
+        assert_eq!(progress.bytes_processed, 128);
+    }
+
+    #[test]
+    fn progress_never_triggers_cleanup_before_terminal_ledger_empty() {
+        let mut progress = HealProgress::new();
+        progress.progress_percentage = 100.0;
+        assert!(!progress.is_completed());
+        progress.mark_completed();
+        assert!(progress.is_completed());
+    }
+
+    #[test]
+    fn progress_counter_overflow_is_marked_unknown_without_aborting_completed_heal() {
+        let mut progress = HealProgress::new();
+        progress.update_object_progress(u64::MAX, u64::MAX, 1, 0, 0);
+        assert_eq!(progress.progress_state, HealProgressState::Unknown);
+        progress.mark_completed();
+        assert!(progress.is_completed());
+        assert_eq!(progress.progress_state, HealProgressState::Unknown);
+
+        let aggregate = aggregate_heal_progress([progress]).expect("progress should aggregate");
+        assert!(aggregate.ledger_complete);
+        assert!(aggregate.counter_unknown);
+        assert_eq!(aggregate.progress_state, HealProgressState::Unknown);
+        assert_eq!(aggregate.progress_percentage, 100.0);
+    }
+
+    #[test]
+    fn aggregate_rejects_mixed_baseline_generations() {
+        let progress = |generation| HealProgress {
+            kind: HealProgressKind::ObjectSweep,
+            objects_scanned: 5,
+            objects_total_count: 10,
+            progress_state: HealProgressState::Running,
+            baseline_generation: Some(generation),
+            baseline_known: true,
+            ..Default::default()
+        };
+
+        let aggregate = aggregate_heal_progress([progress(1), progress(2)]).expect("progress should aggregate");
+        assert!(!aggregate.baseline_known);
+        assert_eq!(aggregate.baseline_generation, None);
+        assert_eq!(aggregate.progress_state, HealProgressState::Indeterminate);
+        assert_eq!(aggregate.progress_percentage, 0.0);
+    }
+
+    #[test]
+    fn aggregate_accepts_multiple_sets_from_one_snapshot_generation() {
+        let progress = |objects_scanned| HealProgress {
+            kind: HealProgressKind::ObjectSweep,
+            objects_scanned,
+            objects_total_count: 10,
+            progress_state: HealProgressState::Running,
+            baseline_generation: Some(7),
+            baseline_known: true,
+            ..Default::default()
+        };
+
+        let aggregate = aggregate_heal_progress([progress(5), progress(3)]).expect("progress should aggregate");
+        assert!(aggregate.baseline_known);
+        assert_eq!(aggregate.baseline_generation, Some(7));
+    }
+
+    #[test]
+    fn stage_updates_do_not_double_count_object_outcomes() {
+        let mut progress = HealProgress::new();
+        progress.update_object_progress(2, 1, 0, 1, 256);
+        progress.update_stage(3, 4);
+        assert_eq!(progress.kind, HealProgressKind::ObjectSweep);
+        assert_eq!(progress.objects_scanned, 2);
+        assert_eq!(progress.objects_healed, 1);
+        assert_eq!(progress.skipped_objects, 1);
+    }
+
     #[test]
     fn test_heal_statistics_new() {
         let stats = HealStatistics::new();

crates/heal/src/heal/resume.rs

@@ -28,10 +28,12 @@ use super::{
 };
 
 mod checkpoint;
+mod gc;
 mod replacement;
 mod utils;
 
-pub use checkpoint::{CheckpointManager, ResumeCheckpoint};
+pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
+pub(crate) use gc::ResumeGc;
 pub(crate) use replacement::replacement_target_identities_match;
 use replacement::replacement_targets_match_identities;
 pub use replacement::{
@@ -51,6 +53,7 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
 const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json";
 const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
 pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
+pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
 const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
 const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
@@ -340,6 +343,12 @@ pub struct ResumeState {
     pub failed_objects: u64,
     /// skipped objects
     pub skipped_objects: u64,
+    /// Terminal versions skipped because they were newer than the heal start.
+    #[serde(default)]
+    pub skipped_new_versions: u64,
+    /// Terminal versions handed to lifecycle expiry.
+    #[serde(default)]
+    pub skipped_ilm_expired: u64,
     /// current bucket
     pub current_bucket: Option<String>,
     /// current object
@@ -354,6 +363,24 @@ pub struct ResumeState {
     pub retry_count: u32,
     /// max retries
     pub max_retries: u32,
+    /// Bytes accounted by the object ledger; additive for old snapshots.
+    #[serde(default)]
+    pub processed_bytes: u64,
+    /// Total bytes from a complete usage snapshot, when available.
+    #[serde(default)]
+    pub total_bytes: u64,
+    /// Generation of the usage snapshot used for the baseline.
+    #[serde(default)]
+    pub baseline_generation: Option<u64>,
+    /// Whether the usage baseline is known. Missing in old snapshots means
+    /// indeterminate rather than a measured zero baseline.
+    #[serde(default)]
+    pub baseline_known: bool,
+    /// Persistent telemetry fence for counter/byte overflow or corruption.
+    /// It must survive a restart so a saturated snapshot is never presented as
+    /// a measured percentage on the next resume.
+    #[serde(default)]
+    pub counter_unknown: bool,
 }
 
 impl ResumeState {
@@ -377,6 +404,8 @@ impl ResumeState {
             successful_objects: 0,
             failed_objects: 0,
             skipped_objects: 0,
+            skipped_new_versions: 0,
+            skipped_ilm_expired: 0,
             current_bucket: None,
             current_object: None,
             completed_buckets: Vec::new(),
@@ -384,6 +413,11 @@ impl ResumeState {
             error_message: None,
             retry_count: 0,
             max_retries: 3,
+            processed_bytes: 0,
+            total_bytes: 0,
+            baseline_generation: None,
+            baseline_known: false,
+            counter_unknown: false,
         }
     }
 
@@ -412,6 +446,39 @@ impl ResumeState {
         self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
     }
 
+    pub fn update_progress_with_bytes(
+        &mut self,
+        processed: u64,
+        successful: u64,
+        failed: u64,
+        skipped: u64,
+        processed_bytes: u64,
+    ) {
+        self.update_progress(processed, successful, failed, skipped);
+        self.processed_bytes = processed_bytes;
+    }
+
+    pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
+        self.skipped_new_versions = new_versions;
+        self.skipped_ilm_expired = ilm_expired;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+    }
+
+    pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
+        self.total_objects = total_objects;
+        self.total_bytes = total_bytes;
+        self.baseline_generation = generation;
+        // This method is called only after a complete usage snapshot has been
+        // validated.  A complete but empty snapshot is still a known baseline.
+        self.baseline_known = true;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+    }
+
+    pub fn mark_counter_unknown(&mut self) {
+        self.counter_unknown = true;
+        self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+    }
+
     pub fn set_current_item(&mut self, bucket: Option<String>, object: Option<String>) {
         self.current_bucket = bucket;
         self.current_object = object;
@@ -437,6 +504,7 @@ impl ResumeState {
         if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) {
             self.pending_buckets.remove(pos);
         }
+        self.resume_cursor = None;
         self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
     }
 
@@ -454,6 +522,10 @@ impl ResumeState {
         self.successful_objects = 0;
         self.failed_objects = 0;
         self.skipped_objects = 0;
+        self.skipped_new_versions = 0;
+        self.skipped_ilm_expired = 0;
+        self.processed_bytes = 0;
+        self.counter_unknown = false;
         self.completed = false;
         // A retry re-scans every bucket from the beginning, so the version
         // cursor must be cleared too — otherwise the retry would resume mid-scan.
@@ -476,14 +548,28 @@ impl ResumeState {
     }
 
     pub fn get_progress_percentage(&self) -> f64 {
+        if self.completed {
+            return 100.0;
+        }
+        if self.counter_unknown {
+            return 0.0;
+        }
+        if !self.baseline_known {
+            return 0.0;
+        }
+        if self.total_bytes > 0 {
+            return ((self.processed_bytes as f64 / self.total_bytes as f64) * 100.0).min(99.999);
+        }
         if self.total_objects == 0 {
             return 0.0;
         }
-        (self.processed_objects as f64 / self.total_objects as f64) * 100.0
+        ((self.processed_objects as f64 / self.total_objects as f64) * 100.0).min(99.999)
     }
 
     pub fn get_success_rate(&self) -> f64 {
-        let total = self.successful_objects + self.failed_objects;
+        let Some(total) = self.successful_objects.checked_add(self.failed_objects) else {
+            return 0.0;
+        };
         if total == 0 {
             return 0.0;
         }
@@ -754,6 +840,14 @@ impl ResumeManager {
             state.successful_objects = 0;
             state.failed_objects = 0;
             state.skipped_objects = 0;
+            state.skipped_new_versions = 0;
+            state.skipped_ilm_expired = 0;
+            state.processed_bytes = 0;
+            state.total_objects = 0;
+            state.total_bytes = 0;
+            state.baseline_generation = None;
+            state.baseline_known = false;
+            state.counter_unknown = false;
             state.completed = false;
             state.completed_buckets.clear();
             state.schema_version = CURRENT_RESUME_SCHEMA;
@@ -838,6 +932,41 @@ impl ResumeManager {
         self.save_state_throttled().await
     }
 
+    pub async fn update_progress_with_bytes(
+        &self,
+        processed: u64,
+        successful: u64,
+        failed: u64,
+        skipped: u64,
+        processed_bytes: u64,
+    ) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.update_progress_with_bytes(processed, successful, failed, skipped, processed_bytes);
+        drop(state);
+        self.save_state_throttled().await
+    }
+
+    pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.set_progress_baseline(total_objects, total_bytes, generation);
+        drop(state);
+        self.save_state_throttled().await
+    }
+
+    pub async fn mark_counter_unknown(&self) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.mark_counter_unknown();
+        drop(state);
+        self.save_state().await
+    }
+
+    pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
+        let mut state = self.state.write().await;
+        state.set_skipped_version_counts(new_versions, ilm_expired);
+        drop(state);
+        self.save_state_throttled().await
+    }
+
     /// Set current item. Called once per healed object, so persistence is
     /// throttled: the in-memory state always updates, but the snapshot is only
     /// written every `PERSIST_EVERY_MUTATIONS` calls or `PERSIST_INTERVAL`.
@@ -882,7 +1011,7 @@ impl ResumeManager {
         let mut state = self.state.write().await;
         state.complete_bucket(bucket);
         drop(state);
-        self.save_state_throttled().await
+        self.save_state().await
     }
 
     /// mark task completed

crates/heal/src/heal/resume/checkpoint.rs

@@ -13,26 +13,51 @@
 // limitations under the License.
 
 use crate::{Error, Result};
+use base64::Engine as _;
 use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
 use std::collections::HashSet;
 use std::path::Path;
 use std::sync::{Arc, Mutex};
 use std::time::{SystemTime, UNIX_EPOCH};
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex as AsyncMutex, RwLock};
 use tracing::{debug, warn};
 
-use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
+use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
+use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET};
 use super::{
-    LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
-    validate_resume_task_id,
+    LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE,
+    delete_resume_file, path_to_str, validate_resume_task_id,
 };
 
 const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
+const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256";
+const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5;
 
-/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
-/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
-/// to the new `compose_key` identities, so a stale checkpoint is discarded.
-pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
+/// Current on-disk schema version for `ResumeCheckpoint`. Schema 5 could
+/// persist dedup identities without the aggregate counters needed to restore
+/// them safely, so stale checkpoints are discarded and replayed.
+pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6;
+
+#[derive(Debug, Clone, Copy)]
+pub enum CheckpointObjectOutcome {
+    Processed,
+    Failed,
+    Skipped,
+}
+
+#[derive(Debug)]
+pub struct CheckpointObjectOutcomeRecord {
+    pub object: String,
+    pub outcome: CheckpointObjectOutcome,
+    pub successful: u64,
+    pub failed: u64,
+    pub skipped: u64,
+    pub bytes: u64,
+    pub skipped_new_versions: u64,
+    pub skipped_ilm_expired: u64,
+    pub counter_unknown: bool,
+}
 
 /// resume checkpoint
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -57,6 +82,35 @@ pub struct ResumeCheckpoint {
     pub failed_objects: HashSet<String>,
     /// skipped objects
     pub skipped_objects: HashSet<String>,
+    /// Aggregate object ledger counters restored alongside the dedup sets.
+    #[serde(default)]
+    pub successful_objects: u64,
+    #[serde(default)]
+    pub failed_object_count: u64,
+    #[serde(default)]
+    pub skipped_object_count: u64,
+    #[serde(default)]
+    pub skipped_new_versions: u64,
+    #[serde(default)]
+    pub skipped_ilm_expired: u64,
+    #[serde(default)]
+    pub processed_bytes: u64,
+    #[serde(default)]
+    pub total_objects: u64,
+    #[serde(default)]
+    pub total_bytes: u64,
+    #[serde(default)]
+    pub baseline_generation: Option<u64>,
+    #[serde(default)]
+    pub baseline_known: bool,
+    /// Persistent telemetry fence for counter/byte overflow or corruption.
+    #[serde(default)]
+    pub counter_unknown: bool,
+    /// Integrity digest over the checkpoint with this field set to `None`.
+    /// Keeping it in the checkpoint makes the payload and its authentication
+    /// record one CAS generation instead of two independently-written files.
+    #[serde(default)]
+    pub integrity_digest: Option<String>,
 }
 
 impl ResumeCheckpoint {
@@ -70,6 +124,18 @@ impl ResumeCheckpoint {
             processed_objects: HashSet::new(),
             failed_objects: HashSet::new(),
             skipped_objects: HashSet::new(),
+            successful_objects: 0,
+            failed_object_count: 0,
+            skipped_object_count: 0,
+            skipped_new_versions: 0,
+            skipped_ilm_expired: 0,
+            processed_bytes: 0,
+            total_objects: 0,
+            total_bytes: 0,
+            baseline_generation: None,
+            baseline_known: false,
+            counter_unknown: false,
+            integrity_digest: None,
         }
     }
 
@@ -91,6 +157,34 @@ impl ResumeCheckpoint {
         self.skipped_objects.insert(object);
     }
 
+    pub fn update_progress(&mut self, successful: u64, failed: u64, skipped: u64, bytes: u64) {
+        self.successful_objects = successful;
+        self.failed_object_count = failed;
+        self.skipped_object_count = skipped;
+        self.processed_bytes = bytes;
+    }
+
+    pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
+        self.total_objects = total_objects;
+        self.total_bytes = total_bytes;
+        self.baseline_generation = generation;
+        // The caller has already validated that this is a complete snapshot;
+        // preserve the distinction between a known empty scope and an old
+        // checkpoint that omitted all baseline fields.
+        self.baseline_known = true;
+    }
+
+    pub fn mark_counter_unknown(&mut self) {
+        self.counter_unknown = true;
+        self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+    }
+
+    pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
+        self.skipped_new_versions = new_versions;
+        self.skipped_ilm_expired = ilm_expired;
+        self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+    }
+
     /// Advance past a fully-processed page: objects below `object_index` are
     /// skipped by position on resume, so the per-object sets no longer need
     /// their entries and would otherwise grow with the whole bucket.
@@ -107,6 +201,17 @@ impl ResumeCheckpoint {
         self.update_position(0, 0);
         self.processed_objects.clear();
         self.skipped_objects.clear();
+        self.successful_objects = 0;
+        self.failed_object_count = 0;
+        self.skipped_object_count = 0;
+        self.skipped_new_versions = 0;
+        self.skipped_ilm_expired = 0;
+        self.processed_bytes = 0;
+        self.total_objects = 0;
+        self.total_bytes = 0;
+        self.baseline_generation = None;
+        self.baseline_known = false;
+        self.counter_unknown = false;
         self.failed_objects.clear();
     }
 }
@@ -116,17 +221,111 @@ pub struct CheckpointManager {
     disk: DiskStore,
     checkpoint: Arc<RwLock<ResumeCheckpoint>>,
     throttle: Mutex<PersistThrottle>,
+    save_lock: AsyncMutex<()>,
+    last_saved: Mutex<Option<EcstoreDiskBytes>>,
 }
 
 impl CheckpointManager {
+    fn blocked_path(task_id: &str) -> std::path::PathBuf {
+        Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"))
+    }
+
+    /// Return whether a checkpoint was permanently isolated after a malformed
+    /// or unsupported snapshot was observed.
+    pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool {
+        if validate_resume_task_id(task_id).is_err() {
+            return false;
+        }
+        let blocked_path = Self::blocked_path(task_id);
+        let Ok(path) = path_to_str(&blocked_path) else {
+            return false;
+        };
+        match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
+            Ok(_) => true,
+            Err(crate::heal::DiskError::FileNotFound) => false,
+            Err(_) => true,
+        }
+    }
+
+    /// Validate the checkpoint while enumerating resumable state. This reads
+    /// the checkpoint once and also isolates malformed or unsupported data.
+    pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> Result<bool> {
+        validate_resume_task_id(task_id)?;
+        if Self::is_blocked(disk, task_id).await {
+            return Err(Error::InvalidCheckpoint(format!("Resume task {task_id} has a blocked checkpoint")));
+        }
+        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
+        let Ok(path) = path_to_str(&file_path) else {
+            return Err(Error::InvalidCheckpoint("Resume checkpoint path is not valid UTF-8".to_string()));
+        };
+        match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
+            Ok(bytes) if bytes.is_empty() => Ok(true),
+            Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec())
+                .await
+                .map(|_| true),
+            Err(crate::heal::DiskError::FileNotFound) => Ok(true),
+            Err(error) => Err(error.into()),
+        }
+    }
+
+    async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) {
+        // This marker is intentionally version-agnostic: an unsupported reader
+        // must stop selector retries until an operator cleans up the snapshot.
+        let blocked_path = Self::blocked_path(task_id);
+        let Ok(path) = path_to_str(&blocked_path) else {
+            return;
+        };
+        let result = EcstoreDiskAPI::compare_and_update_file(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            path,
+            None,
+            Some(EcstoreDiskBytes::from_static(b"blocked")),
+        )
+        .await;
+        match result {
+            Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {}
+            Ok(EcstoreConditionalFileUpdate::Missing) => warn!(
+                target: "rustfs::heal::resume",
+                event = EVENT_HEAL_CHECKPOINT_STATE,
+                component = LOG_COMPONENT_HEAL,
+                subsystem = LOG_SUBSYSTEM_RESUME,
+                task_id,
+                state = "blocked_marker_write_failed",
+                error = "marker target disappeared",
+                "Heal checkpoint could not persist its blocked marker"
+            ),
+            Err(error) => warn!(
+                target: "rustfs::heal::resume",
+                event = EVENT_HEAL_CHECKPOINT_STATE,
+                component = LOG_COMPONENT_HEAL,
+                subsystem = LOG_SUBSYSTEM_RESUME,
+                task_id,
+                state = "blocked_marker_write_failed",
+                error = %error,
+                "Heal checkpoint could not persist its blocked marker"
+            ),
+        }
+    }
+
     /// create new checkpoint manager
     pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
         validate_resume_task_id(&task_id)?;
+        let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}");
+        if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await
+            && error != crate::heal::DiskError::VolumeExists
+        {
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Failed to create checkpoint volume: {error}"),
+            });
+        }
         let checkpoint = ResumeCheckpoint::new(task_id);
         let manager = Self {
             disk,
             checkpoint: Arc::new(RwLock::new(checkpoint)),
             throttle: Mutex::new(PersistThrottle::new()),
+            save_lock: AsyncMutex::new(()),
+            last_saved: Mutex::new(None),
         };
 
         // save initial checkpoint
@@ -140,6 +339,7 @@ impl CheckpointManager {
                 error = %e,
                 "Heal checkpoint persistence failed"
             );
+            return Err(e);
         }
         Ok(manager)
     }
@@ -148,21 +348,32 @@ impl CheckpointManager {
     pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
         validate_resume_task_id(task_id)?;
         let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
-        let mut checkpoint: ResumeCheckpoint =
-            serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
-                message: format!("Failed to deserialize checkpoint: {e}"),
-            })?;
+        Self::load_from_data(disk, task_id, checkpoint_data).await
+    }
+
+    async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec<u8>) -> Result<Self> {
+        validate_resume_task_id(task_id)?;
+        let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) {
+            Ok(checkpoint) => checkpoint,
+            Err(error) => {
+                Self::block_invalid_snapshot(&disk, task_id).await;
+                return Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to deserialize checkpoint: {error}"),
+                });
+            }
+        };
         if checkpoint.task_id != task_id {
+            Self::block_invalid_snapshot(&disk, task_id).await;
             return Err(Error::TaskExecutionFailed {
                 message: "Resume checkpoint task id does not match filename".to_string(),
             });
         }
 
-        // A checkpoint from an older schema stored latest-only dedup identities
-        // that are not comparable to the new per-version `compose_key`
-        // identities. Discard the stale sets and position, then stamp the
-        // current schema so the scan restarts cleanly.
+        // Older checkpoints can contain identities that are not comparable to
+        // the current keys or lack their corresponding aggregate counters.
+        // Discard the stale sets and position so the scan restarts cleanly.
         if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
+            Self::block_invalid_snapshot(&disk, task_id).await;
             return Err(Error::TaskExecutionFailed {
                 message: format!(
                     "Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
@@ -170,7 +381,45 @@ impl CheckpointManager {
                 ),
             });
         }
-        if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
+
+        let integrity_verified = if let Some(expected) = checkpoint.integrity_digest.as_deref() {
+            let actual = Self::checkpoint_digest(&Self::serialize_without_digest(&checkpoint)?);
+            if expected != actual {
+                Self::block_invalid_snapshot(&disk, task_id).await;
+                return Err(Error::InvalidCheckpoint(format!(
+                    "Resume checkpoint digest does not match task {task_id}"
+                )));
+            }
+            true
+        } else if checkpoint.schema_version >= CURRENT_CHECKPOINT_SCHEMA {
+            Self::block_invalid_snapshot(&disk, task_id).await;
+            return Err(Error::InvalidCheckpoint(format!(
+                "Resume checkpoint digest is missing for task {task_id}"
+            )));
+        } else {
+            let digest_path = Self::digest_path(task_id);
+            let digest_path = path_to_str(&digest_path)?;
+            match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, digest_path).await {
+                Ok(expected) => {
+                    let actual = Self::checkpoint_digest(&checkpoint_data);
+                    if expected.as_ref() != actual.as_bytes() {
+                        Self::block_invalid_snapshot(&disk, task_id).await;
+                        return Err(Error::InvalidCheckpoint(format!(
+                            "Resume checkpoint digest does not match task {task_id}"
+                        )));
+                    }
+                    true
+                }
+                Err(crate::heal::DiskError::FileNotFound) => false,
+                Err(error) => {
+                    return Err(Error::TaskExecutionFailed {
+                        message: format!("Failed to read checkpoint digest: {error}"),
+                    });
+                }
+            }
+        };
+
+        if checkpoint.schema_version < CHECKPOINT_PER_VERSION_SCHEMA || !integrity_verified {
             warn!(
                 target: "rustfs::heal::resume",
                 event = EVENT_HEAL_CHECKPOINT_STATE,
@@ -185,15 +434,28 @@ impl CheckpointManager {
             checkpoint.processed_objects.clear();
             checkpoint.failed_objects.clear();
             checkpoint.skipped_objects.clear();
+            checkpoint.successful_objects = 0;
+            checkpoint.failed_object_count = 0;
+            checkpoint.skipped_object_count = 0;
+            checkpoint.skipped_new_versions = 0;
+            checkpoint.skipped_ilm_expired = 0;
+            checkpoint.processed_bytes = 0;
+            checkpoint.total_objects = 0;
+            checkpoint.total_bytes = 0;
+            checkpoint.baseline_generation = None;
+            checkpoint.baseline_known = false;
+            checkpoint.counter_unknown = false;
             checkpoint.current_bucket_index = 0;
             checkpoint.current_object_index = 0;
-            checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
         }
+        checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
 
         Ok(Self {
             disk,
             checkpoint: Arc::new(RwLock::new(checkpoint)),
             throttle: Mutex::new(PersistThrottle::new()),
+            save_lock: AsyncMutex::new(()),
+            last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))),
         })
     }
 
@@ -204,7 +466,7 @@ impl CheckpointManager {
         }
         let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
         match path_to_str(&file_path) {
-            Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
+            Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
                 Ok(data) => !data.is_empty(),
                 Err(_) => false,
             },
@@ -225,7 +487,7 @@ impl CheckpointManager {
         self.save_checkpoint_throttled().await
     }
 
-    /// Advance past a completed page and prune the per-object sets, then persist.
+    /// Persist a completed page position while retaining its identities.
     pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;
         checkpoint.complete_page(bucket_index, object_index);
@@ -233,6 +495,35 @@ impl CheckpointManager {
         self.save_checkpoint_throttled().await
     }
 
+    /// Persist the page position while retaining identities until the resume
+    /// cursor is durable.
+    pub async fn advance_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.update_position(bucket_index, object_index);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// Remove the previous page's dedup identities only after its resume cursor
+    /// has been durably exposed.
+    pub async fn prune_completed_page(&self) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.processed_objects.clear();
+        checkpoint.skipped_objects.clear();
+        checkpoint.failed_objects.clear();
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    /// Advance to the next bucket and clear the final page identities after the
+    /// resume state has durably recorded the completed bucket.
+    pub async fn complete_bucket(&self, next_bucket_index: usize) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.complete_page(next_bucket_index, 0);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
     /// Reset the checkpoint to the start of the scan for a retry, then persist.
     pub async fn reset_for_retry(&self) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;
@@ -267,6 +558,62 @@ impl CheckpointManager {
         self.save_checkpoint_if_due().await
     }
 
+    /// Atomically persist an object's dedup identity with its aggregate result.
+    pub async fn record_object_outcome(&self, record: CheckpointObjectOutcomeRecord) -> Result<()> {
+        let CheckpointObjectOutcomeRecord {
+            object,
+            outcome,
+            successful,
+            failed,
+            skipped,
+            bytes,
+            skipped_new_versions,
+            skipped_ilm_expired,
+            counter_unknown,
+        } = record;
+        let mut checkpoint = self.checkpoint.write().await;
+        match outcome {
+            CheckpointObjectOutcome::Processed => checkpoint.add_processed_object(object),
+            CheckpointObjectOutcome::Failed => checkpoint.add_failed_object(object),
+            CheckpointObjectOutcome::Skipped => checkpoint.add_skipped_object(object),
+        }
+        checkpoint.update_progress(successful, failed, skipped, bytes);
+        checkpoint.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired);
+        if counter_unknown {
+            checkpoint.mark_counter_unknown();
+        }
+        drop(checkpoint);
+        self.save_checkpoint_if_due().await
+    }
+
+    pub async fn update_progress(&self, successful: u64, failed: u64, skipped: u64, bytes: u64) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.update_progress(successful, failed, skipped, bytes);
+        drop(checkpoint);
+        self.save_checkpoint_if_due().await
+    }
+
+    pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.set_progress_baseline(total_objects, total_bytes, generation);
+        drop(checkpoint);
+        self.save_checkpoint_throttled().await
+    }
+
+    pub async fn mark_counter_unknown(&self) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.mark_counter_unknown();
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
+    pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        checkpoint.set_skipped_version_counts(new_versions, ilm_expired);
+        drop(checkpoint);
+        self.save_checkpoint_throttled().await
+    }
+
     async fn save_checkpoint_if_due(&self) -> Result<()> {
         let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true);
         if !should_save {
@@ -292,6 +639,8 @@ impl CheckpointManager {
 
         let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
         delete_resume_file(&self.disk, &checkpoint_file).await?;
+        delete_resume_file(&self.disk, &Self::digest_path(&task_id)).await?;
+        delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?;
 
         debug!(
             target: "rustfs::heal::resume",
@@ -307,21 +656,130 @@ impl CheckpointManager {
 
     /// save checkpoint to disk
     async fn save_checkpoint(&self) -> Result<()> {
-        let checkpoint = self.checkpoint.read().await;
+        // Serialize saves and take the snapshot only after acquiring the lock:
+        // a slower writer must not publish a snapshot taken before a newer one.
+        let _save_guard = self.save_lock.lock().await;
+        let checkpoint = self.checkpoint.read().await.clone();
         validate_resume_task_id(&checkpoint.task_id)?;
-        let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
-            message: format!("Failed to serialize checkpoint: {e}"),
-        })?;
+        let unsigned_checkpoint_data = Self::serialize_without_digest(&checkpoint)?;
+        let digest = Self::checkpoint_digest(&unsigned_checkpoint_data);
+        let mut persisted_checkpoint = checkpoint.clone();
+        persisted_checkpoint.integrity_digest = Some(digest);
+        let checkpoint_data =
+            EcstoreDiskBytes::from(serde_json::to_vec(&persisted_checkpoint).map_err(|e| Error::TaskExecutionFailed {
+                message: format!("Failed to serialize checkpoint: {e}"),
+            })?);
 
         let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
 
         let path_str = path_to_str(&file_path)?;
-        self.disk
-            .write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
+        let last_saved = self
+            .last_saved
+            .lock()
+            .map_err(|_| Error::TaskExecutionFailed {
+                message: "Checkpoint save state lock is poisoned; refusing to save".to_string(),
+            })?
+            .clone();
+        let update = EcstoreDiskAPI::compare_and_update_file(
+            self.disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            path_str,
+            last_saved.clone(),
+            Some(checkpoint_data.clone()),
+        )
+        .await
+        .map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to save checkpoint: {e}"),
+        })?;
+
+        let expected = match update {
+            EcstoreConditionalFileUpdate::Updated => None,
+            EcstoreConditionalFileUpdate::Missing => {
+                return Err(Error::TaskExecutionFailed {
+                    message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
+                });
+            }
+            EcstoreConditionalFileUpdate::Mismatch => {
+                // A healthy manager normally completes the CAS above without
+                // another read or JSON parse. Inspect only after a mismatch so
+                // corruption and future schemas cannot be overwritten blindly.
+                let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
+                    Ok(existing) => existing,
+                    Err(crate::heal::DiskError::FileNotFound) => {
+                        return Err(Error::TaskExecutionFailed {
+                            message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
+                        });
+                    }
+                    Err(error) => {
+                        return Err(Error::TaskExecutionFailed {
+                            message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"),
+                        });
+                    }
+                };
+
+                if existing.is_empty() && last_saved.is_none() {
+                    Some(existing)
+                } else {
+                    let current: ResumeCheckpoint = match serde_json::from_slice(&existing) {
+                        Ok(current) => current,
+                        Err(error) => {
+                            Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
+                            return Err(Error::TaskExecutionFailed {
+                                message: format!("Existing checkpoint is corrupt: {error}"),
+                            });
+                        }
+                    };
+                    if current.task_id != checkpoint.task_id {
+                        Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
+                        return Err(Error::TaskExecutionFailed {
+                            message: "Existing checkpoint task id does not match filename".to_string(),
+                        });
+                    }
+                    if current.schema_version > CURRENT_CHECKPOINT_SCHEMA {
+                        Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
+                        return Err(Error::TaskExecutionFailed {
+                            message: format!(
+                                "Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
+                                current.schema_version
+                            ),
+                        });
+                    }
+                    if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) {
+                        return Err(Error::TaskExecutionFailed {
+                            message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress"
+                                .to_string(),
+                        });
+                    }
+                    Some(existing)
+                }
+            }
+        };
+
+        if let Some(expected) = expected {
+            match EcstoreDiskAPI::compare_and_update_file(
+                self.disk.as_ref(),
+                RUSTFS_META_BUCKET,
+                path_str,
+                Some(expected),
+                Some(checkpoint_data.clone()),
+            )
             .await
             .map_err(|e| Error::TaskExecutionFailed {
-                message: format!("Failed to save checkpoint: {e}"),
-            })?;
+                message: format!("Failed to save checkpoint after CAS mismatch: {e}"),
+            })? {
+                EcstoreConditionalFileUpdate::Updated => {}
+                EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {
+                    return Err(Error::TaskExecutionFailed {
+                        message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(),
+                    });
+                }
+            }
+        }
+
+        let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed {
+            message: "Checkpoint save state lock is poisoned after save".to_string(),
+        })?;
+        *last_saved = Some(checkpoint_data);
 
         debug!(
             target: "rustfs::heal::resume",
@@ -341,11 +799,38 @@ impl CheckpointManager {
         let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
 
         let path_str = path_to_str(&file_path)?;
-        disk.read_all(RUSTFS_META_BUCKET, path_str)
+        HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str)
             .await
             .map(|bytes| bytes.to_vec())
             .map_err(|e| Error::TaskExecutionFailed {
                 message: format!("Failed to read checkpoint file: {e}"),
             })
     }
+
+    fn serialize_without_digest(checkpoint: &ResumeCheckpoint) -> Result<Vec<u8>> {
+        let mut unsigned = checkpoint.clone();
+        unsigned.integrity_digest = None;
+        let mut value = serde_json::to_value(&unsigned).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to serialize checkpoint: {e}"),
+        })?;
+        for field in ["processed_objects", "failed_objects", "skipped_objects"] {
+            let Some(values) = value.get_mut(field).and_then(serde_json::Value::as_array_mut) else {
+                return Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to canonicalize checkpoint field: {field}"),
+                });
+            };
+            values.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
+        }
+        serde_json::to_vec(&value).map_err(|e| Error::TaskExecutionFailed {
+            message: format!("Failed to serialize checkpoint: {e}"),
+        })
+    }
+
+    fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
+        base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
+    }
+
+    fn digest_path(task_id: &str) -> std::path::PathBuf {
+        Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_DIGEST_FILE}"))
+    }
 }

crates/heal/src/heal/resume/gc.rs

@@ -0,0 +1,666 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Bounded inspection of heal resume artifacts.
+//!
+//! The durable owner/CAS and quarantine primitives belong to backlog #1927 and
+//! are not part of the current base revision. This module is therefore
+//! deliberately inspect-only. In particular, it must never turn an age check
+//! into a delete: ordinary heal writers still publish raw files on this base,
+//! so a GC-side compare-and-delete would not fence a concurrent claim.
+
+use metrics::counter;
+use std::{
+    collections::BTreeMap,
+    path::{Component, Path},
+    time::{SystemTime, UNIX_EPOCH},
+};
+use tokio::io::AsyncReadExt;
+
+use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskAPI};
+use super::{
+    LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, REPLACEMENT_COMPLETION_PROOF_FILE, REPLACEMENT_INTENT_FILE,
+    REPLACEMENT_INTENT_SEAL_FILE, RESUME_CHECKPOINT_FILE, RESUME_PROGRESS_FILE, RESUME_STATE_FILE, ResumeCheckpoint, ResumeState,
+    checkpoint::CURRENT_CHECKPOINT_SCHEMA,
+};
+use crate::{Error, Result};
+
+const DEFAULT_ENTRY_BUDGET: usize = 256;
+const DEFAULT_BYTE_BUDGET: usize = 4 * 1024 * 1024;
+const GC_METRIC: &str = "rustfs_heal_resume_gc_inspected_total";
+const GC_ERROR_METRIC: &str = "rustfs_heal_resume_gc_inspect_errors_total";
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ResumeGcConfig {
+    /// Maximum number of directory entries considered in one disk pass.
+    pub(crate) max_entries: usize,
+    /// Maximum number of bytes read in one disk pass.
+    pub(crate) max_bytes: usize,
+}
+
+impl Default for ResumeGcConfig {
+    fn default() -> Self {
+        Self {
+            max_entries: DEFAULT_ENTRY_BUDGET,
+            max_bytes: DEFAULT_BYTE_BUDGET,
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct ResumeGcReport {
+    /// Directory entries visited (including malformed entries).
+    pub(crate) inspected: usize,
+    pub(crate) active_skipped: usize,
+    pub(crate) orphaned: usize,
+    /// Records that must be handed to #1927's quarantine owner.
+    pub(crate) quarantine_required: usize,
+    pub(crate) generation_skipped: usize,
+    pub(crate) clock_skew: usize,
+    pub(crate) read_errors: usize,
+    pub(crate) retained: usize,
+    /// True while #1927's durable claim/quarantine capability is unavailable.
+    pub(crate) destructive_disabled: bool,
+    pub(crate) budget_exhausted: bool,
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct ResumeGc {
+    config: ResumeGcConfig,
+    /// Alternate the first namespace so a full ordinary page cannot starve
+    /// replacement recovery when the list API has no continuation token.
+    recovery_first: bool,
+}
+
+impl ResumeGc {
+    #[cfg(test)]
+    fn with_config(config: ResumeGcConfig) -> Self {
+        Self {
+            config,
+            recovery_first: false,
+        }
+    }
+
+    /// Inspect one bounded page from each resume namespace.
+    ///
+    /// The caller owns scheduling and cancellation. A malformed or unreadable
+    /// artifact is reported and retained so a later pass can retry it; no
+    /// individual artifact error aborts the rest of the bounded page.
+    pub(crate) async fn inspect_disk(&mut self, disk: &DiskStore) -> Result<ResumeGcReport> {
+        let mut report = ResumeGcReport {
+            destructive_disabled: true,
+            ..ResumeGcReport::default()
+        };
+        if self.config.max_entries == 0 || self.config.max_bytes == 0 {
+            report.budget_exhausted = true;
+            return Ok(report);
+        }
+
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        let mut bytes_read = 0usize;
+        let recovery_first = self.recovery_first;
+        self.recovery_first = !self.recovery_first;
+        if recovery_first {
+            inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?;
+            if !report.budget_exhausted {
+                inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?;
+            }
+        } else {
+            inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?;
+            if !report.budget_exhausted {
+                inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?;
+            }
+        }
+
+        counter!(GC_METRIC).increment(u64::try_from(report.inspected).unwrap_or(u64::MAX));
+        counter!(GC_ERROR_METRIC).increment(u64::try_from(report.read_errors).unwrap_or(u64::MAX));
+        Ok(report)
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy)]
+struct ArtifactSet {
+    state: bool,
+    checkpoint: bool,
+    progress: bool,
+    replacement_intent: bool,
+    proof: bool,
+    seal: bool,
+    legacy_marker: bool,
+    temporary: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ArtifactKind {
+    State,
+    Checkpoint,
+    Progress,
+    ReplacementIntent,
+    Proof,
+    Seal,
+    LegacyMarker,
+}
+
+impl ArtifactSet {
+    fn add(&mut self, kind: ArtifactKind, temporary: bool) {
+        self.temporary |= temporary;
+        match kind {
+            ArtifactKind::State => self.state = true,
+            ArtifactKind::Checkpoint => self.checkpoint = true,
+            ArtifactKind::Progress => self.progress = true,
+            ArtifactKind::ReplacementIntent => self.replacement_intent = true,
+            ArtifactKind::Proof => self.proof = true,
+            ArtifactKind::Seal => self.seal = true,
+            ArtifactKind::LegacyMarker => self.legacy_marker = true,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct InspectOptions {
+    max_bytes: usize,
+    now: u64,
+}
+
+struct InspectProgress<'a> {
+    bytes_read: &'a mut usize,
+    report: &'a mut ResumeGcReport,
+}
+
+fn replacement_prefix() -> String {
+    super::replacement_recovery_dir().to_string_lossy().into_owned()
+}
+
+async fn inspect_namespace(
+    config: ResumeGcConfig,
+    disk: &DiskStore,
+    prefix: &str,
+    replacement: bool,
+    now: u64,
+    bytes_read: &mut usize,
+    report: &mut ResumeGcReport,
+) -> Result<()> {
+    let remaining = config.max_entries.saturating_sub(report.inspected);
+    if remaining == 0 {
+        report.budget_exhausted = true;
+        return Ok(());
+    }
+    let count = i32::try_from(remaining).unwrap_or(i32::MAX);
+    let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, prefix, count).await {
+        Ok(entries) => entries,
+        Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(()),
+        Err(error) => return Err(error.into()),
+    };
+    entries.sort_unstable();
+
+    let mut artifacts = BTreeMap::<String, ArtifactSet>::new();
+    for entry in entries {
+        if report.inspected >= config.max_entries {
+            report.budget_exhausted = true;
+            break;
+        }
+        report.inspected += 1;
+        let Some((task_id, kind, temporary)) = artifact_name(&entry, replacement) else {
+            report.quarantine_required += 1;
+            report.retained += 1;
+            continue;
+        };
+        artifacts.entry(task_id).or_default().add(kind, temporary);
+    }
+    if report.inspected >= config.max_entries {
+        report.budget_exhausted = true;
+    }
+
+    for (task_id, artifacts) in artifacts {
+        if *bytes_read >= config.max_bytes {
+            report.budget_exhausted = true;
+            break;
+        }
+        let options = InspectOptions {
+            max_bytes: config.max_bytes,
+            now,
+        };
+        let mut progress = InspectProgress { bytes_read, report };
+        inspect_task(options, disk, prefix, replacement, &task_id, artifacts, &mut progress).await?;
+    }
+    Ok(())
+}
+
+async fn inspect_task(
+    options: InspectOptions,
+    disk: &DiskStore,
+    prefix: &str,
+    replacement: bool,
+    task_id: &str,
+    artifacts: ArtifactSet,
+    progress: &mut InspectProgress<'_>,
+) -> Result<()> {
+    let legacy_replacement = !replacement && !artifacts.state && artifacts.replacement_intent;
+    let state_suffix = if replacement || legacy_replacement {
+        REPLACEMENT_INTENT_FILE
+    } else {
+        RESUME_STATE_FILE
+    };
+    let state_path = artifact_path(prefix, task_id, state_suffix)?;
+    let state = match read_bounded(disk, &state_path, options.max_bytes, progress.bytes_read).await {
+        ReadOutcome::Missing => {
+            progress.report.orphaned += 1;
+            progress.report.retained += 1;
+            return Ok(());
+        }
+        ReadOutcome::TooLarge => {
+            progress.report.quarantine_required += 1;
+            progress.report.retained += 1;
+            progress.report.budget_exhausted = true;
+            return Ok(());
+        }
+        ReadOutcome::Error => {
+            progress.report.read_errors += 1;
+            progress.report.retained += 1;
+            return Ok(());
+        }
+        ReadOutcome::Bytes(bytes) => bytes,
+    };
+
+    let parsed: ResumeState = match serde_json::from_slice(&state) {
+        Ok(state) => state,
+        Err(_) => {
+            progress.report.quarantine_required += 1;
+            progress.report.retained += 1;
+            return Ok(());
+        }
+    };
+    if parsed.schema_version > super::CURRENT_RESUME_SCHEMA || parsed.task_id != task_id {
+        progress.report.quarantine_required += 1;
+        progress.report.retained += 1;
+        return Ok(());
+    }
+    if persistent_age_seconds(options.now, parsed.last_update).is_none() {
+        progress.report.clock_skew += 1;
+        progress.report.retained += 1;
+        return Ok(());
+    }
+    if let Some(generation) = parsed.replacement_generation.as_deref()
+        && !claim_generation_matches(Some(generation), Some(task_id))
+    {
+        progress.report.generation_skipped += 1;
+        progress.report.retained += 1;
+        return Ok(());
+    }
+
+    if !replacement && artifacts.checkpoint {
+        let checkpoint_path = artifact_path(prefix, task_id, RESUME_CHECKPOINT_FILE)?;
+        match read_bounded(disk, &checkpoint_path, options.max_bytes, progress.bytes_read).await {
+            ReadOutcome::Bytes(bytes) => match serde_json::from_slice::<ResumeCheckpoint>(&bytes) {
+                Ok(checkpoint) if checkpoint.schema_version <= CURRENT_CHECKPOINT_SCHEMA && checkpoint.task_id == task_id => {}
+                _ => {
+                    progress.report.quarantine_required += 1;
+                    progress.report.retained += 1;
+                }
+            },
+            ReadOutcome::Missing => {
+                progress.report.orphaned += 1;
+                progress.report.retained += 1;
+            }
+            ReadOutcome::TooLarge => {
+                progress.report.quarantine_required += 1;
+                progress.report.retained += 1;
+                progress.report.budget_exhausted = true;
+            }
+            ReadOutcome::Error => {
+                progress.report.read_errors += 1;
+                progress.report.retained += 1;
+            }
+        }
+    }
+
+    if artifacts.state && artifacts.replacement_intent {
+        // A task cannot have two authoritative state records in one namespace;
+        // preserve both until the durable owner can resolve the generation.
+        progress.report.quarantine_required += 1;
+    }
+
+    if !parsed.completed {
+        progress.report.active_skipped += 1;
+    }
+    // The state and all associated evidence remain recoverable until #1927
+    // supplies a common generation/CAS transition and quarantine owner.
+    progress.report.retained += 1;
+    Ok(())
+}
+
+enum ReadOutcome {
+    Bytes(Vec<u8>),
+    Missing,
+    TooLarge,
+    Error,
+}
+
+async fn read_bounded(disk: &DiskStore, path: &str, max_bytes: usize, bytes_read: &mut usize) -> ReadOutcome {
+    let remaining = max_bytes.saturating_sub(*bytes_read);
+    if remaining == 0 {
+        return ReadOutcome::TooLarge;
+    }
+    let read_len = remaining.saturating_add(1);
+    let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
+        Ok(reader) => reader,
+        Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return ReadOutcome::Missing,
+        Err(_) => return ReadOutcome::Error,
+    };
+    let mut bytes = Vec::with_capacity(read_len.min(64 * 1024));
+    let Ok(read_len) = u64::try_from(read_len) else {
+        return ReadOutcome::TooLarge;
+    };
+    if reader.take(read_len).read_to_end(&mut bytes).await.is_err() {
+        return ReadOutcome::Error;
+    }
+    if bytes.len() > remaining {
+        *bytes_read = max_bytes;
+        return ReadOutcome::TooLarge;
+    }
+    *bytes_read = bytes_read.saturating_add(bytes.len());
+    ReadOutcome::Bytes(bytes)
+}
+
+fn artifact_path(prefix: &str, task_id: &str, suffix: &str) -> Result<String> {
+    if super::validate_resume_task_id(task_id).is_err() {
+        return Err(Error::other("invalid resume task id"));
+    }
+    Path::new(prefix)
+        .join(format!("{task_id}_{suffix}"))
+        .to_str()
+        .map(str::to_owned)
+        .ok_or_else(|| Error::other("invalid resume artifact path"))
+}
+
+/// Parse one directory entry without ever accepting a path component supplied
+/// by a client. DiskAPI filters symlinks, but this check also protects remote
+/// implementations and future mutating callers from traversal/reparse names.
+fn artifact_name(entry: &str, replacement: bool) -> Option<(String, ArtifactKind, bool)> {
+    let path = Path::new(entry);
+    if entry.is_empty() || path.components().count() != 1 || !matches!(path.components().next(), Some(Component::Normal(_))) {
+        return None;
+    }
+    let (stem, temporary) = entry
+        .strip_suffix(".tmp")
+        .map(|stem| (stem, true))
+        .or_else(|| entry.strip_suffix(".bak").map(|stem| (stem, true)))
+        .unwrap_or((entry, false));
+    let suffixes: &[(&str, ArtifactKind)] = if replacement {
+        &[
+            (REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent),
+            (REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof),
+            (REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal),
+        ]
+    } else {
+        &[
+            (RESUME_STATE_FILE, ArtifactKind::State),
+            (RESUME_CHECKPOINT_FILE, ArtifactKind::Checkpoint),
+            (RESUME_PROGRESS_FILE, ArtifactKind::Progress),
+            (LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, ArtifactKind::LegacyMarker),
+            (REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent),
+            (REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof),
+            (REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal),
+        ]
+    };
+    suffixes.iter().find_map(|(suffix, kind)| {
+        stem.strip_suffix(&format!("_{suffix}"))
+            .filter(|task_id| super::validate_resume_task_id(task_id).is_ok())
+            .map(|task_id| (task_id.to_string(), *kind, temporary))
+    })
+}
+
+fn persistent_age_seconds(now: u64, updated: u64) -> Option<u64> {
+    now.checked_sub(updated)
+}
+
+fn claim_generation_matches(observed: Option<&str>, expected: Option<&str>) -> bool {
+    expected.is_none() || observed == expected
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::heal::{DiskOption, Endpoint, new_disk};
+    use tempfile::TempDir;
+    use uuid::Uuid;
+
+    async fn test_disk() -> (TempDir, DiskStore) {
+        let temp = TempDir::new().expect("test disk directory");
+        let endpoint = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("test endpoint");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("test disk");
+        match disk.make_volume(RUSTFS_META_BUCKET).await {
+            Ok(()) | Err(DiskError::VolumeExists) => {}
+            Err(error) => panic!("metadata volume: {error}"),
+        }
+        match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await {
+            Ok(()) | Err(DiskError::VolumeExists) => {}
+            Err(error) => panic!("resume volume: {error}"),
+        }
+        (temp, disk)
+    }
+
+    async fn write_state(disk: &DiskStore, state: &ResumeState) {
+        let path = format!("{BUCKET_META_PREFIX}/{}_{}", state.task_id, RESUME_STATE_FILE);
+        disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into())
+            .await
+            .expect("resume state");
+    }
+
+    async fn write_replacement_state(disk: &DiskStore, state: &ResumeState) {
+        let volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/ahm-replacement");
+        match disk.make_volume(&volume).await {
+            Ok(()) | Err(DiskError::VolumeExists) => {}
+            Err(error) => panic!("replacement volume: {error}"),
+        }
+        let path = format!("{}/{}_{}", replacement_prefix(), state.task_id, REPLACEMENT_INTENT_FILE);
+        disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into())
+            .await
+            .expect("replacement state");
+    }
+
+    #[tokio::test]
+    async fn production_gc_does_not_delete_claimed_resume_state() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        write_state(&disk, &ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![])).await;
+        let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
+        assert_eq!(report.active_skipped, 1);
+        assert!(
+            disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
+                .await
+                .is_ok()
+        );
+    }
+
+    #[tokio::test]
+    async fn production_gc_generation_mismatch_is_skip() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
+        state.replacement_generation = Some(Uuid::new_v4().to_string());
+        write_state(&disk, &state).await;
+        assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().generation_skipped, 1);
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn production_gc_rejects_symlink_or_outside_prefix() {
+        let (_temp, disk) = test_disk().await;
+        let id = Uuid::new_v4().to_string();
+        let root = EcstoreDiskAPI::path(disk.as_ref());
+        let outside = root.join("outside-resume-state");
+        std::fs::write(&outside, b"must remain").expect("outside fixture");
+        let symlink = root
+            .join(RUSTFS_META_BUCKET)
+            .join(BUCKET_META_PREFIX)
+            .join(format!("{id}_{RESUME_STATE_FILE}"));
+        std::os::unix::fs::symlink(&outside, &symlink).expect("symlink fixture");
+        let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
+        assert_eq!(report.inspected, 0, "symlinks are not eligible artifacts");
+        assert!(outside.exists());
+        assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some());
+        assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none());
+        assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none());
+    }
+
+    #[cfg(not(unix))]
+    #[test]
+    fn production_gc_rejects_symlink_or_outside_prefix() {
+        let id = Uuid::new_v4().to_string();
+        assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some());
+        assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none());
+        assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none());
+    }
+
+    #[tokio::test]
+    async fn production_gc_delete_failure_leaves_recoverable_state() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
+        state.mark_completed();
+        write_state(&disk, &state).await;
+        ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
+        assert!(
+            disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
+                .await
+                .is_ok()
+        );
+    }
+
+    #[tokio::test]
+    async fn production_gc_handles_clock_skew_and_restart() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]);
+        state.last_update = u64::MAX;
+        write_state(&disk, &state).await;
+        assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().clock_skew, 1);
+        assert!(persistent_age_seconds(1, 2).is_none());
+    }
+
+    #[tokio::test]
+    async fn production_gc_100k_states_respects_budget() {
+        let (_temp, disk) = test_disk().await;
+        for _ in 0..8 {
+            let state = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]);
+            write_state(&disk, &state).await;
+        }
+        let config = ResumeGcConfig {
+            max_entries: 2,
+            max_bytes: usize::MAX,
+        };
+        let report = ResumeGc::with_config(config).inspect_disk(&disk).await.unwrap();
+        assert!(report.inspected <= 2);
+        assert!(report.budget_exhausted);
+    }
+
+    #[tokio::test]
+    async fn production_gc_recovery_namespace_is_not_starved() {
+        let (_temp, disk) = test_disk().await;
+        let ordinary = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]);
+        write_state(&disk, &ordinary).await;
+        let replacement_id = Uuid::new_v4().to_string();
+        let mut replacement = ResumeState::new(replacement_id, "set".into(), "disk".into(), vec![]);
+        replacement.replacement_generation = Some(replacement.task_id.clone());
+        write_replacement_state(&disk, &replacement).await;
+
+        let mut gc = ResumeGc::with_config(ResumeGcConfig {
+            max_entries: 1,
+            max_bytes: usize::MAX,
+        });
+        assert_eq!(gc.inspect_disk(&disk).await.unwrap().inspected, 1);
+        let second = gc.inspect_disk(&disk).await.unwrap();
+        assert_eq!(second.inspected, 1, "the next bounded pass must start at recovery");
+        assert_eq!(second.active_skipped, 1);
+    }
+
+    #[tokio::test]
+    async fn production_gc_pairs_orphan_checkpoint_and_resume() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let checkpoint = ResumeCheckpoint::new(task_id.clone());
+        let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+        disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&checkpoint).unwrap().into())
+            .await
+            .expect("checkpoint");
+        assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().orphaned, 1);
+    }
+
+    #[tokio::test]
+    async fn production_gc_does_not_delete_slow_active_task() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]);
+        state.last_update = 1;
+        write_state(&disk, &state).await;
+        assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().active_skipped, 1);
+    }
+
+    #[tokio::test]
+    async fn production_gc_disables_on_mixed_version_capability() {
+        assert!(claim_generation_matches(None, None));
+        assert!(!claim_generation_matches(Some("new"), Some("old")));
+        // No #1927 capability means this implementation has no delete path.
+        assert!(ResumeGcConfig::default().max_entries > 0);
+        let (_temp, disk) = test_disk().await;
+        let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect");
+        assert!(report.destructive_disabled);
+    }
+
+    #[tokio::test]
+    async fn production_gc_future_schema_is_not_mtime_deleted() {
+        let (_temp, disk) = test_disk().await;
+        let task_id = Uuid::new_v4().to_string();
+        let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]);
+        state.schema_version = super::super::CURRENT_RESUME_SCHEMA + 1;
+        write_state(&disk, &state).await;
+        let report = ResumeGc::default().inspect_disk(&disk).await.unwrap();
+        assert_eq!(report.quarantine_required, 1);
+        assert!(
+            disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"))
+                .await
+                .is_ok()
+        );
+    }
+
+    #[tokio::test]
+    async fn production_gc_quarantine_cleanup_is_bounded() {
+        let (_temp, disk) = test_disk().await;
+        for _ in 0..4 {
+            let task_id = Uuid::new_v4().to_string();
+            let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}");
+            disk.write_all(RUSTFS_META_BUCKET, &path, b"corrupt".to_vec().into())
+                .await
+                .expect("corrupt state");
+        }
+        let report = ResumeGc::with_config(ResumeGcConfig {
+            max_entries: 2,
+            max_bytes: 1024,
+        })
+        .inspect_disk(&disk)
+        .await
+        .expect("inspect");
+        assert!(report.quarantine_required <= 2);
+        assert!(report.budget_exhausted);
+    }
+}

crates/heal/src/heal/resume/tests.rs

@@ -1296,6 +1296,7 @@ async fn test_resume_state_progress() {
     assert_eq!(progress, 0.0); // total_objects is 0
 
     state.total_objects = 100;
+    state.baseline_known = true;
     let progress = state.get_progress_percentage();
     assert_eq!(progress, 10.0);
 }
@@ -1475,6 +1476,40 @@ fn test_checkpoint_object_sets_dedupe_and_prune() {
     assert!(checkpoint.failed_objects.is_empty());
 }
 
+#[tokio::test]
+async fn checkpoint_page_commit_keeps_ledger_until_cursor_is_durable() {
+    let (_temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
+
+    checkpoint
+        .record_object_outcome(CheckpointObjectOutcomeRecord {
+            object: "bucket/object:v1".to_string(),
+            outcome: CheckpointObjectOutcome::Processed,
+            successful: 1,
+            failed: 0,
+            skipped: 0,
+            bytes: 128,
+            skipped_new_versions: 0,
+            skipped_ilm_expired: 0,
+            counter_unknown: false,
+        })
+        .await
+        .unwrap();
+    checkpoint.advance_page(0, 1).await.unwrap();
+
+    let reloaded = CheckpointManager::load_from_disk(disk.clone(), &task_id).await.unwrap();
+    let snapshot = reloaded.get_checkpoint().await;
+    assert_eq!(snapshot.current_object_index, 1);
+    assert_eq!(snapshot.successful_objects, 1);
+    assert_eq!(snapshot.processed_bytes, 128);
+    assert!(snapshot.processed_objects.contains("bucket/object:v1"));
+
+    checkpoint.prune_completed_page().await.unwrap();
+    let reloaded = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
+    assert!(reloaded.get_checkpoint().await.processed_objects.is_empty());
+}
+
 #[test]
 fn test_checkpoint_loads_legacy_vec_format() {
     // Checkpoints written before the HashSet migration stored the object
@@ -1568,14 +1603,14 @@ async fn test_resumestate_schema_v0_discarded_on_load() {
 }
 
 #[tokio::test]
-async fn test_checkpoint_schema_v4_discarded_on_load() {
+async fn test_checkpoint_schema_v5_discarded_on_load() {
     let (temp_dir, disk) = schema_test_disk().await;
 
-    // The previous checkpoint schema is unsafe once its paired resume
-    // state is discarded: retaining either position would skip work.
+    // Schema v5 can persist failed identities without the aggregate counters
+    // that make those identities safe to deduplicate after an upgrade.
     let task_id = "00000000-0000-4000-8000-000000000002";
     let legacy = r#"{
-            "schema_version": 4,
+            "schema_version": 5,
             "task_id": "00000000-0000-4000-8000-000000000002",
             "checkpoint_time": 1700000000,
             "current_bucket_index": 2,
@@ -1600,6 +1635,32 @@ async fn test_checkpoint_schema_v4_discarded_on_load() {
     temp_dir.close().expect("remove schema test directory");
 }
 
+#[tokio::test]
+async fn downgraded_unsigned_checkpoint_resets_untrusted_progress() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
+    manager.add_processed_object("victim-a".to_string()).await.unwrap();
+    manager.update_position(2, 500).await.unwrap();
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    let bytes = disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path).await.unwrap();
+    let mut downgraded: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
+    downgraded["schema_version"] = serde_json::json!(CURRENT_CHECKPOINT_SCHEMA - 1);
+    downgraded.as_object_mut().unwrap().remove("integrity_digest");
+    downgraded["processed_objects"] = serde_json::json!(["victim-b"]);
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&downgraded).unwrap().into())
+        .await
+        .expect("write downgraded checkpoint");
+
+    let manager = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
+    let checkpoint = manager.get_checkpoint().await;
+    assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA);
+    assert_eq!(checkpoint.current_bucket_index, 0);
+    assert_eq!(checkpoint.current_object_index, 0);
+    assert!(checkpoint.processed_objects.is_empty());
+    temp_dir.close().unwrap();
+}
+
 #[tokio::test]
 async fn current_normal_resume_schema_preserves_progress() {
     let (temp_dir, disk) = schema_test_disk().await;
@@ -1639,6 +1700,120 @@ async fn current_normal_resume_schema_preserves_progress() {
     temp_dir.close().expect("remove schema test directory");
 }
 
+#[test]
+fn progress_checkpoint_restores_bytes_and_generation() {
+    let mut checkpoint = ResumeCheckpoint::new("progress-checkpoint".to_string());
+    checkpoint.set_progress_baseline(9, 4096, Some(77));
+    checkpoint.update_progress(4, 1, 2, 2048);
+    checkpoint.set_skipped_version_counts(3, 1);
+    checkpoint.mark_counter_unknown();
+
+    let restored: ResumeCheckpoint =
+        serde_json::from_slice(&serde_json::to_vec(&checkpoint).expect("serialize checkpoint")).expect("deserialize checkpoint");
+    assert_eq!(restored.processed_bytes, 2048);
+    assert_eq!(restored.total_objects, 9);
+    assert_eq!(restored.total_bytes, 4096);
+    assert_eq!(restored.baseline_generation, Some(77));
+    assert!(restored.baseline_known);
+    assert_eq!(restored.skipped_new_versions, 3);
+    assert_eq!(restored.skipped_ilm_expired, 1);
+    assert!(restored.counter_unknown);
+}
+
+#[test]
+fn old_progress_schema_migrates_missing_fields_to_unknown() {
+    let state = ResumeState::new(
+        "legacy-progress".to_string(),
+        "erasure_set".to_string(),
+        "pool_0_set_0".to_string(),
+        Vec::new(),
+    );
+    let mut value = serde_json::to_value(state).expect("serialize legacy-compatible state");
+    let object = value.as_object_mut().expect("state must be an object");
+    for field in [
+        "processed_bytes",
+        "total_bytes",
+        "baseline_generation",
+        "baseline_known",
+        "skipped_new_versions",
+        "skipped_ilm_expired",
+    ] {
+        object.remove(field);
+    }
+    object.insert("total_objects".to_string(), serde_json::json!(10));
+    object.insert("processed_objects".to_string(), serde_json::json!(5));
+    let restored: ResumeState = serde_json::from_value(value).expect("deserialize old progress state");
+    assert_eq!(restored.processed_bytes, 0);
+    assert_eq!(restored.total_bytes, 0);
+    assert_eq!(restored.baseline_generation, None);
+    assert!(!restored.baseline_known, "missing baseline must remain unknown");
+    assert_eq!(restored.get_progress_percentage(), 0.0);
+    assert_eq!(restored.skipped_new_versions, 0);
+    assert_eq!(restored.skipped_ilm_expired, 0);
+}
+
+#[test]
+fn progress_counter_unknown_survives_resume_round_trip() {
+    let mut state = ResumeState::new(
+        "overflow-progress".to_string(),
+        "erasure_set".to_string(),
+        "pool_0_set_0".to_string(),
+        Vec::new(),
+    );
+    state.mark_counter_unknown();
+
+    let restored: ResumeState =
+        serde_json::from_slice(&serde_json::to_vec(&state).expect("serialize resume state")).expect("deserialize resume state");
+    assert!(restored.counter_unknown);
+}
+
+#[tokio::test]
+async fn checkpoint_progress_survives_a_torn_resume_summary_write() {
+    let (_temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let _resume = ResumeManager::new(
+        disk.clone(),
+        task_id.clone(),
+        "erasure_set".to_string(),
+        "pool_0_set_0".to_string(),
+        vec!["bucket".to_string()],
+    )
+    .await
+    .expect("resume state should persist");
+    let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("checkpoint should persist");
+
+    // This is the ordering used by the erasure-set loop: the checkpoint is
+    // durable before the summary write.  Stop here to model a crash in the
+    // inter-store window and verify that the recovery authority retains the
+    // telemetry fence and bytes.
+    checkpoint
+        .update_progress(3, 0, 0, 1024)
+        .await
+        .expect("checkpoint progress should persist");
+    checkpoint.mark_counter_unknown().await.expect("unknown fence should persist");
+    checkpoint
+        .update_position(0, 3)
+        .await
+        .expect("checkpoint position should persist");
+
+    let restored_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+        .await
+        .expect("checkpoint should reload")
+        .get_checkpoint()
+        .await;
+    let restored_resume = ResumeManager::load_from_disk(disk, &task_id)
+        .await
+        .expect("resume summary should reload")
+        .get_state()
+        .await;
+    assert!(restored_checkpoint.counter_unknown);
+    assert_eq!(restored_checkpoint.processed_bytes, 1024);
+    assert_eq!(restored_checkpoint.current_object_index, 3);
+    assert!(!restored_resume.counter_unknown, "summary is intentionally the torn/older store");
+}
+
 #[tokio::test]
 async fn future_resume_and_checkpoint_schemas_are_rejected() {
     let (temp_dir, disk) = schema_test_disk().await;
@@ -1675,6 +1850,369 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() {
     temp_dir.close().expect("remove schema test directory");
 }
 
+#[tokio::test]
+async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    let truncated = b"{\"schema_version\":5,\"task_id\":";
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into())
+        .await
+        .expect("write truncated checkpoint fixture");
+
+    let error = manager
+        .update_position(2, 7)
+        .await
+        .expect_err("a truncated checkpoint must fail closed during save");
+    assert!(error.to_string().contains("Existing checkpoint is corrupt"));
+    assert_eq!(
+        disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+            .await
+            .expect("read truncated checkpoint fixture"),
+        truncated.as_slice()
+    );
+    assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
+    temp_dir.close().expect("remove checkpoint save test directory");
+}
+
+#[tokio::test]
+async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    let mut future = ResumeCheckpoint::new(task_id.clone());
+    future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
+    let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into())
+        .await
+        .expect("write future checkpoint fixture");
+
+    let error = manager
+        .update_position(2, 7)
+        .await
+        .expect_err("a future schema must fail closed during save");
+    assert!(error.to_string().contains("Existing checkpoint schema"));
+    assert_eq!(
+        disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+            .await
+            .expect("read future checkpoint fixture"),
+        future_bytes
+    );
+    assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
+    temp_dir.close().expect("remove future schema test directory");
+}
+
+#[tokio::test]
+async fn checkpoint_digest_rejects_same_length_progress_tampering() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    manager
+        .add_processed_object("victim-a".to_string())
+        .await
+        .expect("persist checkpoint progress");
+    manager.update_position(1, 1).await.expect("flush checkpoint progress");
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    let original = disk
+        .read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+        .await
+        .expect("read checkpoint fixture");
+    let tampered = original
+        .windows(b"victim-a".len())
+        .position(|window| window == b"victim-a")
+        .map(|index| {
+            let mut bytes = original.to_vec();
+            bytes[index..index + b"victim-a".len()].copy_from_slice(b"victim-b");
+            bytes
+        })
+        .expect("checkpoint should contain the processed object");
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, tampered.into())
+        .await
+        .expect("write tampered checkpoint fixture");
+
+    assert!(CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err());
+    assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
+    temp_dir.close().expect("remove digest test directory");
+}
+
+#[tokio::test]
+async fn checkpoint_integrity_survives_missing_legacy_sidecar() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
+    manager.update_position(2, 9).await.unwrap();
+
+    let digest_path = format!("{BUCKET_META_PREFIX}/{task_id}_ahm_checkpoint.sha256");
+    delete_resume_file(&disk, Path::new(&digest_path)).await.unwrap();
+
+    let restored = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
+    let checkpoint = restored.get_checkpoint().await;
+    assert_eq!(checkpoint.current_bucket_index, 2);
+    assert_eq!(checkpoint.current_object_index, 9);
+    assert!(checkpoint.integrity_digest.is_some());
+    temp_dir.close().unwrap();
+}
+
+#[tokio::test]
+async fn checkpoint_integrity_survives_multi_object_reload() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
+    for index in 0..32 {
+        manager.add_processed_object(format!("processed-{index}")).await.unwrap();
+        manager.add_failed_object(format!("failed-{index}")).await.unwrap();
+        manager.add_skipped_object(format!("skipped-{index}")).await.unwrap();
+    }
+    manager.update_position(2, 9).await.unwrap();
+
+    CheckpointManager::load_from_disk(disk, &task_id)
+        .await
+        .expect("a healthy multi-object checkpoint must survive reload");
+    temp_dir.close().unwrap();
+}
+
+#[tokio::test]
+async fn checkpoint_integrity_rejects_a_removed_embedded_digest() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
+    manager.update_position(2, 9).await.unwrap();
+
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    let bytes = disk
+        .read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+        .await
+        .expect("read checkpoint fixture");
+    let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
+    value["current_object_index"] = serde_json::json!(10);
+    value.as_object_mut().unwrap().remove("integrity_digest");
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&value).unwrap().into())
+        .await
+        .expect("write tampered checkpoint fixture");
+
+    assert!(
+        CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err(),
+        "a current checkpoint without its embedded digest must fail closed"
+    );
+    assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
+    temp_dir.close().unwrap();
+}
+
+#[tokio::test]
+async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+    disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new())
+        .await
+        .expect("write empty checkpoint fixture");
+
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("a new manager must rebuild an empty checkpoint");
+    manager
+        .update_position(3, 11)
+        .await
+        .expect("rebuilt checkpoint must remain writable");
+    assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await);
+    temp_dir.close().expect("remove empty checkpoint test directory");
+}
+
+#[tokio::test]
+async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    manager.cleanup().await.expect("delete checkpoint fixture");
+
+    let error = manager
+        .update_position(1, 2)
+        .await
+        .expect_err("an old manager must not resurrect a deleted checkpoint");
+    assert!(error.to_string().contains("removed after this manager saved it"));
+    assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await);
+    temp_dir.close().expect("remove deleted checkpoint test directory");
+}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn checkpoint_cleanup_leaves_no_task_specific_lock_artifact() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    let lock_path = Path::new(BUCKET_META_PREFIX)
+        .join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"))
+        .with_extension("rustfs-cas.lock");
+    let lock_path = temp_dir.path().join(RUSTFS_META_BUCKET).join(lock_path);
+
+    manager.cleanup().await.expect("delete checkpoint fixture");
+
+    assert!(
+        !lock_path.exists(),
+        "successful checkpoint cleanup must not leave a task-specific lock artifact"
+    );
+}
+
+#[tokio::test]
+async fn an_empty_blocked_marker_still_blocks_resume_selection() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create checkpoint manager");
+    let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
+    disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new())
+        .await
+        .expect("write empty blocked marker fixture");
+
+    assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
+    assert!(CheckpointManager::is_resumable(&disk, &task_id).await.is_err());
+    // Recovery requires replacing/cleaning the snapshot, then removing the
+    // marker; ordinary selector retries are intentionally not an unlock path.
+    manager.cleanup().await.expect("clean blocked checkpoint");
+    assert!(!CheckpointManager::is_blocked(&disk, &task_id).await);
+    temp_dir.close().expect("remove empty blocked marker test directory");
+}
+
+#[tokio::test]
+async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let tasks = [
+        (ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()),
+        (ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")),
+    ];
+    for (task_id, marker) in &tasks {
+        ResumeManager::new(
+            disk.clone(),
+            task_id.clone(),
+            "erasure_set".to_string(),
+            "pool_0_set_0".to_string(),
+            vec!["bucket".to_string()],
+        )
+        .await
+        .expect("create healthy resume state");
+        CheckpointManager::new(disk.clone(), task_id.clone())
+            .await
+            .expect("create healthy checkpoint");
+        let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
+        let checkpoint_bytes = disk
+            .read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+            .await
+            .expect("read healthy checkpoint before blocking");
+        let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
+        disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone())
+            .await
+            .expect("write blocked marker");
+
+        assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
+        assert_eq!(
+            disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
+                .await
+                .expect("read healthy checkpoint after blocking"),
+            checkpoint_bytes
+        );
+    }
+    temp_dir.close().expect("remove blocked selector test directory");
+}
+
+#[tokio::test]
+async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let task_id = ResumeUtils::generate_task_id();
+    let first = CheckpointManager::new(disk.clone(), task_id.clone())
+        .await
+        .expect("create first checkpoint manager");
+    let second = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+        .await
+        .expect("load second checkpoint manager");
+
+    second
+        .update_position(4, 20)
+        .await
+        .expect("persist newer checkpoint progress");
+    let error = first
+        .update_position(1, 3)
+        .await
+        .expect_err("stale checkpoint manager must not overwrite newer progress");
+    assert!(error.to_string().contains("newer progress"));
+
+    let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+        .await
+        .expect("load newer checkpoint progress")
+        .get_checkpoint()
+        .await;
+    assert_eq!(persisted.current_bucket_index, 4);
+    assert_eq!(persisted.current_object_index, 20);
+    temp_dir.close().expect("remove stale manager test directory");
+}
+
+#[tokio::test]
+async fn resumable_selector_isolates_future_and_corrupt_checkpoints() {
+    let (temp_dir, disk) = schema_test_disk().await;
+    let future_task = ResumeUtils::generate_task_id();
+    let corrupt_task = ResumeUtils::generate_task_id();
+    for task_id in [&future_task, &corrupt_task] {
+        ResumeManager::new(
+            disk.clone(),
+            task_id.to_string(),
+            "erasure_set".to_string(),
+            "pool_0_set_0".to_string(),
+            vec!["bucket".to_string()],
+        )
+        .await
+        .expect("create resumable state fixture");
+    }
+
+    let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}");
+    let mut future = ResumeCheckpoint::new(future_task.clone());
+    future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
+    let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
+    disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into())
+        .await
+        .expect("write future checkpoint fixture");
+    let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}");
+    let corrupt_bytes = b"{truncated";
+    disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into())
+        .await
+        .expect("write corrupt checkpoint fixture");
+
+    assert!(CheckpointManager::is_resumable(&disk, &future_task).await.is_err());
+    assert!(CheckpointManager::is_resumable(&disk, &corrupt_task).await.is_err());
+    assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
+    for (task_id, path, bytes) in [
+        (&future_task, future_path, future_bytes),
+        (&corrupt_task, corrupt_path, corrupt_bytes.to_vec()),
+    ] {
+        assert_eq!(
+            disk.read_all(RUSTFS_META_BUCKET, &path)
+                .await
+                .expect("read isolated checkpoint bytes"),
+            bytes
+        );
+        let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
+        assert!(
+            !disk
+                .read_all(RUSTFS_META_BUCKET, &blocked_path)
+                .await
+                .expect("read checkpoint blocked marker")
+                .is_empty()
+        );
+    }
+    temp_dir.close().expect("remove selector isolation test directory");
+}
+
 #[test]
 fn test_persist_throttle_batches_until_threshold() {
     let mut throttle = PersistThrottle::new();

crates/heal/src/heal/resume/utils.rs

@@ -21,7 +21,7 @@ use uuid::Uuid;
 use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
 use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
 use super::{
-    EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
+    CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
     REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
     replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
 };
@@ -67,6 +67,7 @@ impl ResumeUtils {
                 // Extract task ID from filename: {task_id}_ahm_resume_state.json
                 if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
                     && validate_resume_task_id(task_id).is_ok()
+                    && CheckpointManager::is_resumable(disk, task_id).await?
                 {
                     task_ids.push(task_id.to_string());
                 }

crates/heal/src/heal/storage.rs

@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use tracing::{debug, error, warn};
 
+use super::progress::stable_generation;
 use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
 use super::storage_api::storage::{
     BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
@@ -34,6 +35,9 @@ pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
 pub struct HealBucketUsageBaseline {
     pub objects_count: u64,
     pub bytes: u64,
+    /// Stable identity of the validated usage snapshot and selected scope.
+    /// `None` is retained for test/legacy providers that cannot expose one.
+    pub generation: Option<u64>,
 }
 
 pub struct HealLifecycleExpiryContext {
@@ -785,11 +789,52 @@ impl HealStorageAPI for ECStoreHealStorage {
         let mut baseline = HealBucketUsageBaseline::default();
         for bucket in buckets {
             if let Some(usage) = info.buckets_usage.get(bucket) {
-                baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
-                baseline.bytes = baseline.bytes.saturating_add(usage.size);
+                baseline.objects_count = match baseline.objects_count.checked_add(usage.objects_count) {
+                    Some(total) => total,
+                    // A corrupt/overflowing usage snapshot is not a usable
+                    // denominator.  Leave progress indeterminate instead of
+                    // turning saturation into a plausible percentage.
+                    None => return Ok(None),
+                };
+                baseline.bytes = match baseline.bytes.checked_add(usage.size) {
+                    Some(total) => total,
+                    None => return Ok(None),
+                };
             }
         }
 
+        let identity = info.snapshot_identity();
+        let mut canonical = Vec::new();
+        match identity.last_update {
+            Some(last_update) => {
+                canonical.push(1);
+                canonical.extend_from_slice(
+                    &last_update
+                        .duration_since(std::time::UNIX_EPOCH)
+                        .unwrap_or_default()
+                        .as_nanos()
+                        .to_be_bytes(),
+                );
+            }
+            None => canonical.push(0),
+        }
+        for value in [identity.scanner_cycle, identity.scanner_epoch] {
+            match value {
+                Some(value) => {
+                    canonical.push(1);
+                    canonical.extend_from_slice(&value.to_be_bytes());
+                }
+                None => canonical.push(0),
+            }
+        }
+        let mut scope = buckets.to_vec();
+        scope.sort_unstable();
+        for bucket in scope {
+            canonical.extend_from_slice(&(bucket.len() as u64).to_be_bytes());
+            canonical.extend_from_slice(bucket.as_bytes());
+        }
+        baseline.generation = Some(stable_generation(&[&canonical]));
+
         Ok(Some(baseline))
     }
 

crates/heal/src/heal/task.rs

@@ -649,7 +649,7 @@ impl HealTask {
 
         let mut progress = self.progress.write().await;
         progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
-        progress.update_progress(0, 1, 0, 0);
+        progress.update_stage(1, 1);
         Ok(())
     }
 
@@ -733,7 +733,7 @@ impl HealTask {
             "Heal object skipped for data usage cache after transient error"
         );
         let mut progress = self.progress.write().await;
-        progress.update_progress(3, 3, 0, 0);
+        progress.update_stage(3, 3);
         true
     }
 
@@ -757,7 +757,7 @@ impl HealTask {
         );
         let mut progress = self.progress.write().await;
         progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
-        progress.update_progress(4, 4, 0, 0);
+        progress.update_stage(4, 4);
         true
     }
 
@@ -831,6 +831,10 @@ impl HealTask {
 
         match &result {
             Ok(_) => {
+                // A stage can reach its final step before the durable resume
+                // ledger and cleanup fences commit. Publish terminal 100 only
+                // after the enclosing operation has returned success.
+                self.progress.write().await.mark_completed();
                 let mut status = self.status.write().await;
                 *status = HealTaskStatus::Completed;
                 demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {

crates/heal/src/heal/task/heal_bucket.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
 use super::*;
+use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
 
 impl HealTask {
     pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
@@ -32,7 +33,7 @@ impl HealTask {
         {
             let mut progress = self.progress.write().await;
             progress.set_current_object(Some(format!("bucket: {bucket}")));
-            progress.update_progress(0, 3, 0, 0);
+            progress.update_stage(0, 3);
         }
 
         // Step 1: Check if bucket exists
@@ -66,7 +67,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(1, 3, 0, 0);
+            progress.update_stage(1, 3);
         }
 
         // Step 2: Perform bucket heal using ecstore
@@ -122,7 +123,7 @@ impl HealTask {
 
                 if !self.options.recursive {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
                 Ok(())
             }
@@ -142,7 +143,7 @@ impl HealTask {
                 );
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
                 Err(Error::TaskExecutionFailed {
                     message: format!("Failed to heal bucket {bucket}: {e}"),
@@ -245,6 +246,7 @@ impl HealTask {
         let mut scanned = 0u64;
         let mut healed = 0u64;
         let mut failed = 0u64;
+        let mut skipped = 0u64;
         let mut retryable_failed = 0u64;
         let mut permanent_failed = 0u64;
         let mut bytes = 0u64;
@@ -286,16 +288,14 @@ impl HealTask {
                 let mut retry = Vec::with_capacity(pending.len());
                 for item in pending {
                     self.check_control_flags().await?;
+                    let mut telemetry_unknown = false;
                     let object = item.name.as_str();
-                    if retry_attempt == 0 {
-                        scanned = scanned.saturating_add(1);
-                    }
                     {
                         let mut progress = self.progress.write().await;
                         progress.set_current_object(Some(format!("{bucket}/{object}")));
-                        progress.update_progress(scanned, healed, failed, bytes);
                     }
 
+                    let mut terminal_outcome = true;
                     let error = match self
                         .await_with_control(
                             self.storage
@@ -304,13 +304,13 @@ impl HealTask {
                         .await
                     {
                         Ok((result, None)) => {
-                            healed = healed.saturating_add(1);
-                            bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
+                            telemetry_unknown |= !increment_counter(&mut healed);
+                            telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
                             self.record_result_item(result).await;
                             None
                         }
                         Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
-                            healed = healed.saturating_add(1);
+                            telemetry_unknown |= !increment_counter(&mut healed);
                             debug!(
                                 target: "rustfs::heal::task",
                                 event = EVENT_HEAL_BUCKET_RESULT,
@@ -329,6 +329,7 @@ impl HealTask {
 
                     if let Some(err) = error {
                         if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                            telemetry_unknown |= !increment_counter(&mut skipped);
                             warn!(
                                 target: "rustfs::heal::task",
                                 event = EVENT_HEAL_BUCKET_RESULT,
@@ -342,6 +343,7 @@ impl HealTask {
                                 "Heal bucket object repair skipped due to transient metadata error"
                             );
                         } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                            terminal_outcome = false;
                             debug!(
                                 target: "rustfs::heal::task",
                                 event = EVENT_HEAL_BUCKET_RESULT,
@@ -357,7 +359,7 @@ impl HealTask {
                             );
                             retry.push(item);
                         } else {
-                            failed = failed.saturating_add(1);
+                            telemetry_unknown |= !increment_counter(&mut failed);
                             if err.is_recoverable_heal() {
                                 retryable_failed = retryable_failed.saturating_add(1);
                             } else {
@@ -383,8 +385,19 @@ impl HealTask {
                         }
                     }
 
+                    if terminal_outcome {
+                        telemetry_unknown |= !increment_counter(&mut scanned);
+                    }
+
+                    if !terminal_outcome {
+                        continue;
+                    }
+
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(scanned, healed, failed, bytes);
+                    progress.update_object_progress(scanned, healed, failed, skipped, bytes);
+                    if telemetry_unknown {
+                        progress.mark_unknown();
+                    }
                 }
                 pending = retry;
                 retry_attempt = retry_attempt.saturating_add(1);
@@ -432,6 +445,9 @@ impl HealTask {
     }
 
     pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
+        if matches!(self.options.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal) {
+            return Ok(());
+        }
         let baseline = match self
             .await_with_control(self.storage.erasure_set_usage_baseline(buckets))
             .await
@@ -442,9 +458,18 @@ impl HealTask {
             Err(_) => return Ok(()),
         };
 
-        let HealBucketUsageBaseline { objects_count, bytes } = baseline;
+        let HealBucketUsageBaseline {
+            objects_count,
+            bytes,
+            generation,
+        } = baseline;
+        let generation = generation.map(|snapshot_generation| stable_generation(&[&snapshot_generation.to_be_bytes()]));
         let mut progress = self.progress.write().await;
-        progress.set_total_baseline(objects_count, bytes);
+        if let Some(generation) = generation {
+            progress.set_total_baseline_with_generation(objects_count, bytes, generation);
+        } else {
+            progress.set_total_baseline(objects_count, bytes);
+        }
         Ok(())
     }
 }

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -32,7 +32,7 @@ impl HealTask {
         {
             let mut progress = self.progress.write().await;
             progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
-            progress.update_progress(0, 4, 0, 0);
+            progress.update_stage(0, 4);
         }
 
         let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
@@ -248,7 +248,7 @@ impl HealTask {
                         );
                         {
                             let mut progress = self.progress.write().await;
-                            progress.update_progress(4, 4, 0, 0);
+                            progress.update_stage(4, 4);
                         }
                         return Err(Error::TaskExecutionFailed {
                             message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
@@ -304,7 +304,7 @@ impl HealTask {
                 );
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(4, 4, 0, 0);
+                    progress.update_stage(4, 4);
                 }
                 return Err(Error::TaskExecutionFailed {
                     message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
@@ -314,7 +314,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(1, 4, 0, 0);
+            progress.update_stage(1, 4);
         }
 
         // The rebuilt disks are formatted now: mark them as healing so
@@ -343,7 +343,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(2, 4, 0, 0);
+            progress.update_stage(2, 4);
         }
 
         // Step 3: Heal bucket structure
@@ -427,7 +427,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(3, 4, 0, 0);
+            progress.update_stage(3, 4);
         }
 
         // Step 4: Execute erasure set heal with resume
@@ -470,9 +470,7 @@ impl HealTask {
         };
 
         {
-            let mut progress = self.progress.write().await;
-            let bytes_processed = progress.bytes_processed;
-            progress.update_progress(4, 4, 0, bytes_processed);
+            self.progress.write().await.update_stage(4, 4);
         }
 
         match result {

crates/heal/src/heal/task/heal_metadata.rs

@@ -32,7 +32,7 @@ impl HealTask {
         {
             let mut progress = self.progress.write().await;
             progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
-            progress.update_progress(0, 3, 0, 0);
+            progress.update_stage(0, 3);
         }
 
         // Step 1: Check if object exists
@@ -74,7 +74,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(1, 3, 0, 0);
+            progress.update_stage(1, 3);
         }
 
         // Step 2: Perform metadata heal using ecstore
@@ -122,7 +122,7 @@ impl HealTask {
                     );
                     {
                         let mut progress = self.progress.write().await;
-                        progress.update_progress(3, 3, 0, 0);
+                        progress.update_stage(3, 3);
                     }
                     return Err(Error::TaskExecutionFailed {
                         message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -145,7 +145,7 @@ impl HealTask {
 
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
                 self.record_result_item(result).await;
                 Ok(())
@@ -167,7 +167,7 @@ impl HealTask {
                 );
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
                 Err(Error::TaskExecutionFailed {
                     message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -194,7 +194,7 @@ impl HealTask {
         {
             let mut progress = self.progress.write().await;
             progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
-            progress.update_progress(0, 3, 0, 0);
+            progress.update_stage(0, 3);
         }
 
         // Step 1: Check if object exists
@@ -236,7 +236,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(1, 3, 0, 0);
+            progress.update_stage(1, 3);
         }
 
         // Step 2: Perform EC decode heal using ecstore
@@ -284,7 +284,7 @@ impl HealTask {
                     );
                     {
                         let mut progress = self.progress.write().await;
-                        progress.update_progress(3, 3, 0, 0);
+                        progress.update_stage(3, 3);
                     }
                     return Err(Error::TaskExecutionFailed {
                         message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
@@ -309,7 +309,7 @@ impl HealTask {
 
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, object_size);
+                    progress.update_object_progress(1, 1, 0, 0, object_size);
                 }
                 self.record_result_item(result).await;
                 Ok(())
@@ -331,7 +331,7 @@ impl HealTask {
                 );
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
                 Err(Error::TaskExecutionFailed {
                     message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),

crates/heal/src/heal/task/heal_object.rs

@@ -36,7 +36,7 @@ impl HealTask {
         {
             let mut progress = self.progress.write().await;
             progress.set_current_object(Some(format!("{bucket}/{object}")));
-            progress.update_progress(0, 4, 0, 0);
+            progress.update_stage(0, 4);
         }
 
         // Step 1: Check if object exists and get metadata
@@ -132,7 +132,7 @@ impl HealTask {
 
         {
             let mut progress = self.progress.write().await;
-            progress.update_progress(1, 3, 0, 0);
+            progress.update_stage(1, 3);
         }
 
         // Step 2: directly call ecstore to perform heal
@@ -187,7 +187,7 @@ impl HealTask {
                         );
                         {
                             let mut progress = self.progress.write().await;
-                            progress.update_progress(3, 3, 0, 0);
+                            progress.update_stage(3, 3);
                         }
                         return Ok(());
                     }
@@ -207,7 +207,7 @@ impl HealTask {
 
                     {
                         let mut progress = self.progress.write().await;
-                        progress.update_progress(3, 3, 0, 0);
+                        progress.update_stage(3, 3);
                     }
 
                     if Self::should_return_typed_heal_error(&e) {
@@ -249,7 +249,7 @@ impl HealTask {
 
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, object_size);
+                    progress.update_object_progress(1, 1, 0, 0, object_size);
                 }
                 self.record_result_item(result).await;
                 Ok(())
@@ -275,7 +275,7 @@ impl HealTask {
                     );
                     {
                         let mut progress = self.progress.write().await;
-                        progress.update_progress(3, 3, 0, 0);
+                        progress.update_stage(3, 3);
                     }
                     return Ok(());
                 }
@@ -295,7 +295,7 @@ impl HealTask {
 
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(3, 3, 0, 0);
+                    progress.update_stage(3, 3);
                 }
 
                 if Self::should_return_typed_heal_error(&e) {
@@ -414,7 +414,7 @@ impl HealTask {
 
                 {
                     let mut progress = self.progress.write().await;
-                    progress.update_progress(4, 4, 0, object_size);
+                    progress.update_object_progress(1, 1, 0, 0, object_size);
                 }
                 self.record_result_item(result).await;
                 Ok(())

crates/heal/src/heal/task/tests.rs

@@ -22,6 +22,7 @@ use std::sync::Mutex;
 use tempfile::TempDir;
 
 use super::super::storage_api::status::BucketInfo;
+use crate::heal::progress::{HealProgressState, aggregate_heal_progress};
 
 #[tokio::test]
 async fn retry_request_carries_remaining_timeout_budget() {
@@ -2124,6 +2125,7 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
         usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
             objects_count: 10,
             bytes: 8,
+            generation: Some(1),
         })),
         ..Default::default()
     });
@@ -2147,10 +2149,104 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
     let progress = task.get_progress().await;
     assert_eq!(progress.objects_total_count, 10);
     assert_eq!(progress.objects_total_size, 8);
+    assert!(progress.baseline_generation.is_some());
+    assert!(progress.baseline_known);
     assert_eq!(progress.bytes_processed, 2);
     assert!((progress.progress_percentage - 25.0).abs() < 0.001);
 }
 
+#[tokio::test]
+async fn erasure_sets_from_one_usage_snapshot_share_baseline_generation() {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage {
+        usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
+            objects_count: 10,
+            bytes: 8,
+            generation: Some(7),
+        })),
+        ..Default::default()
+    });
+    let buckets = vec!["bucket-a".to_string()];
+    let task_for_set = |set_disk_id: &str| {
+        HealTask::from_request(
+            HealRequest::new(
+                HealType::ErasureSet {
+                    buckets: buckets.clone(),
+                    set_disk_id: set_disk_id.to_string(),
+                },
+                HealOptions::default(),
+                HealPriority::Normal,
+            ),
+            storage.clone(),
+        )
+    };
+    let first = task_for_set("pool_0_set_0");
+    let second = task_for_set("pool_0_set_1");
+
+    first
+        .apply_erasure_set_usage_baseline(&buckets)
+        .await
+        .expect("first baseline");
+    second
+        .apply_erasure_set_usage_baseline(&buckets)
+        .await
+        .expect("second baseline");
+    first.progress.write().await.update_object_progress(0, 0, 0, 0, 0);
+    second.progress.write().await.update_object_progress(0, 0, 0, 0, 0);
+    let first = first.get_progress().await;
+    let second = second.get_progress().await;
+
+    let expected_generation = first.baseline_generation;
+    assert!(expected_generation.is_some());
+    assert_eq!(second.baseline_generation, expected_generation);
+    let aggregate = aggregate_heal_progress([first, second]).expect("aggregate progress");
+    assert!(aggregate.baseline_known);
+    assert_eq!(aggregate.baseline_generation, expected_generation);
+    assert_eq!(aggregate.progress_state, HealProgressState::Running);
+}
+
+#[tokio::test]
+async fn erasure_set_disk_walk_keeps_cluster_usage_baseline_indeterminate() {
+    for (scan_mode, source) in [
+        (HealScanMode::Deep, HealRequestSource::Admin),
+        (HealScanMode::Normal, HealRequestSource::AutoHeal),
+    ] {
+        let temp = TempDir::new().expect("temporary directory should be created");
+        let disk = make_resume_disk(&temp).await;
+        let storage = Arc::new(MockStorage {
+            resume_disk: Mutex::new(Some(disk)),
+            usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
+                objects_count: 10,
+                bytes: 8,
+                generation: Some(1),
+            })),
+            ..Default::default()
+        });
+        let mut request = HealRequest::new(
+            HealType::ErasureSet {
+                buckets: vec!["bucket-a".to_string()],
+                set_disk_id: "pool_0_set_0".to_string(),
+            },
+            HealOptions {
+                scan_mode,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        request.source = source;
+        let task = HealTask::from_request(request, storage);
+
+        task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
+            .await
+            .expect("erasure set heal should complete");
+
+        let progress = task.get_progress().await;
+        assert!(!progress.baseline_known);
+        assert_eq!(progress.baseline_generation, None);
+        assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
+    }
+}
+
 #[tokio::test]
 async fn erasure_set_heal_ignores_usage_baseline_errors() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/heal/src/lib.rs

@@ -19,7 +19,7 @@ pub use error::{Error, Result};
 pub use heal::{
     HealManager, HealOperationsSnapshot, HealOptions, HealPriority, HealPriorityCounts, HealRequest, HealSourceCounts, HealType,
     channel::HealChannelProcessor,
-    progress::HealProgress,
+    progress::{HealProgress, aggregate_heal_progress},
     resume::{ReplacementRecoveryRecord, ReplacementRecoveryState, ResumeUtils},
 };
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;

crates/heal/tests/mrf_pipeline_test.rs

@@ -35,6 +35,7 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
 
 const META_BUCKET: &str = ".rustfs.sys";
 const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
+const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
 
 async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
     let env = rustfs_test_utils::TestECStoreEnv::builder()
@@ -79,14 +80,18 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]
     body
 }
 
-fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
+fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
     for path in disk_paths {
-        let journal = path.join(META_BUCKET).join(JOURNAL_REL);
+        let journal = path.join(META_BUCKET).join(relative_path);
         std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
         std::fs::write(&journal, data).expect("write journal fixture");
     }
 }
 
+fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
+    write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
+}
+
 async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
 where
     F: FnMut() -> Fut,
@@ -187,8 +192,73 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
             .all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
         "the journal file must be removed after a successful replay"
     );
+    assert!(
+        disk_paths
+            .iter()
+            .all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
+        "the authoritative journal file must also be removed after replay"
+    );
 
     let snapshot = manager.operations_snapshot().await;
     assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
     assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
 }
+
+/// A canonical snapshot and its compatibility mirror may differ after a
+/// partial flush. Replay must choose the complete canonical epoch instead of
+/// combining records that never coexisted in memory.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[serial]
+async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
+    let (disk_paths, storage) = heal_env().await;
+    let mut endpoints: Vec<Endpoint> = disk_paths
+        .iter()
+        .map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
+        .collect();
+    for (i, endpoint) in endpoints.iter_mut().enumerate() {
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(i);
+    }
+    let pool = PoolEndpoints {
+        legacy: false,
+        set_count: 1,
+        drives_per_set: endpoints.len(),
+        endpoints: Endpoints::from(endpoints),
+        cmd_line: "mrf-authoritative-test".to_string(),
+        platform: String::new(),
+    };
+    init_local_disks(EndpointServerPools::from(vec![pool]))
+        .await
+        .expect("local disks should register");
+
+    let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0);
+    let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
+    write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative);
+    write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &legacy);
+
+    let manager = make_manager(storage);
+    let replayed = mrf_queue::replay_journal_once(&manager).await;
+    assert_eq!(replayed, 1, "only the authoritative snapshot epoch may replay");
+
+    let snapshot = manager.operations_snapshot().await;
+    assert_eq!(snapshot.queued_by_source.mrf, 1);
+    assert!(
+        disk_paths.iter().all(|path| {
+            !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
+                && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
+        }),
+        "replay cleanup must remove both journal paths"
+    );
+
+    // A scoped-only snapshot is valid during a rollout where no legacy
+    // compatibility mirror was written. Missing legacy files must not leave
+    // the runtime in a permanent cleanup-retry state.
+    let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
+    write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
+    assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
+    assert!(disk_paths.iter().all(|path| {
+        !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
+            && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
+    }));
+}

crates/io-metrics/src/internode_metrics.rs

@@ -61,6 +61,8 @@ pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch
 pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
 pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
 pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT: &str = "batch_read_version_coalescer_wait";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP: &str = "batch_read_version_response_map";
 
 const OPERATION_LABEL: &str = "operation";
 const BACKEND_LABEL: &str = "backend";
@@ -1431,6 +1433,8 @@ mod tests {
         );
         assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip");
         assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode");
+        assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, "batch_read_version_coalescer_wait");
+        assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, "batch_read_version_response_map");
         assert_eq!(
             INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
             "rustfs_system_network_internode_signature_v1_fallback_total"