test(ecstore): cover final sweep cancel fence

fix(ecstore): fence final decommission sweep
fix(ecstore): remove duplicate decommission error helper
2026-08-22 20:36:38 +00:00 · 2026-08-23 03:21:15 +08:00 · 2026-08-23 03:15:33 +08:00 · 2026-08-23 03:00:41 +08:00 · 2026-08-23 02:58:32 +08:00 · 2026-08-23 01:45:15 +08:00
102 changed files with 12385 additions and 1926 deletions
@@ -1,2 +1,2 @@
-sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
-sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
+sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
+sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'

+# The durable ILM decommission regressions build isolated multi-pool stores and
+# deliberately take source or target disks offline while checking fencing.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'

+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -14,9 +14,10 @@

 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
  Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
  appended as a comment; otherwise a new issue is created. This is the
  single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).

@@ -38,6 +39,30 @@ inputs:
      Set to an empty string to skip labeling.
    required: false
    default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
+  details-file:
+    description: "Optional Markdown file appended to the issue body."
+    required: false
+    default: ""

 runs:
  using: "composite"
@@ -48,17 +73,22 @@ runs:
        GH_TOKEN: ${{ inputs.github-token }}
        WORKFLOW_NAME: ${{ inputs.workflow-name }}
        ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
+        DETAILS_FILE: ${{ inputs.details-file }}
      run: |
        set -euo pipefail

        title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"

-        # Failed job names for this run attempt. The alert job runs while the
-        # run as a whole is still in progress, so inspect the jobs that have
-        # already completed with a non-success conclusion.
+        # Inspect the reported run attempt. It can be the current in-workflow
+        # failure or a completed run observed by the external watchdog.
        failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
          --paginate \
          --jq '.jobs[]
                | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
          failed_jobs="- (failed job not recorded yet — see the run page)"
        fi

+        details=""
+        if [ -n "${DETAILS_FILE}" ]; then
+          if [ -f "${DETAILS_FILE}" ]; then
+            details="$(cat "${DETAILS_FILE}")"
+          else
+            details="Details file was not available: \`${DETAILS_FILE}\`"
+          fi
+        fi
+
        body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Run of **${WORKFLOW_NAME}** did not complete successfully.

-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
+        - Event: \`${SOURCE_EVENT}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`

-        Failed jobs:
+        Non-success jobs:
        ${failed_jobs}
+
+        ${details}
        EOF
        )"

@@ -0,0 +1,14 @@
+[
+  { "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+]
@@ -46,7 +46,7 @@ on:
    # advisory could sit unnoticed for seven days. The check list is unchanged —
    # splitting it into a light daily advisories-only run and a weekly full run
    # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '23 3 * * *' # Daily 03:23 UTC
  workflow_dispatch:

 permissions:
@@ -52,7 +52,7 @@ on:
      - ".dockerignore"
      - "flake.lock"
  schedule:
-    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
+    - cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
  workflow_dispatch:
    inputs:
      build_docker:
@@ -1032,3 +1032,23 @@ jobs:

          echo "🎉 Released $TAG successfully!"
          echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -94,6 +94,9 @@ concurrency:

 env:
  CARGO_TERM_COLOR: always
+  # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
+  # ci.yml or the writer and readers use disjoint cache keys.
+  RUST_BACKTRACE: 1

 jobs:
  # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
@@ -101,7 +104,7 @@ jobs:
  warm-ci-dev:
    name: Warm ci-dev
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -191,7 +194,7 @@ jobs:
  warm-ci-feat-rio:
    name: Warm ci-feat-rio
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -219,7 +222,7 @@ jobs:
  warm-ci-feat-proto:
    name: Warm ci-feat-proto
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -126,7 +126,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -59,7 +59,7 @@ on:
  merge_group:
    types: [ checks_requested ]
  schedule:
-    - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
+    - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
  workflow_dispatch:

 permissions:
@@ -161,7 +161,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -1032,3 +1035,37 @@ jobs:
          path: artifacts/s3tests-single/**
          if-no-files-found: ignore
          retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -37,7 +37,7 @@ on:
    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
+    - cron: "43 7 * * 0"

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -40,7 +40,7 @@ on:
  schedule:
    # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
    # stale (01:30) and performance-ab (06:00).
-    - cron: "0 4 * * *"
+    - cron: "29 4 * * *"

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -196,6 +196,9 @@ jobs:
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

+      - name: Verify protocol socket oracle
+        run: ss -tn state CLOSE-WAIT >/dev/null
+
      # The suite owns fixed protocol ports and serializes its internal cases.
      - name: Verify protocol e2e membership
        env:
@@ -90,10 +90,14 @@ on:
        description: "Optional pytest -m expression"
        required: false
        default: ""
+      testexpr:
+        description: "Optional pytest -k expression"
+        required: false
+        default: ""
  schedule:
    # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
    # single-node and the 4-node distributed topologies (matrix below).
-    - cron: "0 2 * * 0"
+    - cron: "19 2 * * 0"

 env:
  # main user
@@ -116,6 +120,7 @@ env:
  XDIST: ${{ github.event.inputs.xdist || '4' }}
  MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
  MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
+  TESTEXPR: ${{ github.event.inputs.testexpr || '' }}
  S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
  TEST_TIMEOUT: "300"

@@ -269,14 +274,20 @@ jobs:
          EOF

          cat > haproxy.cfg <<'EOF'
+          global
+            log stdout format raw local0 info
+
          defaults
            mode http
+            log global
+            log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP'
            timeout connect 5s
            timeout client  30s
            timeout server  30s

          frontend fe_s3
            bind *:9000
+            option http-buffer-request
            default_backend be_s3

          backend be_s3
@@ -314,6 +325,7 @@ jobs:
          XDIST="${XDIST}" \
          MAXFAIL="${MAXFAIL}" \
          MARKEXPR="${MARKEXPR}" \
+          TESTEXPR="${TESTEXPR}" \
          ./scripts/s3-tests/run.sh

      - name: Publish compatibility report
@@ -30,7 +30,7 @@ on:
      - "Cargo.lock"
      - ".github/workflows/fuzz.yml"
  schedule:
-    - cron: "0 2 * * *"
+    - cron: "17 2 * * *"
  workflow_dispatch:
    inputs:
      profile:
@@ -121,3 +121,21 @@ jobs:
          cargo nextest run --run-ignored ignored-only --no-tests=fail \
            -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
            -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -45,13 +45,6 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.

-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: mint

 on:
@@ -70,13 +63,13 @@ on:
          - core
          - full
      mint-image:
-        description: "Mint image reference"
+        description: "Mint image reference (empty = pinned default)"
        required: false
-        default: "minio/mint:edge"
+        default: ""
  schedule:
    # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
    # 3h) has finished, so the two never contend for the same runner pool.
-    - cron: "0 6 * * 0"
+    - cron: "41 6 * * 0"

 env:
  S3_ACCESS_KEY: rustfsadmin-ci
@@ -16,7 +16,7 @@ name: Nightly GNU Build

 on:
  schedule:
-    - cron: "0 0 * * *"
+    - cron: "7 0 * * *"
      timezone: "Asia/Shanghai"
  workflow_dispatch:

@@ -194,3 +194,23 @@ jobs:

      - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
        run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -22,22 +22,15 @@
 # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
 # blocked (rustfs/backlog#935 correction 1).

-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Performance A/B

 on:
  schedule:
-    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
+    - cron: "31 6 * * *" # 06:31 UTC nightly, against main
  workflow_dispatch:
    inputs:
      duration:
-        description: "warp duration per round (short by default to fit the double-build budget)"
+        description: "warp duration per round"
        required: false
        default: "12s"
        type: string
@@ -46,12 +39,8 @@ on:
        required: false
        default: false
        type: boolean
-  push:
-    # Every main commit pre-builds and caches its release binary (perf-3) so the
-    # nightly A/B restores a ready baseline instead of paying the double build.
-    branches: [main]
-
 permissions:
+  actions: read
  contents: read

 env:
@@ -59,83 +48,19 @@ env:
  RUST_BACKTRACE: 1

 jobs:
-  # perf-3: on every push to main, build the release binary once and cache it
-  # keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
-  # restore this instead of paying the ~32min-per-side source
-  # build. That double build is what pushed the expanded 24-cell nightly past its
-  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
-  # builds off the shared cargo cache keep each push cheap, and building on the
-  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
-  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
-  # are cancelled ~7/8 of the time and are not a reliable baseline.
-  build-baseline-cache:
-    name: Build + cache baseline binary
-    if: github.event_name == 'push'
-    runs-on: sm-standard-2
-    # Latest-wins: consumers only ever restore the binary for the *current*
-    # origin/main tip, so when pushes land faster than the ~65min build, a
-    # superseded build's output is dead weight — cancel it instead of stacking
-    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
-    # most costs one same-commit self-heal in the A/B job.
-    concurrency:
-      group: perf-baseline-build-main
-      cancel-in-progress: true
-    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
-    # single release build past 60min on this runner — every cache build on
-    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
-    # time of 1h0m0s") and the cache never populated. The measured binary must
-    # keep the production profile, so the budget absorbs the build instead.
-    timeout-minutes: 100
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Build release rustfs
-        run: cargo build --release --bin rustfs
-
-      - name: Stage binary for cache
-        run: |
-          set -euo pipefail
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-
-      - name: Cache baseline binary by SHA
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ github.sha }}
-
  warp-ab:
    name: Warp A/B budget gate
-    # Always run on schedule / manual dispatch. Never on push — that event only
-    # feeds build-baseline-cache above.
-    if: >-
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch'
    runs-on: sm-standard-2
-    # With perf-3's cached baseline binary the common (cache-hit) nightly is
-    # measurement-only and finishes well under 50min. This ceiling stays
-    # generous only to absorb the same-commit cache-miss self-heal (~65min
-    # single build with the post-#4806 LTO profile + measurement). A timeout
-    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
-    # not just failure). perf-6 recalibrates the budget once the noise study
-    # lands.
-    timeout-minutes: 120
+    # A normal nightly restores the last successful binary and builds only the
+    # candidate; daily access keeps that cache warm. A cache miss may build both
+    # and needs room for the A/B run plus artifact and cache publication.
+    timeout-minutes: 180
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          fetch-depth: 0 # baseline is built from origin/main
+          fetch-depth: 0 # baseline may be an earlier successful scheduled head

      - name: Setup Rust environment
        uses: ./.github/actions/setup
@@ -163,24 +88,55 @@ jobs:
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

-      # perf-3: resolve the commits so the cache can be keyed by SHA. The
-      # baseline is origin/main; the candidate is the checked-out ref. On the
-      # nightly (checkout == main) they are the same commit, so one cached binary
-      # serves both phases and the run does zero source builds.
+      # A failed regression run must keep comparing against the last known-good
+      # scheduled head. Otherwise the next nightly would absorb the regression
+      # into its baseline and turn green without a fix.
+      - name: Find last successful scheduled baseline
+        id: scheduled_baseline
+        if: github.event_name == 'schedule'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        with:
+          result-encoding: string
+          script: |
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "performance-ab.yml",
+              event: "schedule",
+              status: "success",
+              per_page: 1,
+            });
+            return data.workflow_runs[0]?.head_sha ?? "";
+
+      # Manual runs compare a selected ref with current main. Scheduled runs
+      # compare current main with the last successful scheduled head. With no
+      # history, the first run measures the candidate against itself and seeds
+      # that head only if the complete rig succeeds.
      - name: Resolve baseline / candidate commits
        id: commits
+        env:
+          SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
        run: |
          set -euo pipefail
-          baseline_sha="$(git rev-parse origin/main)"
          candidate_sha="$(git rev-parse HEAD)"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
+            if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+              echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
+              exit 1
+            fi
+          else
+            baseline_sha="$(git rev-parse origin/main)"
+          fi
+          git cat-file -e "${baseline_sha}^{commit}"
          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
          echo "baseline commit: $baseline_sha"
          echo "candidate commit: $candidate_sha"

-      # Exact-key restore of the baseline binary built by build-baseline-cache
-      # when origin/main last landed. A miss (binary evicted or not built yet)
-      # leaves cache-hit unset and the rig falls back to a source build.
+      # Exact-key restore of the candidate binary saved by its successful
+      # scheduled run. A miss leaves cache-hit unset and falls back to a source
+      # build of that known-good head.
      - name: Restore cached baseline binary
        id: baseline_cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
@@ -270,11 +226,11 @@ jobs:
            elif [[ "$selfheal_built" == "true" ]]; then
              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
            else
-              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
+              base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
            fi
            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
-              # Nightly on main: the candidate is the same commit as the baseline,
-              # so reuse the one binary for both phases and skip all builds.
+              # No commits landed since the last successful baseline, so reuse
+              # the one binary for both phases and measure only rig drift.
              args+=(--candidate-bin "$base_bin")
              cand_src="same binary as baseline (same commit)"
            elif [[ "$candidate_built" == "true" ]]; then
@@ -362,6 +318,23 @@ jobs:
            fi
          } >> "$GITHUB_STEP_SUMMARY"

+      - name: Stage successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        run: |
+          set -euo pipefail
+          cp candidate-bin/rustfs baseline-bin/rustfs
+
+      - name: Cache successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
+
      # Scheduled failure alerting is handled by the alert-on-failure job below
      # (perf-2 consuming ci-8's schedule-failure-issue composite action).

@@ -30,7 +30,7 @@ name: Runner Hygiene

 on:
  schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+    - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
  workflow_dispatch:

 permissions:
@@ -0,0 +1,57 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Freshness
+
+on:
+  schedule:
+    - cron: "47 23 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scheduled-validation-freshness
+  cancel-in-progress: false
+
+jobs:
+  check-freshness:
+    name: Check scheduled validation freshness
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check latest scheduled runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set +e
+          python3 scripts/check_scheduled_validation_freshness.py \
+            --report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
+          status=$?
+          cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
+          exit "${status}"
+      - name: Open or update freshness issue
+        if: failure()
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}
@@ -48,11 +48,6 @@ pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json";
 // RUSTFS_COMPAT_TODO(scanner-usage-v2): keep .usage.json readable and removable during rolling upgrades from pre-v2 scanners. Remove after supported direct-upgrade sources all write .usage.v2.json.
 pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";

-/// Fixed bucket for objects whose storage class is not in the scanner's
-/// cycle-local tier registry.  Keeping this key fixed prevents untrusted or
-/// stale tier names from growing persisted per-tier maps without bound.
-pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
-
 /// Returns true when `existing_last_update` is ahead of `now` by more than
 /// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
 /// trusted for staleness comparisons and a fresh snapshot save must be allowed.
@@ -83,146 +78,12 @@ impl TierStats {
            && self.num_objects.checked_add(u.num_objects).is_some()
    }

-    /// Add tier counters without allowing a counter to wrap.
-    pub fn checked_add(&self, u: &TierStats) -> Option<TierStats> {
-        Some(TierStats {
-            total_size: self.total_size.checked_add(u.total_size)?,
-            num_versions: self.num_versions.checked_add(u.num_versions)?,
-            num_objects: self.num_objects.checked_add(u.num_objects)?,
-        })
-    }
-
    /// True when this tier contributed nothing, i.e. merging it is a no-op.
    pub fn is_empty(&self) -> bool {
        self.total_size == 0 && self.num_versions == 0 && self.num_objects == 0
    }
 }

-/// Bounded diagnostics for objects whose tier is absent from the cycle
-/// registry. Counters are authoritative; diagnostics are only a small,
-/// redacted reconciliation aid and may be dropped at the configured caps.
-pub const UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP: usize = 64;
-pub const UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP: usize = 4096;
-pub const UNKNOWN_TIER_DIAGNOSTIC_TTL: Duration = Duration::from_secs(60 * 60);
-const UNKNOWN_TIER_DIAGNOSTIC_KEY_BYTES: usize = 256;
-
-#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
-pub struct UnknownTierStats {
-    /// Logical bytes retained in the scanner's normal usage total.
-    pub unknown_bytes: u64,
-    /// Physical bytes recorded in the per-tier accounting dimension.
-    ///
-    /// Older writers only had `unknown_bytes`; decoding those snapshots keeps
-    /// this field at zero and the scanner fills it for new observations.
-    #[serde(default)]
-    pub unknown_physical_bytes: u64,
-    pub unknown_objects: u64,
-    pub unknown_versions: u64,
-    pub diagnostics_dropped: u64,
-    #[serde(default)]
-    pub diagnostics: Vec<String>,
-    #[serde(default)]
-    pub diagnostics_at: Option<SystemTime>,
-}
-
-impl UnknownTierStats {
-    /// Record one observation where the logical and physical dimensions are
-    /// the same. Kept as a small compatibility helper for callers that only
-    /// have one size value.
-    pub fn record(&mut self, tier: &str, bytes: u64, versions: u64, objects: u64) {
-        self.record_dimensions(tier, bytes, bytes, versions, objects);
-    }
-
-    /// Record one observation without conflating logical usage with physical
-    /// tier bytes. Both counters are saturating so malformed metadata cannot
-    /// wrap an aggregate.
-    pub fn record_dimensions(&mut self, tier: &str, logical_bytes: u64, physical_bytes: u64, versions: u64, objects: u64) {
-        self.unknown_bytes = self.unknown_bytes.saturating_add(logical_bytes);
-        self.unknown_physical_bytes = self.unknown_physical_bytes.saturating_add(physical_bytes);
-        self.unknown_objects = self.unknown_objects.saturating_add(objects);
-        self.unknown_versions = self.unknown_versions.saturating_add(versions);
-
-        let digest = {
-            let mut hasher = DefaultHasher::new();
-            // Bound hashing work for hostile metadata while retaining enough
-            // length/prefix entropy to reconcile repeated observations.
-            hasher.write_usize(tier.len());
-            hasher.write(&tier.as_bytes()[..tier.len().min(UNKNOWN_TIER_DIAGNOSTIC_KEY_BYTES)]);
-            hasher.write_u8(u8::from(tier.bytes().any(|byte| byte.is_ascii_control())));
-            format!("tier-hash:{:016x}", hasher.finish())
-        };
-        let now = SystemTime::now();
-        if self
-            .diagnostics_at
-            .is_some_and(|at| now.duration_since(at).unwrap_or_default() > UNKNOWN_TIER_DIAGNOSTIC_TTL)
-        {
-            self.diagnostics.clear();
-        }
-        self.diagnostics_at = Some(now);
-        if self.diagnostics.iter().any(|entry| entry == &digest) {
-            return;
-        }
-        let current_bytes: usize = self.diagnostics.iter().map(String::len).sum();
-        if self.diagnostics.len() >= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP
-            || current_bytes.saturating_add(digest.len()) > UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP
-        {
-            self.diagnostics_dropped = self.diagnostics_dropped.saturating_add(1);
-            return;
-        }
-        self.diagnostics.push(digest);
-    }
-
-    pub fn merge(&mut self, other: &Self) {
-        self.unknown_bytes = self.unknown_bytes.saturating_add(other.unknown_bytes);
-        self.unknown_physical_bytes = self.unknown_physical_bytes.saturating_add(other.unknown_physical_bytes);
-        self.unknown_objects = self.unknown_objects.saturating_add(other.unknown_objects);
-        self.unknown_versions = self.unknown_versions.saturating_add(other.unknown_versions);
-        self.diagnostics_dropped = self.diagnostics_dropped.saturating_add(other.diagnostics_dropped);
-        let now = SystemTime::now();
-        if self
-            .diagnostics_at
-            .zip(other.diagnostics_at)
-            .is_some_and(|(left, right)| now.duration_since(left.max(right)).unwrap_or_default() > UNKNOWN_TIER_DIAGNOSTIC_TTL)
-        {
-            self.diagnostics.clear();
-        }
-        self.diagnostics_at = Some(now);
-        for diagnostic in &other.diagnostics {
-            if self.diagnostics.iter().any(|entry| entry == diagnostic) {
-                continue;
-            }
-            let current_bytes: usize = self.diagnostics.iter().map(String::len).sum();
-            if self.diagnostics.len() >= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP
-                || current_bytes.saturating_add(diagnostic.len()) > UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP
-            {
-                self.diagnostics_dropped = self.diagnostics_dropped.saturating_add(1);
-                continue;
-            }
-            self.diagnostics.push(diagnostic.clone());
-        }
-    }
-
-    pub fn fits_add(&self, other: &Self) -> bool {
-        self.unknown_bytes.checked_add(other.unknown_bytes).is_some()
-            && self
-                .unknown_physical_bytes
-                .checked_add(other.unknown_physical_bytes)
-                .is_some()
-            && self.unknown_objects.checked_add(other.unknown_objects).is_some()
-            && self.unknown_versions.checked_add(other.unknown_versions).is_some()
-            && self.diagnostics_dropped.checked_add(other.diagnostics_dropped).is_some()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.unknown_bytes == 0
-            && self.unknown_physical_bytes == 0
-            && self.unknown_objects == 0
-            && self.unknown_versions == 0
-            && self.diagnostics_dropped == 0
-            && self.diagnostics.is_empty()
-    }
-}
-
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct AllTierStats {
    pub tiers: HashMap<String, TierStats>,
@@ -263,31 +124,6 @@ impl AllTierStats {
            .iter()
            .all(|(tier, right)| self.tiers.get(tier).is_none_or(|left| left.fits_add(right)))
    }
-
-    /// Fold keys from an older cache that are no longer present in the
-    /// current registry into the fixed unknown bucket. Built-in storage
-    /// classes remain known even when no remote tier is configured.
-    pub fn fold_unknown_tiers<'a, I>(&mut self, known_tiers: I)
-    where
-        I: IntoIterator<Item = &'a str>,
-    {
-        let known: HashSet<&str> = known_tiers.into_iter().collect();
-        let mut unknown = self.tiers.remove(UNKNOWN_TIER).unwrap_or_default();
-        let retired: Vec<String> = self
-            .tiers
-            .keys()
-            .filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
-            .cloned()
-            .collect();
-        for tier in retired {
-            if let Some(stats) = self.tiers.remove(&tier) {
-                unknown = unknown.add(&stats);
-            }
-        }
-        if !unknown.is_empty() {
-            self.tiers.insert(UNKNOWN_TIER.to_string(), unknown);
-        }
-    }
 }

 /// Bucket target usage info provides replication statistics
@@ -375,10 +211,6 @@ pub struct DataUsageInfo {
    /// tier exists, so an absent value means "not accounted", never "zero".
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tier_stats: Option<AllTierStats>,
-    /// Bounded diagnostics and separate logical/physical counters for objects
-    /// classified into [`UNKNOWN_TIER`].
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub unknown_tier_stats: Option<UnknownTierStats>,

    /// Total number of buckets in this cluster
    pub buckets_count: u64,
@@ -504,8 +336,6 @@ pub struct SizeSummary {
    pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
    /// Per-tier accounting, keyed by storage class or remote tier name
    pub tier_stats: HashMap<String, TierStats>,
-    /// Counters and bounded diagnostics for unknown tiers in this summary.
-    pub unknown_tier_stats: UnknownTierStats,
 }

 /// Replication target size summary
@@ -851,9 +681,6 @@ pub struct DataUsageEntry {
    /// observed tier-classified objects.
    #[serde(default)]
    pub all_tier_stats: Option<AllTierStats>,
-    /// Bounded unknown-tier reconciliation state for this cache entry.
-    #[serde(default)]
-    pub unknown_tier_stats: Option<UnknownTierStats>,
 }

 impl Serialize for DataUsageEntry {
@@ -864,7 +691,7 @@ impl Serialize for DataUsageEntry {
        // Keep entries map-encoded so older readers can ignore fields appended
        // by newer scanner versions during rolling upgrades. The derived
        // (array) encoding made any appended field a decode error for them.
-        let mut state = serializer.serialize_map(Some(12))?;
+        let mut state = serializer.serialize_map(Some(11))?;
        state.serialize_entry("children", &self.children)?;
        state.serialize_entry("size", &self.size)?;
        state.serialize_entry("objects", &self.objects)?;
@@ -876,7 +703,6 @@ impl Serialize for DataUsageEntry {
        state.serialize_entry("compacted", &self.compacted)?;
        state.serialize_entry("failed_objects", &self.failed_objects)?;
        state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
-        state.serialize_entry("unknown_tier_stats", &self.unknown_tier_stats)?;
        state.end()
    }
 }
@@ -918,11 +744,6 @@ impl DataUsageEntry {
        if let Some(o_tiers) = other.all_tier_stats.as_ref().filter(|tiers| !tiers.is_empty()) {
            self.all_tier_stats.get_or_insert_with(AllTierStats::new).merge(o_tiers);
        }
-        if let Some(other_unknown) = other.unknown_tier_stats.as_ref() {
-            self.unknown_tier_stats
-                .get_or_insert_with(UnknownTierStats::default)
-                .merge(other_unknown);
-        }

        self.obj_sizes.merge_from(&other.obj_sizes);
        self.obj_versions.merge_from(&other.obj_versions);
@@ -936,15 +757,6 @@ impl DataUsageEntry {
        self.all_tier_stats.get_or_insert_with(AllTierStats::new).add_sizes(tiers);
    }

-    pub fn add_unknown_tier_stats(&mut self, stats: &UnknownTierStats) {
-        if stats.is_empty() {
-            return;
-        }
-        self.unknown_tier_stats
-            .get_or_insert_with(UnknownTierStats::default)
-            .merge(stats);
-    }
-
    pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
        let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
            && self.versions.checked_add(other.versions).is_some()
@@ -1008,12 +820,8 @@ impl DataUsageEntry {
            (_, None) | (None, Some(_)) => true,
            (Some(left), Some(right)) => left.fits_merge(right),
        };
-        let unknown_tier_stats_fit = match (&self.unknown_tier_stats, &other.unknown_tier_stats) {
-            (_, None) | (None, Some(_)) => true,
-            (Some(left), Some(right)) => left.fits_add(right),
-        };

-        if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit || !unknown_tier_stats_fit {
+        if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit {
            return false;
        }
        self.merge(other);
@@ -1531,7 +1339,6 @@ impl DataUsageCache {
            delete_markers_total_count: flat.delete_markers as u64,
            objects_total_size: flat.size as u64,
            tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
-            unknown_tier_stats: flat.unknown_tier_stats,
            buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
            buckets_usage,
            usage_snapshot_complete: self.info.snapshot_complete,
@@ -1959,16 +1766,6 @@ impl SizeSummary {
        self.replica_count = self.replica_count.saturating_add(other.replica_count);
        self.pending_count = self.pending_count.saturating_add(other.pending_count);
        self.failed_count = self.failed_count.saturating_add(other.failed_count);
-        self.unknown_tier_stats.merge(&other.unknown_tier_stats);
-
-        // A disk/bucket aggregate is assembled from many object summaries.
-        // Keep the per-tier dimension in lockstep with the scalar counters;
-        // dropping this map here would recreate the original silent-loss bug
-        // at the cross-disk merge boundary.
-        for (tier, stats) in &other.tier_stats {
-            let entry = self.tier_stats.entry(tier.clone()).or_default();
-            *entry = entry.add(stats);
-        }

        // Merge replication target stats
        for (target, stats) in &other.repl_target_stats {
@@ -2081,33 +1878,6 @@ mod tests {
        );
    }

-    #[test]
-    fn retired_tier_stats_fold_into_fixed_unknown_bucket() {
-        let mut stats = AllTierStats::default();
-        stats.tiers.insert(
-            "RETIRED".to_string(),
-            TierStats {
-                total_size: 9,
-                num_versions: 2,
-                num_objects: 1,
-            },
-        );
-        stats.tiers.insert(
-            "WARM".to_string(),
-            TierStats {
-                total_size: 4,
-                num_versions: 1,
-                num_objects: 1,
-            },
-        );
-
-        stats.fold_unknown_tiers(["WARM"]);
-
-        assert!(!stats.tiers.contains_key("RETIRED"));
-        assert_eq!(stats.tiers.get("WARM").map(|v| v.total_size), Some(4));
-        assert_eq!(stats.tiers.get(UNKNOWN_TIER).map(|v| v.total_size), Some(9));
-    }
-
    #[test]
    fn checked_merge_rejects_overflowing_tier_totals() {
        let mut left = tier_entry(
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -189,34 +189,121 @@ pub async fn wait_for_kms_ready(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
+    wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
+}
+
+async fn wait_for_kms_ready_with_timeout(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    total_deadline: Duration,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let start = tokio::time::Instant::now();
+    let deadline = start + total_deadline;
    let mut backoff = Duration::from_millis(200);
    let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;

    loop {
-        if !first_attempt {
-            if start.elapsed() >= total_deadline {
-                return Err("KMS failed to become ready within 5 seconds".into());
-            }
-            sleep(backoff).await;
-            backoff = (backoff * 2).min(max_backoff);
-        }
-        first_attempt = false;
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
+        match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
+            Ok(Ok(status)) => {
+                let backend_status = serde_json::from_str::<serde_json::Value>(&status)
+                    .ok()
+                    .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
+                if backend_status.as_deref() == Some("healthy") {
+                    info!("KMS is ready (status: {})", status);
+                    return Ok(());
                }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
+                warn!(
+                    backend_status = backend_status.as_deref().unwrap_or("missing"),
+                    elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
+                    "KMS not ready yet, retrying…"
+                );
            }
+            Ok(Err(e)) => {
+                let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+                warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
+            }
+            Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
        }
+
+        let now = tokio::time::Instant::now();
+        if now >= deadline {
+            return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
+        }
+        sleep((now + backoff).min(deadline) - now).await;
+        backoff = (backoff * 2).min(max_backoff);
+    }
+}
+
+#[cfg(test)]
+mod readiness_tests {
+    use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    #[tokio::test]
+    async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let requests = Arc::new(AtomicUsize::new(0));
+        let server_requests = Arc::clone(&requests);
+        let server = tokio::spawn(async move {
+            for backend_status in ["error", "healthy"] {
+                let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+                let mut request = Vec::new();
+                let mut chunk = [0_u8; 1024];
+                while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    let read = socket.read(&mut chunk).await.expect("read readiness request");
+                    if read == 0 {
+                        break;
+                    }
+                    request.extend_from_slice(&chunk[..read]);
+                }
+                server_requests.fetch_add(1, Ordering::SeqCst);
+
+                let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                socket.write_all(response.as_bytes()).await.expect("write readiness response");
+            }
+        });
+
+        wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
+            .await
+            .expect("KMS should become ready after the healthy response");
+
+        let observed_requests = requests.load(Ordering::SeqCst);
+        server.abort();
+        assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
+    }
+
+    #[tokio::test]
+    async fn kms_readiness_deadline_covers_a_stalled_status_request() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+            let mut request = [0_u8; 1024];
+            let _ = socket.read(&mut request).await.expect("read readiness request");
+            std::future::pending::<()>().await;
+        });
+
+        let result = tokio::time::timeout(
+            Duration::from_secs(1),
+            wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
+        )
+        .await
+        .expect("readiness helper must enforce its own deadline");
+
+        server.abort();
+        assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
    }
 }

@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    let bucket = "copy-object-self-copy-bucket-default-sse-test";
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    let bucket = "copy-object-version-restore-sse-test";
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -24,7 +24,6 @@ use super::common::{
    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::info;

 /// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = Arc::new(kms_env.base_env.create_s3_client());
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait for KMS initialization
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -19,7 +19,6 @@
 //! multipart upload behaviour.

 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::{error, info};

 use super::common::{
@@ -45,8 +44,8 @@ impl VaultKmsTestContext {

        start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;

-        // Allow Vault to finish initialising token auth and transit engine.
-        sleep(Duration::from_secs(2)).await;
+        // Wait for KMS to finish initialising.
+        super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;

        Ok(Self { env })
    }
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
 #[cfg(test)]
 mod kms_fault_recovery_test;

-#[cfg(test)]
-mod test_runner;
-
 #[cfg(test)]
 mod bucket_default_encryption_test;

@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -1,499 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-#![allow(dead_code)]
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Unified KMS test suite runner
-//!
-//! This module provides a unified interface for running KMS tests with categorization,
-//! filtering, and comprehensive reporting capabilities.
-
-use crate::common::init_logging;
-use std::time::Instant;
-use tokio::time::{Duration, sleep};
-use tracing::{debug, error, info, warn};
-
-/// Test category for organization and filtering
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TestCategory {
-    CoreFunctionality,
-    MultipartEncryption,
-    EdgeCases,
-    FaultRecovery,
-    Comprehensive,
-    Performance,
-}
-
-impl TestCategory {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            TestCategory::CoreFunctionality => "core-functionality",
-            TestCategory::MultipartEncryption => "multipart-encryption",
-            TestCategory::EdgeCases => "edge-cases",
-            TestCategory::FaultRecovery => "fault-recovery",
-            TestCategory::Comprehensive => "comprehensive",
-            TestCategory::Performance => "performance",
-        }
-    }
-}
-
-/// Test definition with metadata
-#[derive(Debug, Clone)]
-pub struct TestDefinition {
-    pub name: String,
-    pub description: String,
-    pub category: TestCategory,
-    pub estimated_duration: Duration,
-    pub is_critical: bool,
-}
-
-impl TestDefinition {
-    pub fn new(
-        name: impl Into<String>,
-        description: impl Into<String>,
-        category: TestCategory,
-        estimated_duration: Duration,
-        is_critical: bool,
-    ) -> Self {
-        Self {
-            name: name.into(),
-            description: description.into(),
-            category,
-            estimated_duration,
-            is_critical,
-        }
-    }
-}
-
-/// Test execution result
-#[derive(Debug, Clone)]
-pub struct TestResult {
-    pub test_name: String,
-    pub category: TestCategory,
-    pub success: bool,
-    pub duration: Duration,
-    pub error_message: Option<String>,
-}
-
-impl TestResult {
-    pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
-        Self {
-            test_name,
-            category,
-            success: true,
-            duration,
-            error_message: None,
-        }
-    }
-
-    pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
-        Self {
-            test_name,
-            category,
-            success: false,
-            duration,
-            error_message: Some(error),
-        }
-    }
-}
-
-/// Comprehensive test suite configuration
-#[derive(Debug, Clone)]
-pub struct TestSuiteConfig {
-    pub categories: Vec<TestCategory>,
-    pub include_critical_only: bool,
-    pub max_duration: Option<Duration>,
-    pub parallel_execution: bool,
-}
-
-impl Default for TestSuiteConfig {
-    fn default() -> Self {
-        Self {
-            categories: vec![
-                TestCategory::CoreFunctionality,
-                TestCategory::MultipartEncryption,
-                TestCategory::EdgeCases,
-                TestCategory::FaultRecovery,
-                TestCategory::Comprehensive,
-            ],
-            include_critical_only: false,
-            max_duration: None,
-            parallel_execution: false,
-        }
-    }
-}
-
-/// Unified KMS test suite runner
-pub struct KMSTestSuite {
-    tests: Vec<TestDefinition>,
-    config: TestSuiteConfig,
-}
-
-impl KMSTestSuite {
-    /// Create a new test suite with default configuration
-    pub fn new() -> Self {
-        let tests = vec![
-            // Core Functionality Tests
-            TestDefinition::new(
-                "test_local_kms_end_to_end",
-                "End-to-end KMS test with all encryption types",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_local_kms_key_isolation",
-                "Test KMS key isolation and security",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(45),
-                true,
-            ),
-            // Multipart Encryption Tests
-            TestDefinition::new(
-                "test_local_kms_multipart_upload",
-                "Test large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step1_basic_single_file_encryption",
-                "Basic single file encryption test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step2_basic_multipart_upload_without_encryption",
-                "Basic multipart upload without encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step3_multipart_upload_with_sse_s3",
-                "Multipart upload with SSE-S3 encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step4_large_multipart_upload_with_encryption",
-                "Large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(90),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step5_all_encryption_types_multipart",
-                "All encryption types multipart test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            // Edge Cases Tests
-            TestDefinition::new(
-                "test_kms_zero_byte_file_encryption",
-                "Test encryption of zero-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_single_byte_file_encryption",
-                "Test encryption of single-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_boundary_conditions",
-                "Test multipart upload boundary conditions",
-                TestCategory::EdgeCases,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_invalid_key_scenarios",
-                "Test invalid key scenarios",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_concurrent_encryption",
-                "Test concurrent encryption operations",
-                TestCategory::EdgeCases,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_key_validation_security",
-                "Test key validation security",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            // Fault Recovery Tests
-            TestDefinition::new(
-                "test_kms_key_directory_unavailable",
-                "Test KMS when key directory is unavailable",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_corrupted_key_files",
-                "Test KMS with corrupted key files",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_upload_interruption",
-                "Test multipart upload interruption recovery",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_resource_constraints",
-                "Test KMS under resource constraints",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(90),
-                false,
-            ),
-            // Comprehensive Tests
-            TestDefinition::new(
-                "test_comprehensive_kms_full_workflow",
-                "Full KMS workflow comprehensive test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(300),
-                true,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_stress_test",
-                "KMS stress test with large datasets",
-                TestCategory::Comprehensive,
-                Duration::from_secs(400),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_key_isolation",
-                "Comprehensive key isolation test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(180),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_concurrent_operations",
-                "Comprehensive concurrent operations test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(240),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_performance_benchmark",
-                "KMS performance benchmark test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(360),
-                false,
-            ),
-        ];
-
-        Self {
-            tests,
-            config: TestSuiteConfig::default(),
-        }
-    }
-
-    /// Configure the test suite
-    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
-        self.config = config;
-        self
-    }
-
-    /// Filter tests based on category
-    pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| &test.category == category).collect()
-    }
-
-    /// Filter tests based on criticality
-    pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| test.is_critical).collect()
-    }
-
-    /// Get test summary by category
-    pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
-        let mut summary = std::collections::HashMap::new();
-        for test in &self.tests {
-            summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
-        }
-        summary
-    }
-
-    /// Run the complete test suite
-    pub async fn run_test_suite(&self) -> Vec<TestResult> {
-        init_logging();
-        info!("🚀 Starting unified KMS test suite");
-
-        let start_time = Instant::now();
-        let mut results = Vec::new();
-
-        // Filter tests based on configuration
-        let tests_to_run: Vec<&TestDefinition> = self
-            .tests
-            .iter()
-            .filter(|test| self.config.categories.contains(&test.category))
-            .filter(|test| !self.config.include_critical_only || test.is_critical)
-            .collect();
-
-        info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
-        for (i, test) in tests_to_run.iter().enumerate() {
-            info!("  {}. {} ({})", i + 1, test.name, test.category.as_str());
-        }
-
-        // Execute tests
-        for (i, test_def) in tests_to_run.iter().enumerate() {
-            info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
-            info!("   📝 Description: {}", test_def.description);
-            info!("   🏷️  Category: {}", test_def.category.as_str());
-            info!("   ⏱️  Estimated duration: {:?}", test_def.estimated_duration);
-
-            let test_start = Instant::now();
-            let result = self.run_single_test(test_def).await;
-            let test_duration = test_start.elapsed();
-
-            match result {
-                Ok(_) => {
-                    info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
-                    results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
-                }
-                Err(e) => {
-                    error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
-                    results.push(TestResult::failure(
-                        test_def.name.clone(),
-                        test_def.category.clone(),
-                        test_duration,
-                        e.to_string(),
-                    ));
-                }
-            }
-
-            // Add delay between tests to avoid resource conflicts
-            if i < tests_to_run.len() - 1 {
-                debug!("⏸️  Waiting two seconds before the next test...");
-                sleep(Duration::from_secs(2)).await;
-            }
-        }
-
-        let total_duration = start_time.elapsed();
-        self.print_test_summary(&results, total_duration);
-
-        results
-    }
-
-    /// Run a single test by dispatching to the appropriate test function
-    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        // This is a placeholder for test dispatch logic
-        // In a real implementation, this would dispatch to actual test functions
-        warn!("⚠️  Test '{}' is not implemented in the unified runner; skipping", test_def.name);
-        Ok(())
-    }
-
-    /// Print comprehensive test summary
-    fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
-        info!("📊 KMS test suite summary");
-        info!("⏱️  Total duration: {:.2} seconds", total_duration.as_secs_f64());
-        info!("📈 Total tests: {}", results.len());
-
-        let passed = results.iter().filter(|r| r.success).count();
-        let failed = results.iter().filter(|r| !r.success).count();
-
-        info!("✅ Passed: {}", passed);
-        info!("❌ Failed: {}", failed);
-        info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
-
-        // Summary by category
-        let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
-        for result in results {
-            let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
-            *total += 1;
-            if result.success {
-                *passed_count += 1;
-            }
-        }
-
-        info!("📊 Category summary:");
-        for (category, (total, passed_count)) in category_summary {
-            info!(
-                "  🏷️  {}: {}/{} ({:.1}%)",
-                category.as_str(),
-                passed_count,
-                total,
-                (passed_count as f64 / total as f64) * 100.0
-            );
-        }
-
-        // List failed tests
-        if failed > 0 {
-            warn!("❌ Failing tests:");
-            for result in results.iter().filter(|r| !r.success) {
-                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
-            }
-        }
-    }
-}
-
-/// Quick test suite for critical tests only
-#[tokio::test]
-async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let config = TestSuiteConfig {
-        categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
-        include_critical_only: true,
-        max_duration: Some(Duration::from_secs(600)), // 10 minutes max
-        parallel_execution: false,
-    };
-
-    let suite = KMSTestSuite::new().with_config(config);
-    let results = suite.run_test_suite().await;
-
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
-    }
-
-    info!("✅ All critical tests passed");
-    Ok(())
-}
-
-/// Full comprehensive test suite
-#[tokio::test]
-async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let suite = KMSTestSuite::new();
-    let results = suite.run_test_suite().await;
-
-    let total_tests = results.len();
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
-
-    info!("📊 Full suite success rate: {:.1}%", success_rate);
-
-    // Allow up to 10% failure rate for non-critical tests
-    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
-    }
-
-    info!("✅ Full test suite succeeded");
-    Ok(())
-}
@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
 use rustfs_signer::pre_sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let bucket = "object-lambda-e2e-disabled-target";
    let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
        .send()
        .await?;

-    configure_webhook_target_with_key_values(
-        &env,
-        "transformer",
-        vec![
-            ("endpoint", "http://127.0.0.1:9/transform".to_string()),
-            ("auth_token", "secret-token".to_string()),
-            ("enable", "off".to_string()),
-        ],
+    let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
+    tokio::fs::create_dir_all(&queue_dir).await?;
+    let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
+    let directive = format!(
+        "notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
+    );
+    let disable_response = signed_request(
+        http::Method::PUT,
+        &config_url,
+        &env.access_key,
+        &env.secret_key,
+        Some(directive.into_bytes()),
+        Some("text/plain"),
    )
    .await?;
-    wait_for_target_visibility(&env, "transformer").await?;
+    let disable_status = disable_response.status();
+    let disable_body = disable_response.text().await?;
+    assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");

    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let bucket = "object-lambda-e2e-invalid-endpoint";

@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let response = send_configure_webhook_target_request(
        &env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
    init_logging();

    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.set_env(ENV_NOTIFY_ENABLE, "true");
+    cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
    cluster.start().await?;

    let bucket = "listen-notification-cluster";
@@ -15,7 +15,6 @@
 use crate::common::{RustFSTestClusterEnvironment, init_logging};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::CompletedMultipartUpload;
 use tokio::time::{Duration, sleep};
 use tracing::info;
 use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
    }
 }

-async fn complete_reports_missing_upload(
+async fn multipart_listing_reports_missing_upload(
    client: &aws_sdk_s3::Client,
    bucket: &str,
    key: &str,
    upload_id: &str,
 ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
-    let result = client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload_id)
-        .multipart_upload(CompletedMultipartUpload::builder().build())
-        .send()
-        .await;
-    match result {
-        Ok(_) => Ok(false),
-        Err(SdkError::ServiceError(err)) => {
-            let code = err.err().meta().code().unwrap_or("");
-            if code == "NoSuchUpload" {
-                Ok(true)
-            } else {
-                Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
-            }
-        }
-        Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
-    }
+    let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
+
+    Ok(!result
+        .uploads()
+        .iter()
+        .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
 }

 async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
        let mut all_cleaned = true;
        for (idx, client) in clients.iter().enumerate() {
            let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
-            let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
-            if !(list_parts_missing && complete_missing) {
+            let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
+            if !(list_parts_missing && listing_missing) {
                info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
                all_cleaned = false;
                break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
        1,
        "multipart upload should be visible before background cleanup"
    );
+    assert!(
+        !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
+        "multipart upload listing should contain the upload before background cleanup"
+    );

    wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;

@@ -9136,6 +9136,7 @@ mod tests {
        assert_eq!(loaded.report.scanned, 37);
        assert_eq!(loaded.report.eligible, 11);
        assert_eq!(loaded.report.enqueued, 5);
+        assert_eq!(loaded.cursor_revision, Some(1));
        assert!(loaded.lease_expires_at_unix_nanos > 0);
        let token = loaded
            .report
@@ -9154,6 +9155,30 @@ mod tests {
        assert_eq!(admission.lease_id, loaded.lease_id);
        assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);

+        let mut same_marker_report = report.clone();
+        same_marker_report.scanned += 1;
+        persist_manual_transition_page_checkpoint(
+            &checkpoint_options,
+            &same_marker_report,
+            Some("logs/page-end".to_string()),
+            Some("opaque-next-version".to_string()),
+        )
+        .await
+        .expect("same-marker version checkpoint should persist through the durable progress sink");
+        let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
+            .await
+            .expect("same-marker version checkpoint should reload");
+        assert_eq!(same_marker_checkpointed.cursor_revision, Some(2));
+        let (_, version_marker) = decode_manual_transition_continuation_token(
+            same_marker_checkpointed
+                .report
+                .continuation_token
+                .as_deref()
+                .expect("same-marker version checkpoint should persist a cursor"),
+        )
+        .expect("same-marker version cursor should decode");
+        assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
+
        create_test_bucket(&ecstore, &bucket).await;
        let lifecycle_xml = format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9235,7 @@ mod tests {
        assert_eq!(checkpointed.report.scanned, 1000);
        assert_eq!(checkpointed.report.eligible, 1000);
        assert_eq!(checkpointed.report.dry_run_eligible, 1000);
+        assert_eq!(checkpointed.cursor_revision, Some(3));
        let token = checkpointed
            .report
            .continuation_token
@@ -24,6 +24,10 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::{
    ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
 };
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::{
+    MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
+    MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
@@ -34,10 +38,10 @@ use crate::store::ECStore;
 pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
 pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
 pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
-pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
-pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
-pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
-pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
+pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
 pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
 pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
 pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +199,8 @@ pub struct ManualTransitionJobRecord {
    pub updated_at_unix_nanos: i128,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub completed_at_unix_nanos: Option<i128>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cursor_revision: Option<u64>,
    pub report: ManualTransitionRunReport,
    pub queue_snapshot: ManualTransitionQueueSnapshot,
    #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +230,7 @@ impl ManualTransitionJobRecord {
            created_at_unix_nanos: now,
            updated_at_unix_nanos: now,
            completed_at_unix_nanos: None,
+            cursor_revision: Some(0),
            report: ManualTransitionRunReport {
                bucket: bucket.to_string(),
                prefix: options.prefix.clone(),
@@ -238,7 +245,7 @@ impl ManualTransitionJobRecord {

    pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
        self.scan_completed = true;
-        self.report.merge_scan_report_preserving_worker(&report);
+        self.merge_scan_report(&report);
        self.queue_snapshot = queue_snapshot;
        self.error = None;
        self.mark_terminal_if_worker_drained();
@@ -316,7 +323,7 @@ impl ManualTransitionJobRecord {
            }
        }
        self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
        self.mark_terminal_if_worker_drained();
    }

@@ -359,14 +366,14 @@ impl ManualTransitionJobRecord {
        self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
        self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
        self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
        self.mark_terminal_if_worker_drained();
        true
    }

    pub fn mark_cancel_requested(&mut self) {
        self.cancel_requested = true;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
    }

    pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +387,7 @@ impl ManualTransitionJobRecord {
    pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
        if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
            self.lease_expires_at_unix_nanos = 0;
-            self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+            self.advance_updated_at();
        }
    }

@@ -398,7 +405,7 @@ impl ManualTransitionJobRecord {

    pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
        let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
        self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
        self.queue_snapshot = queue_snapshot;
    }
@@ -438,11 +445,18 @@ impl ManualTransitionJobRecord {

    pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
        if self.state == ManualTransitionJobState::Running {
-            self.report.merge_scan_report_preserving_worker(&report);
+            self.merge_scan_report(&report);
            self.renew_lease(queue_snapshot);
        }
    }

+    fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
+        if self.report.continuation_token != report.continuation_token {
+            self.cursor_revision = Some(self.cursor_revision.unwrap_or(0).saturating_add(1));
+        }
+        self.report.merge_scan_report_preserving_worker(report);
+    }
+
    pub fn mark_unknown_if_unowned(&mut self) {
        if self.state == ManualTransitionJobState::Running {
            self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
    }

    fn mark_updated_terminal(&mut self) {
+        self.advance_updated_at();
+        self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
+    }
+
+    fn advance_updated_at(&mut self) {
        let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
-        self.completed_at_unix_nanos = Some(now);
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
    }

    fn mark_terminal_if_worker_drained(&mut self) {
@@ -1109,7 +1127,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
 pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
    let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
    let data = job.encode().map_err(manual_transition_job_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1161,9 @@ pub async fn save_manual_transition_job_record_if_current(
    let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
    let data = job.encode().map_err(manual_transition_job_store_error)?;
    config_boundary::save_config_with_opts_quiet(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1173,8 @@ pub async fn save_manual_transition_job_record_if_current(
            ..Default::default()
        },
    )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 /// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1612,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
    let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
    let data = serde_json::to_vec(admission).map_err(Error::other)?;
    config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1624,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
            ..Default::default()
        },
    )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1663,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
    let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
    let data = serde_json::to_vec(admission).map_err(Error::other)?;
    match config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1681,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
            Err(Error::PreconditionFailed)
        }
        result => result,
-    }
+    }?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1975,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
    job_id: Uuid,
    lease_id: Uuid,
 ) -> EcstoreResult<bool> {
-    let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
-        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
+    let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
+        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
        Ok(_) => return Ok(false),
        Err(Error::ConfigNotFound) => return Ok(true),
        Err(err) => return Err(err),
    };
    let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
+    let data = serde_json::to_vec(&admission).map_err(Error::other)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
    match config_boundary::delete_config_if_match(api, &object, &etag).await {
        Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
        Err(Error::PreconditionFailed) => Ok(false),
@@ -2577,6 +2601,25 @@ mod tests {
        assert!(decoded.report.tier_failure_by_reason.is_empty());
    }

+    #[test]
+    fn manual_transition_job_record_decodes_legacy_cursor_without_revision() {
+        let options = ManualTransitionRunOptions::default();
+        let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let encoded = record.encode().expect("job record should encode");
+        let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        value["job"]
+            .as_object_mut()
+            .expect("job should be object")
+            .remove("cursor_revision");
+        let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
+        value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
+        let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
+
+        let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
+
+        assert_eq!(decoded.cursor_revision, None);
+    }
+
    #[test]
    fn manual_transition_job_record_rejects_unknown_report_fields() {
        let options = ManualTransitionRunOptions::default();
@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
 mod config_boundary;
 pub mod core;
+mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;
 pub mod transition_transaction;
+
+pub(crate) use durable_namespace::{
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
+};
@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, warn};

 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
    Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
-pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
+pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;

 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
        )
        .await?;
    }
+    let path = tier_delete_journal_object_name(je);
+    let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
+    let target_pool_indices = api
+        .record_durable_ilm_decommission_terminal_target_pools(&path, &data)
+        .await
+        .map_err(std::io::Error::other)?;
+    if let Some(target_pool_indices) = target_pool_indices {
+        for target_pool_idx in target_pool_indices {
+            match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
+                Ok(()) | Err(Error::ConfigNotFound) => {}
+                Err(err) => return Err(std::io::Error::other(err)),
+            }
+        }
+        return Ok(());
+    }
    remove_tier_delete_journal_entry(api, je).await
 }

@@ -21,6 +21,7 @@ use tracing::{debug, warn};
 use uuid::Uuid;

 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
    delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
-pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
+pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
 pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;

 pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
    let object =
        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
    let data = transaction.encode().map_err(transition_transaction_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
    TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
 }

-pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
-    let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
+pub(crate) async fn delete_transition_transaction_record(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<()> {
+    let object =
+        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
+    let data = transaction.encode().map_err(transition_transaction_store_error)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
    match config_boundary::delete_config(api, &object).await {
        Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
        Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
    if probe != TransitionOperatorProbe::Missing {
        return Err(TransitionOperatorError::CandidateNotMissing(probe));
    }
-    delete_transition_transaction_record(api, transaction_id)
+    delete_transition_transaction_record(api, &transaction)
        .await
        .map_err(TransitionOperatorError::Store)
 }
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
    match transaction.state {
        TransitionTransactionState::Uploaded => {
            delete_transition_remote_candidate(api.clone(), transaction).await?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
        }
        TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
            Ok(true) => {
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
            }
            Ok(false) => {
                delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
            }
            Err(err) if transition_source_is_missing(&err) => {
                delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
            }
            Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
        TransitionTransactionState::LocalCommitStarted => {
            match local_commit_matches_transaction(api.clone(), transaction).await {
                Ok(true) => {
-                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    delete_transition_transaction_record(api, transaction).await?;
                    Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
                }
                Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
            }
        }
        TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
        }
        TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
        .map_err(Error::other)?
    {
        TransitionCandidateProbe::Missing => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
        }
        TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
            )
            .await
            .map_err(Error::other)?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
        }
        TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
        .map_err(transition_transaction_store_error)?;
    save_transition_transaction_record(api.clone(), &cleanup).await?;
    delete_transition_remote_candidate(api.clone(), &cleanup).await?;
-    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    delete_transition_transaction_record(api, &cleanup).await?;
    Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }

@@ -42,8 +42,9 @@ use futures::lock::Mutex;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_io_metrics::internode_metrics::{
-    INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
-    INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+    INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
+    INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
+    INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
 };
 use rustfs_protos::ChannelClass;
 use rustfs_protos::evict_failed_connection;
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
 const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
 const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
 const BATCH_METADATA_RPC_OFF: &str = "off";
 const BATCH_METADATA_RPC_AUTO: &str = "auto";
 const BATCH_METADATA_RPC_ON: &str = "on";
@@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
 }

 fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
-    rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
+    rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
+        .or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
        .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
        .as_deref()
        .map(parse_batch_metadata_rpc_mode)
@@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
    }
 }

+fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
+    if let Some(started_at) = started_at {
+        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
+    }
+}
+
 /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
 /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
 /// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items(
    Ok(batch_read_version_resps)
 }

+fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
+    req.items
+        .iter()
+        .fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
+            total
+                .saturating_add(item.org_volume.len())
+                .saturating_add(item.volume.len())
+                .saturating_add(item.path.len())
+                .saturating_add(item.version_id.len())
+        })
+}
+
+fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
+    response
+        .batch_read_version_resps
+        .iter()
+        .map(String::len)
+        .sum::<usize>()
+        .saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
+}
+
 fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
    file_info.validate_for_metadata_read().map_err(Into::into)
 }
@@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
            state = "started",
            "Remote disk RPC started"
        );
+        let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
+        let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
        let batch_read_version_req = compat_json(&req)?;
        let batch_read_version_req_bin = encode_msgpack(&req)?;
-
+        record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
+        let request_payload_bytes = batch_read_version_attribution_enabled
+            .then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
        let batch_result = self
            .execute_with_timeout_for_op(
                "batch_read_version",
                move || async move {
                    let disk = self.disk_ref().await;
+                    let disk_len = disk.len();
                    let mut client = self
                        .get_bulk_client()
                        .await
@@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
                        batch_read_version_req_bin: batch_read_version_req_bin.into(),
                    });

+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
+                    if let Some(request_payload_bytes) = request_payload_bytes {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
+                            request_payload_bytes.saturating_add(disk_len),
+                        );
+                    }
+                    let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
                    let response = match client.batch_read_version(request).await {
-                        Ok(response) => response.into_inner(),
+                        Ok(response) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            response.into_inner()
+                        }
                        Err(status) if status.code() == Code::Unimplemented => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                            if mode.should_fallback_on_unimplemented() {
                                record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
                                warn!(
@@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
                            }

                            record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                            warn!(
                                event = EVENT_REMOTE_DISK_RPC,
                                component = LOG_COMPONENT_ECSTORE,
@@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
                            );
                            return Err(Error::from(status));
                        }
-                        Err(status) => return Err(Error::from(status)),
+                        Err(status) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            return Err(Error::from(status));
+                        }
                    };

                    if !response.success {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                        return Err(response.error.unwrap_or_default().into());
                    }

-                    decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
+                        batch_read_version_response_payload_len(&response),
+                    );
+                    let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
+                    match decode_batch_read_version_response_items(response, &self.endpoint) {
+                        Ok(batch_read_version_resps) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            Ok(Some(batch_read_version_resps))
+                        }
+                        Err(err) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            Err(err)
+                        }
+                    }
                },
                get_max_timeout_duration(),
            )
@@ -4621,6 +4687,7 @@ mod tests {
            } else {
                "file version not found".to_string()
            },
+            error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
        }
    }

@@ -4740,6 +4807,7 @@ mod tests {
    fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
        temp_env::with_vars(
            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
            ],
@@ -4749,10 +4817,25 @@ mod tests {
        );
    }

+    #[test]
+    fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
+                (ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
+                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
+            ],
+            || {
+                assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
+            },
+        );
+    }
+
    #[test]
    fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
        temp_env::with_vars(
            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
            ],
@@ -14,9 +14,10 @@

 use rustfs_io_metrics::internode_metrics::{
    INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
-    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
-    INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
-    INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
+    INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+    global_internode_metrics,
 };
 use std::time::Duration;

@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
    );
 }

+pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_outgoing_request_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_stage_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        stage,
+        duration,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics()
+        .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_sent_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_recv_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+    record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
+}
+
 pub(crate) fn record_remote_disk_grpc_read_version_error() {
    if !rustfs_io_metrics::get_stage_metrics_enabled() {
        return;
@@ -406,6 +406,25 @@ where
    Ok(data)
 }

+pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
+where
+    S: EcstoreObjectIO,
+{
+    let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
+    Ok(data)
+}
+
+pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
+    api: Arc<S>,
+    file: &str,
+    max_bytes: usize,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
+}
+
 /// Read an existing config object without treating an empty payload as absent.
 /// Callers that validate their own payload format need to distinguish corruption
 /// from `ConfigNotFound`.
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
    namespace::NamespaceLocking as _,
    object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::ECStore;
+use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
 fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
    is_unversioned_data_movement_object(source)
        && is_unversioned_data_movement_object(target)
-        && !target.delete_marker
        && source
            .mod_time
            .zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
 pub(crate) struct SourceCleanupBucketFence<'a> {
    pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
    pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
 }

 fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
 struct SourceCleanupDeleteBarrierState {
    bucket: String,
    object: String,
+    fence_pending: tokio::sync::Notify,
    arrived: tokio::sync::Notify,
+    is_paused: AtomicBool,
    release: tokio::sync::Notify,
 }

@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
 }

 #[cfg(test)]
-static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
+static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
    std::sync::OnceLock::new();

 #[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
        let state = Arc::new(SourceCleanupDeleteBarrierState {
            bucket: bucket.to_string(),
            object: object.to_string(),
+            fence_pending: tokio::sync::Notify::new(),
            arrived: tokio::sync::Notify::new(),
+            is_paused: AtomicBool::new(false),
            release: tokio::sync::Notify::new(),
        });
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
            .lock()
            .expect("source cleanup delete barrier mutex should not poison");
-        assert!(slot.is_none(), "source cleanup delete barrier must be unique");
-        *slot = Some(Arc::clone(&state));
+        assert!(
+            !barriers
+                .iter()
+                .any(|barrier| barrier.bucket == bucket && barrier.object == object),
+            "source cleanup delete barrier must be unique per object"
+        );
+        barriers.push(Arc::clone(&state));
        Self { state }
    }

@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
            .expect("source cleanup should reach the pre-delete barrier");
    }

+    pub(crate) async fn wait_until_fence_pending(&self) {
+        tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
+            .await
+            .expect("source cleanup should attempt the fixed mutation fence");
+    }
+
+    pub(crate) fn is_paused(&self) -> bool {
+        self.state.is_paused.load(Ordering::Acquire)
+    }
+
    pub(crate) fn release(&self) {
        self.state.release.notify_one();
    }
 }

+#[cfg(test)]
+pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("source cleanup delete barrier mutex should not poison")
+        .iter()
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.fence_pending.notify_one();
+    }
+}
+
 #[cfg(test)]
 impl Drop for SourceCleanupDeleteBarrier {
    fn drop(&mut self) {
        self.state.release.notify_one();
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
            .lock()
            .expect("source cleanup delete barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
-            *slot = None;
-        }
+        barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
    }
 }

 #[cfg(test)]
 async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
-    let barrier = SOURCE_CLEANUP_DELETE_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
        .lock()
        .expect("source cleanup delete barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .iter()
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
        .cloned();
    if let Some(barrier) = barrier {
+        barrier.is_paused.store(true, Ordering::Release);
        barrier.arrived.notify_one();
        barrier.release.notified().await;
    }
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
    op_label: &str,
 ) -> std::result::Result<ObjectInfo, SourceCleanupError> {
    let cleanup_key = encode_dir_object(object);
-    let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
-    let _guard = ns_lock
-        .get_write_lock(get_lock_acquire_timeout())
-        .await
-        .map_err(Error::from)?;
+    let source_guard = if bucket_fence
+        .object_mutation_fence
+        .is_some_and(SourceCleanupMutationFence::source_lock_covered)
+    {
+        None
+    } else {
+        let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
+        Some(
+            ns_lock
+                .get_write_lock(get_lock_acquire_timeout())
+                .await
+                .map_err(Error::from)?,
+        )
+    };

    if bucket_fence
        .lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
            "{op_label}: bucket incarnation fence was lost before source cleanup"
        ))));
    }
+    if bucket_fence
+        .object_mutation_fence
+        .is_some_and(SourceCleanupMutationFence::is_lock_lost)
+    {
+        return Err(SourceCleanupError::Storage(Error::other(format!(
+            "{op_label}: object mutation fence was lost before source cleanup"
+        ))));
+    }

    ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;

@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
        expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
        ..Default::default()
    };
-    opts.add_namespace_lock_guard(&_guard);
+    if let Some(source_guard) = source_guard.as_ref() {
+        opts.add_namespace_lock_guard(source_guard);
+    }
+    if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
+        object_mutation_fence.add_namespace_lock_fence(&mut opts);
+    }
    if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
        opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
    }
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
    }
 }

+pub(crate) async fn migrate_decommission_object(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+) -> Result<()> {
+    let source = rd.object_info.clone();
+    let _mutation_fence = store
+        .acquire_decommission_object_mutation_fence(&bucket, &source.name)
+        .await?;
+    let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
+        .await?
+        .ok_or(Error::FileNotFound)?;
+    if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
+        return Err(Error::FileNotFound);
+    }
+
+    migrate_object_inner(
+        store,
+        pool_idx,
+        bucket,
+        rd,
+        source_bucket_incarnation_id,
+        op_label,
+        Some(&_mutation_fence),
+    )
+    .await
+}
+
 pub(crate) async fn migrate_object(
    store: Arc<ECStore>,
    pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
    rd: GetObjectReader,
    source_bucket_incarnation_id: Option<uuid::Uuid>,
    op_label: &str,
+) -> Result<()> {
+    migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
+}
+
+async fn migrate_object_inner(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+    mutation_fence: Option<&ObjectLockDiagGuard>,
 ) -> Result<()> {
    let object_info = rd.object_info.clone();
    let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
        let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
        new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
        let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
+            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
            .await
        {
            Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
            if let Err(err) = store
                .clone()
                .complete_multipart_upload_for_data_movement(
-                    target_pool_idx,
+                    (target_pool_idx, mutation_fence),
                    &bucket,
                    &object_info.name,
                    &res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
    let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
    put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
    let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
+        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
        .await
        .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
    if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
    }

    #[test]
-    fn test_precondition_conflict_rejects_newer_delete_marker() {
-        let source = ObjectInfo {
-            size: 128,
-            etag: Some("etag-source".to_string()),
-            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            delete_marker: true,
-            etag: None,
-            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
-            ..source.clone()
-        };
+    fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
+        for version_id in [None, Some(Uuid::nil())] {
+            let source = ObjectInfo {
+                version_id,
+                size: 128,
+                etag: Some("etag-source".to_string()),
+                mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            let target = ObjectInfo {
+                delete_marker: true,
+                etag: None,
+                mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+                ..source.clone()
+            };

-        let should_resume =
-            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
-                .expect("delete marker conflict should be evaluated");
+            assert!(
+                resolve_data_movement_overwrite_resume_result(
+                    &Error::PreconditionFailed,
+                    Ok(Some(target.clone())),
+                    &source,
+                    0,
+                    1,
+                )
+                .expect("newer null delete marker should be evaluated")
+            );

-        assert!(!should_resume);
+            let mut same_time = target.clone();
+            same_time.mod_time = source.mod_time;
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
+                    .expect("same-generation null delete marker should be rejected")
+            );
+
+            let mut versioned = target;
+            versioned.version_id = Some(Uuid::new_v4());
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
+                    .expect("a UUID delete marker must not erase a null source version")
+            );
+        }
    }

    #[test]
@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
 #[cfg(target_os = "linux")]
 const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
 const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
+const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
 const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
 const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
 const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
@@ -6106,6 +6107,43 @@ impl LocalDisk {
        Ok((bytes, modtime))
    }

+    async fn write_missing_delete_marker(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        object_dir: &Path,
+        xl_path: &Path,
+        rollback_dir: Option<Uuid>,
+    ) -> Result<()> {
+        if let Some(rollback_dir) = rollback_dir {
+            let rollback_path = object_dir.join(rollback_dir.to_string());
+            fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
+            fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
+                .await
+                .map_err(to_file_error)?;
+        }
+        if let Err(err) = self.write_metadata("", volume, path, fi).await {
+            if let Some(rollback_dir) = rollback_dir
+                && let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
+            {
+                warn!(
+                    event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
+                    result = "failed",
+                    volume,
+                    path,
+                    rollback_dir = %rollback_dir,
+                    error = ?restore_err,
+                    "Disk local delete rollback failed"
+                );
+            }
+            return Err(err);
+        }
+        Ok(())
+    }
+
    async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
        let volume_dir = self.io_get_bucket_path(volume)?;
        let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
@@ -6123,7 +6161,20 @@ impl LocalDisk {
            return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
        }

-        let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
+        let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
+            Ok(data) => data,
+            Err(DiskError::FileNotFound) => {
+                // `deleted` alone can be an explicit marker purge; only
+                // `mark_deleted` may create metadata that was not present.
+                let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
+                    return Err(DiskError::FileNotFound);
+                };
+                return self
+                    .write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
+                    .await;
+            }
+            Err(err) => return Err(err),
+        };

        if data.is_empty() {
            return Err(DiskError::FileNotFound);
@@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk {
                }

                if fi.deleted && force_del_marker {
-                    if let Some(rollback_dir) = rollback_dir {
-                        let rollback_path = file_path.join(rollback_dir.to_string());
-                        fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
-                        fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
-                            .await
-                            .map_err(to_file_error)?;
-                    }
-                    if let Err(err) = self.write_metadata("", volume, path, fi).await {
-                        if let Some(rollback_dir) = rollback_dir
-                            && let Err(restore_err) =
-                                restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
-                        {
-                            warn!(
-                                volume,
-                                path,
-                                rollback_dir = %rollback_dir,
-                                error = ?restore_err,
-                                "failed to restore metadata after delete marker commit error"
-                            );
-                        }
-                        return Err(err);
-                    }
-                    return Ok(());
+                    return self
+                        .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
+                        .await;
                }

                return if fi.version_id.is_some() {
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_DISK: &str = "disk";
 const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
+const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
+const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;

 pub fn part_transaction_path(part_path: &str) -> String {
    match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
 use endpoint::Endpoint;
 use error::DiskError;
 use error::{Error, Result};
+use futures::stream::{self, StreamExt};
 use local::LocalDisk;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
        }
    }

+    #[tracing::instrument(level = "trace", skip_all)]
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        match self {
+            Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
+            Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
+        }
+    }
+
    #[tracing::instrument(level = "trace", skip_all)]
    async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
        match self {
@@ -1028,36 +1039,47 @@ where
    D: DiskAPI + ?Sized,
 {
    validate_batch_read_version_item_count(req.items.len())?;
+    let parallelism = batch_read_version_server_parallelism();

-    let mut responses = Vec::with_capacity(req.items.len());
-    for (index, item) in req.items.iter().enumerate() {
-        let response = match disk
-            .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
-            .await
-        {
-            Ok(file_info) => BatchReadVersionResp {
-                index,
-                path: item.path.clone(),
-                version_id: item.version_id.clone(),
-                success: true,
-                file_info,
-                error: String::new(),
-            },
-            Err(err) => BatchReadVersionResp {
-                index,
-                path: item.path.clone(),
-                version_id: item.version_id.clone(),
-                success: false,
-                file_info: FileInfo::default(),
-                error: err.to_string(),
-            },
-        };
-        responses.push(response);
-    }
+    let mut responses = stream::iter(req.items.into_iter().enumerate())
+        .map(|(index, item)| async move {
+            match disk
+                .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
+                .await
+            {
+                Ok(file_info) => BatchReadVersionResp {
+                    index,
+                    path: item.path,
+                    version_id: item.version_id,
+                    success: true,
+                    file_info,
+                    error: String::new(),
+                    error_code: 0,
+                },
+                Err(err) => BatchReadVersionResp {
+                    index,
+                    path: item.path,
+                    version_id: item.version_id,
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: err.to_string(),
+                    error_code: err.to_u32(),
+                },
+            }
+        })
+        .buffer_unordered(parallelism)
+        .collect::<Vec<_>>()
+        .await;
+    responses.sort_unstable_by_key(|response| response.index);

    Ok(responses)
 }

+fn batch_read_version_server_parallelism() -> usize {
+    rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
+        .clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
+}
+
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct CheckPartsResp {
    pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
    pub success: bool,
    pub file_info: FileInfo,
    pub error: String,
+    #[serde(default)]
+    pub error_code: u32,
 }

 pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
        assert!(!partial_valid_location.valid());
    }

+    #[test]
+    fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
+            assert_eq!(batch_read_version_server_parallelism(), 4);
+        });
+    }
+
+    #[test]
+    fn batch_read_version_server_parallelism_honors_env_with_bounds() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 8);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 1);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
+            assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
+        });
+    }
+
    /// Test FileInfoVersions find_version_index
    #[test]
    fn test_file_info_versions_find_version_index() {
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
    cluster::rpc::shutdown_background_monitors();
 }

+/// Publish that the process is ready to serve user-object GET traffic.
+///
+/// Experimental metadata coalescing is allowed to run only after this point so
+/// startup and internal metadata reads keep the original per-disk path.
+pub fn mark_get_metadata_read_version_coalescing_service_ready() {
+    runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
+}
+
 #[cfg(test)]
 mod rio_tests {
    #[test]
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
 pub struct NamespaceLockFence {
    signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
    #[cfg(test)]
-    forced_lost: Arc<std::sync::atomic::AtomicBool>,
+    forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
 }

 impl Debug for NamespaceLockFence {
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
        Self {
            signals: Arc::default(),
            #[cfg(test)]
-            forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
        }
    }

    pub(crate) fn is_lock_lost(&self) -> bool {
        #[cfg(test)]
-        if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
+        if self
+            .forced_lost
+            .iter()
+            .any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
+        {
            return true;
        }
        self.signals.iter().any(|signal| signal.is_lost())
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
    }

    fn extend(&mut self, other: &Self) {
-        if Arc::ptr_eq(&self.signals, &other.signals) {
-            return;
+        if !Arc::ptr_eq(&self.signals, &other.signals) {
+            Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
        }
-        Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
        #[cfg(test)]
-        if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
-            self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+        if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
+            Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
        }
    }

    #[cfg(test)]
    pub(crate) fn lost_for_test() -> Self {
        let fence = Self::new();
-        fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+        fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
        fence
    }

    #[cfg(test)]
    pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
        let fence = Self::new();
-        (fence.clone(), Arc::clone(&fence.forced_lost))
+        (fence.clone(), Arc::clone(&fence.forced_lost[0]))
    }
 }

@@ -411,6 +414,13 @@ impl ObjectOptions {
        self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
    }

+    #[cfg(test)]
+    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+        self.namespace_lock_fence
+            .get_or_insert_with(NamespaceLockFence::new)
+            .extend(fence);
+    }
+
    pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
        self.lifecycle_delete_all_journal
            .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
 use rustfs_lock::client::LockClient;
 use std::{
    collections::HashMap,
-    sync::{Arc, OnceLock},
+    sync::{
+        Arc, OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
    time::SystemTime,
 };
 use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;

+static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
+
+pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
+}
+
+pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
+}
+
 // Global singletons for backward compatibility with MinIO port.
 // These should be migrated to AppContext over time.
 // See issue #730 for migration plan.
@@ -160,6 +160,10 @@ pub struct InstanceContext {
    /// workers (scanner/heal/tier/lifecycle) without touching another instance.
    /// Replaces the process-global cancel-token static.
    background_cancel_token: OnceLock<CancellationToken>,
+    /// Serializes decommission data-movement operations with cancellation and
+    /// a subsequent restart. Readers are held across one object side effect;
+    /// the transition path takes the writer after cancelling the routine.
+    decommission_operation_gate: Arc<RwLock<()>>,
    /// Resolves object-encryption material at the application boundary.
    object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
    tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
            local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
            bucket_metadata_sys: std::sync::Mutex::new(None),
            background_cancel_token: OnceLock::new(),
+            decommission_operation_gate: Arc::new(RwLock::new(())),
            object_encryption_resolver: OnceLock::new(),
            tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
            transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
        self.lock_manager.clone()
    }

+    pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
+        Arc::clone(&self.decommission_operation_gate)
+    }
+
    /// Install the application-owned object-encryption resolver once.
    pub fn set_object_encryption_resolver(
        &self,
@@ -334,6 +334,7 @@ impl ECStore {
                            lifecycle_guard: bucket_incarnation_fence
                                .as_ref()
                                .and_then(|guard| guard.namespace_lock_guard()),
+                            ..Default::default()
                        },
                        "rebalance",
                    ),
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
    GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
    record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
 };
-use crate::disk::disk_store::DiskStoreRenameDataExt;
+use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
 use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
 use crate::disk::{
-    DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
-    PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
+    BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
+    OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
+    STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
 };
 use crate::erasure::coding::BitrotReader;
 use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
    future::Future,
    pin::Pin,
    sync::{
-        OnceLock,
+        Arc, OnceLock,
        atomic::{AtomicUsize, Ordering},
    },
    task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
    [bucket, object].join("/")
 }

+fn read_version_coalescing_enabled() -> bool {
+    let enabled = || {
+        rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
+            .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
+    };
+
+    #[cfg(test)]
+    {
+        enabled()
+    }
+
+    #[cfg(not(test))]
+    {
+        static ENABLED: OnceLock<bool> = OnceLock::new();
+        *ENABLED.get_or_init(enabled)
+    }
+}
+
+fn read_version_coalescing_delay() -> Duration {
+    #[cfg(test)]
+    {
+        let micros = rustfs_utils::get_env_u64(
+            ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+            DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+        );
+        Duration::from_micros(micros)
+    }
+
+    #[cfg(not(test))]
+    {
+        static DELAY: OnceLock<Duration> = OnceLock::new();
+        *DELAY.get_or_init(|| {
+            Duration::from_micros(rustfs_utils::get_env_u64(
+                ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+                DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+            ))
+        })
+    }
+}
+
+struct CoalescedReadVersionRequest {
+    item: BatchReadVersionItem,
+    tx: oneshot::Sender<disk::error::Result<FileInfo>>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+struct ReadVersionCoalescerKey {
+    disk: usize,
+    incl_free_versions: bool,
+    read_data: bool,
+    healing: bool,
+}
+
+impl ReadVersionCoalescerKey {
+    fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
+        Self {
+            disk: Arc::as_ptr(disk) as usize,
+            incl_free_versions: opts.incl_free_versions,
+            read_data: opts.read_data,
+            healing: opts.healing,
+        }
+    }
+}
+
+#[derive(Default)]
+struct ReadVersionCoalescer {
+    lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
+}
+
+fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
+    static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
+    COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
+}
+
+fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
+    counter!(
+        METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
+        "event" => event,
+        "item_count" => item_count.to_string()
+    )
+    .increment(1);
+}
+
+async fn read_version_via_coalescer(
+    disk: DiskStore,
+    org_bucket: &str,
+    bucket: &str,
+    object: &str,
+    version_id: &str,
+    opts: &ReadOptions,
+    allow_coalescing: bool,
+) -> disk::error::Result<FileInfo> {
+    if !allow_coalescing || !read_version_coalescing_enabled() {
+        return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
+    }
+    if !matches!(disk.as_ref(), Disk::Remote(_)) {
+        record_read_version_coalescer_event("bypass_non_remote", 1);
+        return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
+    }
+
+    let (tx, rx) = oneshot::channel();
+    let item = BatchReadVersionItem {
+        org_volume: org_bucket.to_string(),
+        volume: bucket.to_string(),
+        path: object.to_string(),
+        version_id: version_id.to_string(),
+    };
+    let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
+    let pending = {
+        let mut coalescer = read_version_coalescer().lock().await;
+        let lane = coalescer.lanes.entry(lane_key).or_default();
+        let schedule_delayed_flush = lane.is_empty();
+        lane.push(CoalescedReadVersionRequest { item, tx });
+        if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
+            coalescer.lanes.remove(&lane_key)
+        } else if schedule_delayed_flush {
+            let disk = disk.clone();
+            let task_opts = *opts;
+            tokio::spawn(async move {
+                tokio::time::sleep(read_version_coalescing_delay()).await;
+                flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
+            });
+            None
+        } else {
+            None
+        }
+    };
+
+    if let Some(pending) = pending {
+        flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
+    }
+
+    rx.await
+        .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
+}
+
+async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
+    let pending = {
+        let mut coalescer = read_version_coalescer().lock().await;
+        coalescer.lanes.remove(&lane_key).unwrap_or_default()
+    };
+    flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
+}
+
+async fn flush_read_version_coalescer_pending(
+    lane_key: ReadVersionCoalescerKey,
+    disk: DiskStore,
+    opts: ReadOptions,
+    pending: Vec<CoalescedReadVersionRequest>,
+) {
+    if pending.is_empty() {
+        return;
+    }
+
+    #[cfg(test)]
+    {
+        let mut observed_paths = HashSet::new();
+        for request in &pending {
+            if observed_paths.insert(request.item.path.as_str()) {
+                disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
+            }
+        }
+    }
+
+    let mut senders = Vec::with_capacity(pending.len());
+    let mut items = Vec::with_capacity(pending.len());
+    for request in pending {
+        senders.push(request.tx);
+        items.push(request.item);
+    }
+
+    let expected_items = items.clone();
+    record_read_version_coalescer_event("attempted_batch", items.len());
+    let result =
+        match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
+            .await
+        {
+            Ok(result) => result,
+            Err(_) => Err(DiskError::Timeout),
+        };
+    match result {
+        Ok(responses) => {
+            let results = map_batch_read_version_responses(&expected_items, responses);
+            for (tx, result) in senders.into_iter().zip(results) {
+                let _ = tx.send(result);
+            }
+        }
+        Err(err) => {
+            let message = err.to_string();
+            for tx in senders {
+                let _ = tx.send(Err(DiskError::other(message.clone())));
+            }
+        }
+    }
+}
+
+fn map_batch_read_version_responses(
+    expected_items: &[BatchReadVersionItem],
+    responses: Vec<BatchReadVersionResp>,
+) -> Vec<crate::disk::error::Result<FileInfo>> {
+    let mut results = (0..expected_items.len())
+        .map(|_| Err(DiskError::other("coalesced read_version response missing")))
+        .collect::<Vec<_>>();
+    let mut seen = vec![false; expected_items.len()];
+    for response in responses {
+        let Some(expected) = expected_items.get(response.index) else {
+            continue;
+        };
+        let Some(slot) = results.get_mut(response.index) else {
+            continue;
+        };
+        if seen[response.index] {
+            *slot = Err(DiskError::other("coalesced read_version response duplicate index"));
+            continue;
+        }
+        seen[response.index] = true;
+        if response.path != expected.path || response.version_id != expected.version_id {
+            *slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
+        } else {
+            *slot = if response.success {
+                Ok(response.file_info)
+            } else {
+                Err(batch_read_version_response_error(response.error_code, response.error))
+            };
+        }
+    }
+    results
+}
+
+fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
+    match DiskError::from_u32(error_code) {
+        Some(DiskError::Io(_)) | None => DiskError::other(error),
+        Some(error) => error,
+    }
+}
+
 pub(in crate::set_disk) fn bounded_metadata_fanout_order(
    bucket: &str,
    object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
    order
 }
 use tokio::io::{AsyncRead, ReadBuf};
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex, RwLock, oneshot};
 use tokio::task::JoinSet;

 pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
 pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
+const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
+const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
 pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
 /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
 /// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
            false,
            true,
            0,
+            false,
        )
        .await?;
        Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
            true,
            caller_allows_early_stop,
            default_parity_count,
+            false,
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
+        disks: &[Option<DiskStore>],
+        org_bucket: &str,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        read_data: bool,
+        incl_free_versions: bool,
+        caller_allows_early_stop: bool,
+        default_parity_count: usize,
+    ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
+        Self::read_all_fileinfo_inner(
+            disks,
+            org_bucket,
+            bucket,
+            object,
+            version_id,
+            read_data,
+            false,
+            incl_free_versions,
+            true,
+            caller_allows_early_stop,
+            default_parity_count,
+            true,
        )
        .await
    }
@@ -2408,6 +2680,7 @@ impl SetDisks {
        // subset would fail write quorum (backlog#872 regression).
        caller_allows_early_stop: bool,
        default_parity_count: usize,
+        allow_coalescing: bool,
    ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
        let early_stop_enabled =
            caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
                healing,
                incl_free_versions,
                default_parity_count,
+                allow_coalescing,
            )
            .await;
        }
@@ -2446,6 +2720,7 @@ impl SetDisks {
            healing,
            incl_free_versions,
            observe,
+            allow_coalescing,
        )
        .await
    }
@@ -2461,6 +2736,7 @@ impl SetDisks {
        healing: bool,
        incl_free_versions: bool,
        observe: bool,
+        allow_coalescing: bool,
    ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
        let fanout_start = observe.then(Instant::now);
        let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
                    if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
                        tokio::time::sleep(delay).await;
                    }
-                    disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
+                    read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
                        .await
                } else {
                    Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
        healing: bool,
        incl_free_versions: bool,
        default_parity_count: usize,
+        allow_coalescing: bool,
    ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
        let fanout_start = Instant::now();
        let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
                        if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
                            tokio::time::sleep(delay).await;
                        }
-                        disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
+                        read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
                            .await
                    } else {
                        Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {

    /// Kind label for the per-disk `read_version` metadata RPC.
    pub const KIND_READ_VERSION: &str = "read_version";
+    pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";

    /// Registry key: (object, kind, disk_index).
    type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
        drop(dirs);
    }

+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn metadata_read_version_coalescer_bypasses_local_disks() {
+        const DISKS: usize = 4;
+        let bucket = "coalesced-read-version-local-bypass-bucket";
+        let object_a = "coalesced-local-object-a";
+        let object_b = "coalesced-local-object-b";
+        let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
+
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
+            ],
+            async {
+                let calls = disk_call_counters::observe(object_a);
+                let disks_a = disks.clone();
+                let disks_b = disks.clone();
+                let read_a = tokio::spawn(async move {
+                    SetDisks::read_all_fileinfo_observed_for_get_object(
+                        &disks_a, "", bucket, object_a, "", false, false, false, 2,
+                    )
+                    .await
+                    .map(|(file_infos, errors, _)| (file_infos, errors))
+                });
+                tokio::task::yield_now().await;
+                let read_b = tokio::spawn(async move {
+                    SetDisks::read_all_fileinfo_observed_for_get_object(
+                        &disks_b, "", bucket, object_b, "", false, false, false, 2,
+                    )
+                    .await
+                    .map(|(file_infos, errors, _)| (file_infos, errors))
+                });
+
+                let (metadata_a, errs_a) = read_a
+                    .await
+                    .expect("first read task should not panic")
+                    .expect("first coalesced read should resolve");
+                let (metadata_b, errs_b) = read_b
+                    .await
+                    .expect("second read task should not panic")
+                    .expect("second coalesced read should resolve");
+
+                assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
+                assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
+                assert!(errs_a.iter().all(Option::is_none));
+                assert!(errs_b.iter().all(Option::is_none));
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_READ_VERSION),
+                    DISKS as u64,
+                    "local disks still execute the ordinary per-disk read_version path"
+                );
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
+                    0,
+                    "GET coalescing targets internode RPC count only and must not batch local disk reads"
+                );
+            },
+        )
+        .await;
+
+        drop(dirs);
+    }
+
+    #[tokio::test]
+    async fn metadata_read_version_coalescer_requires_get_object_intent() {
+        const DISKS: usize = 4;
+        let bucket = "coalesced-read-version-default-bypass-bucket";
+        let object = "default-bypass-object";
+        let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
+
+        temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
+            let calls = disk_call_counters::observe(object);
+            let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
+                .await
+                .expect("default metadata read should resolve");
+
+            assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
+            assert!(errs.iter().all(Option::is_none));
+            assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
+            assert_eq!(
+                calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
+                0,
+                "non-GET metadata paths must bypass coalescer even when the env gate is enabled"
+            );
+        })
+        .await;
+
+        drop(dirs);
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_preserves_index_and_errors() {
+        let expected_items = vec![
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-b".to_string(),
+                version_id: "v-b".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-c".to_string(),
+                version_id: "v-c".to_string(),
+            },
+        ];
+        let ok_file_info = FileInfo {
+            name: "object-a".to_string(),
+            ..Default::default()
+        };
+        let responses = vec![
+            BatchReadVersionResp {
+                index: 2,
+                path: "object-c".to_string(),
+                version_id: "v-c".to_string(),
+                success: false,
+                file_info: FileInfo::default(),
+                error: "disk read failed".to_string(),
+                error_code: 0,
+            },
+            BatchReadVersionResp {
+                index: 0,
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+                success: true,
+                file_info: ok_file_info,
+                error: String::new(),
+                error_code: 0,
+            },
+        ];
+
+        let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
+        let first = results
+            .next()
+            .expect("slot 0 should exist")
+            .expect("slot 0 should map the success response by index");
+        assert_eq!(first.name, "object-a");
+
+        let missing = results
+            .next()
+            .expect("slot 1 should exist")
+            .expect_err("slot 1 should stay missing");
+        assert!(
+            missing.to_string().contains("response missing"),
+            "unexpected missing response error: {missing}"
+        );
+
+        let failed = results
+            .next()
+            .expect("slot 2 should exist")
+            .expect_err("slot 2 should map the response error");
+        assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
+        assert!(results.next().is_none());
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
+        let expected_items = vec![
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-b".to_string(),
+                version_id: "v-b".to_string(),
+            },
+        ];
+        let results = map_batch_read_version_responses(
+            &expected_items,
+            vec![
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: DiskError::FileNotFound.to_string(),
+                    error_code: DiskError::FileNotFound.to_u32(),
+                },
+                BatchReadVersionResp {
+                    index: 1,
+                    path: "object-b".to_string(),
+                    version_id: "v-b".to_string(),
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: DiskError::FileVersionNotFound.to_string(),
+                    error_code: DiskError::FileVersionNotFound.to_u32(),
+                },
+            ],
+        );
+
+        assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
+        assert!(matches!(
+            results.get(1).expect("slot 1 should exist"),
+            Err(DiskError::FileVersionNotFound)
+        ));
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
+        let expected_items = vec![BatchReadVersionItem {
+            org_volume: String::new(),
+            volume: "bucket".to_string(),
+            path: "object-a".to_string(),
+            version_id: "v-a".to_string(),
+        }];
+        let mismatched = map_batch_read_version_responses(
+            &expected_items,
+            vec![BatchReadVersionResp {
+                index: 0,
+                path: "object-b".to_string(),
+                version_id: "v-a".to_string(),
+                success: true,
+                file_info: FileInfo {
+                    name: "object-b".to_string(),
+                    ..Default::default()
+                },
+                error: String::new(),
+                error_code: 0,
+            }],
+        )
+        .pop()
+        .expect("slot 0 should exist")
+        .expect_err("identity mismatch should fail closed");
+        assert!(
+            mismatched.to_string().contains("identity mismatch"),
+            "unexpected mismatch error: {mismatched}"
+        );
+
+        let duplicate = map_batch_read_version_responses(
+            &expected_items,
+            vec![
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: true,
+                    file_info: FileInfo {
+                        name: "object-a".to_string(),
+                        ..Default::default()
+                    },
+                    error: String::new(),
+                    error_code: 0,
+                },
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: true,
+                    file_info: FileInfo {
+                        name: "object-a".to_string(),
+                        ..Default::default()
+                    },
+                    error: String::new(),
+                    error_code: 0,
+                },
+            ],
+        )
+        .pop()
+        .expect("slot 0 should exist")
+        .expect_err("duplicate response index should fail closed");
+        assert!(
+            duplicate.to_string().contains("duplicate index"),
+            "unexpected duplicate error: {duplicate}"
+        );
+    }
+
    /// Isolation guard: unobserved objects record nothing (so parallel tests do
    /// not inflate one another), and a scope clears its own counts on drop.
    #[tokio::test]
@@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters;
 mod ctx;
 mod metadata;
 mod ops;
+#[cfg(test)]
+pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
+#[cfg(test)]
+pub(crate) use ops::object::DeleteObjectCommitBarrier;
 #[cfg(feature = "test-util")]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
@@ -3025,6 +3029,16 @@ pub struct SetDisks {
    storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
 }

+// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
+// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
+pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    left.iter()
+        .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
+        && right
+            .iter()
+            .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;

 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -3600,6 +3614,15 @@ impl SetDisks {
        &self.ctx
    }

+    /// Whether both sets' namespace-lock implementations cover the same object key.
+    pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
+        match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
+            (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
+            (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
+            _ => false,
+        }
+    }
+
    /// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
    #[cfg(test)]
    pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
@@ -4584,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
 }

 fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
-    opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
+    opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
 }

 fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
-    let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
+    let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
    let mut delete_marker = opts.versioned;

    if opts.version_id.is_some() {
@@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint};
 use crate::multipart_listing::paginate_multipart_listing;
 use futures::{StreamExt, stream};
 use std::future::Future;
+#[cfg(test)]
+use std::sync::atomic::AtomicBool;
 #[cfg(any(test, feature = "test-util"))]
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
@@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard {
 #[cfg(any(test, feature = "test-util"))]
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum MultipartCommitPause {
+    NewUploadBeforeLockLost,
    PutPartBeforeLockAcquire,
    PutPartBeforeLockLost,
    PutPartAfterRename,
@@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier {
    }
 }

+#[cfg(test)]
+struct NewMultipartUploadCommitObservationState {
+    bucket: String,
+    object: String,
+    committed: AtomicBool,
+}
+
+#[cfg(test)]
+pub(crate) struct NewMultipartUploadCommitObservation {
+    state: Arc<NewMultipartUploadCommitObservationState>,
+}
+
+#[cfg(test)]
+static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl NewMultipartUploadCommitObservation {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(NewMultipartUploadCommitObservationState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            committed: AtomicBool::new(false),
+        });
+        let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("new multipart upload commit observation mutex should not poison");
+        assert!(slot.is_none(), "new multipart upload commit observation must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) fn committed(&self) -> bool {
+        self.state.committed.load(Ordering::Acquire)
+    }
+}
+
+#[cfg(test)]
+impl Drop for NewMultipartUploadCommitObservation {
+    fn drop(&mut self) {
+        let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("new multipart upload commit observation mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
+    let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("new multipart upload commit observation mutex should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket && state.object == object)
+        .cloned();
+    if let Some(state) = state {
+        state.committed.store(true, Ordering::Release);
+    }
+}
+
 #[cfg(any(test, feature = "test-util"))]
 async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
    let barrier = {
@@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {

        let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);

+        #[cfg(any(test, feature = "test-util"))]
+        pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
+        if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "new_multipart_upload_commit",
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                required: 1,
+                achieved: 0,
+            });
+        }
+        if opts
+            .namespace_lock_fence
+            .as_ref()
+            .is_some_and(NamespaceLockFence::is_lock_lost)
+        {
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "new_multipart_upload_outer_lock",
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                required: 1,
+                achieved: 0,
+            });
+        }
        ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
        Self::write_unique_file_info(
            &shuffle_disks,
@@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
        )
        .await
        .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
+        #[cfg(test)]
+        observe_new_multipart_upload_commit(bucket, object);

        // evalDisks

@@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{
    transition_transaction::{
        TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
        TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
-        save_transition_transaction_record,
+        load_transition_transaction_record, save_transition_transaction_record,
    },
 };
 use crate::bucket::quota::reservation;
@@ -1294,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
            (prepared.snapshot, prepared.object_info)
        } else {
            match self
-                .get_object_fileinfo(
+                .get_object_fileinfo_for_get_object_reader(
                    bucket,
                    object,
                    opts,
@@ -2497,6 +2497,7 @@ impl SetDisks {
                        })
                        .await?,
                    );
+                    notify_put_object_commit_namespace_acquired(bucket, object);
                }
                #[cfg(not(any(test, feature = "test-util")))]
                {
@@ -4260,7 +4261,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction,

 async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
    if let Some(api) = api {
-        return delete_transition_transaction_record(api.clone(), transaction_id).await;
+        let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
+            Ok(transaction) => transaction,
+            Err(Error::ConfigNotFound) => return Ok(()),
+            Err(err) => return Err(err),
+        };
+        return delete_transition_transaction_record(api.clone(), &transaction).await;
    }
    Ok(())
 }
@@ -4644,6 +4650,7 @@ struct PutObjectCommitBarrierState {
    arrived: tokio::sync::Notify,
    release: tokio::sync::Notify,
    namespace_pending: tokio::sync::Notify,
+    namespace_acquired: std::sync::atomic::AtomicBool,
 }

 #[cfg(any(test, feature = "test-util"))]
@@ -4665,6 +4672,7 @@ impl PutObjectCommitBarrier {
            arrived: tokio::sync::Notify::new(),
            release: tokio::sync::Notify::new(),
            namespace_pending: tokio::sync::Notify::new(),
+            namespace_acquired: std::sync::atomic::AtomicBool::new(false),
        });
        let mut slot = PUT_OBJECT_COMMIT_BARRIER
            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -4699,6 +4707,10 @@ impl PutObjectCommitBarrier {
            .await
            .expect("put object should wait for the namespace lock after leaving the commit barrier");
    }
+
+    pub fn namespace_acquired(&self) -> bool {
+        self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
+    }
 }

 #[cfg(any(test, feature = "test-util"))]
@@ -4755,6 +4767,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
    }
 }

+#[cfg(any(test, feature = "test-util"))]
+fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
+    let barrier = PUT_OBJECT_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("put object commit barrier mutex should not poison")
+        .iter()
+        .find(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
+        })
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
+    }
+}
+
 #[cfg(test)]
 struct DeleteObjectCommitBarrierState {
    bucket: String,
@@ -4764,7 +4792,7 @@ struct DeleteObjectCommitBarrierState {
 }

 #[cfg(test)]
-struct DeleteObjectCommitBarrier {
+pub(crate) struct DeleteObjectCommitBarrier {
    state: Arc<DeleteObjectCommitBarrierState>,
 }

@@ -4774,7 +4802,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option

 #[cfg(test)]
 impl DeleteObjectCommitBarrier {
-    fn install(bucket: &str, object: &str) -> Self {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
        let state = Arc::new(DeleteObjectCommitBarrierState {
            bucket: bucket.to_string(),
            object: object.to_string(),
@@ -4790,13 +4818,13 @@ impl DeleteObjectCommitBarrier {
        Self { state }
    }

-    async fn wait_until_paused(&self) {
+    pub(crate) async fn wait_until_paused(&self) {
        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
            .await
            .expect("delete object should reach the deterministic commit barrier");
    }

-    fn release(&self) {
+    pub(crate) fn release(&self) {
        self.state.release.notify_one();
    }
 }
@@ -5918,6 +5946,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            if dobj.version_id.is_none() && (version_suspended || versioned) {
                vr.mod_time = Some(OffsetDateTime::now_utc());
                vr.deleted = true;
+                vr.mark_deleted = true;
                if versioned {
                    vr.version_id = Some(Uuid::new_v4());
                }
@@ -11804,6 +11833,7 @@ mod transition_upload_integrity_tests {
                crate::data_movement::SourceCleanupBucketFence {
                    expected_incarnation_id: None,
                    lifecycle_guard: Some(&bucket_guard),
+                    ..Default::default()
                },
                "test_data_movement",
            )
@@ -259,10 +259,33 @@ impl SetDisks {
        read_data: bool,
        caller_allows_early_stop: bool,
    ) -> Result<GetObjectFileInfo> {
-        self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
+        self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
            .await
    }

+    #[tracing::instrument(level = "debug", skip(self))]
+    #[hotpath::measure(impl_type = "SetDisks")]
+    pub(super) async fn get_object_fileinfo_for_get_object_reader(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        read_data: bool,
+        caller_allows_early_stop: bool,
+    ) -> Result<GetObjectFileInfo> {
+        let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
+            && crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
+        self.get_object_fileinfo_gated_inner(
+            bucket,
+            object,
+            opts,
+            read_data,
+            caller_allows_early_stop,
+            allow_read_version_coalescing,
+        )
+        .await
+    }
+
    /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
    /// quorum fanout. Read-before-write callers (object tagging) must use this:
    /// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
        opts: &ObjectOptions,
        read_data: bool,
        allow_early_stop: bool,
+    ) -> Result<GetObjectFileInfo> {
+        self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
+            .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn get_object_fileinfo_gated_inner(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        read_data: bool,
+        allow_early_stop: bool,
+        allow_read_version_coalescing: bool,
    ) -> Result<GetObjectFileInfo> {
        let vid = opts.version_id.clone().unwrap_or_default();
        let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
        // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
        // core/io_primitives.rs); unsafe requests and callers that opt out
        // (allow_early_stop=false) fall back to full-wait.
-        let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
-            &disks,
-            "",
-            bucket,
-            object,
-            vid.as_str(),
-            read_data,
-            false,
-            opts.incl_free_versions,
-            allow_early_stop,
-            self.default_parity_count,
-        )
-        .await?;
+        let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
+            Self::read_all_fileinfo_observed_for_get_object(
+                &disks,
+                "",
+                bucket,
+                object,
+                vid.as_str(),
+                read_data,
+                opts.incl_free_versions,
+                allow_early_stop,
+                self.default_parity_count,
+            )
+            .await?
+        } else {
+            Self::read_all_fileinfo_observed(
+                &disks,
+                "",
+                bucket,
+                object,
+                vid.as_str(),
+                read_data,
+                false,
+                opts.incl_free_versions,
+                allow_early_stop,
+                self.default_parity_count,
+            )
+            .await?
+        };
        let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
            GET_OBJECT_PATH_INTERNAL_META
        } else {
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-pub(crate) use object::ObjectLockDiagGuard;
+pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
 pub use object::{
    PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
    SnapshotConsistencyError,
@@ -400,7 +400,7 @@ impl ECStore {
        object: &str,
        opts: &ObjectOptions,
    ) -> Result<MultipartUploadResult> {
-        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
+        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
            .await
            .map(|(res, _, _)| res)
    }
@@ -410,20 +410,22 @@ impl ECStore {
        bucket: &str,
        object: &str,
        opts: &ObjectOptions,
+        mutation_fence: Option<&ObjectLockDiagGuard>,
    ) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
        check_new_multipart_args(bucket, object)?;
-        let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
-        let opts = &opts;
+        let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;

        if self.single_pool() {
+            self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
+                .await;
            return self.pools[0]
-                .new_multipart_upload(bucket, object, opts)
+                .new_multipart_upload(bucket, object, &opts)
                .await
                .map(|res| (res, 0, opts.expected_bucket_incarnation_id));
        }

        if opts.data_movement && opts.version_id.is_some() {
-            let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
+            let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
            if idx == opts.src_pool_idx {
                return Err(StorageError::DataMovementOverwriteErr(
                    bucket.to_owned(),
@@ -431,7 +433,9 @@ impl ECStore {
                    opts.version_id.clone().unwrap_or_default(),
                ));
            }
-            let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+            self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                .await;
+            let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
            return Ok((res, idx, opts.expected_bucket_incarnation_id));
        }

@@ -454,7 +458,9 @@ impl ECStore {
            .await?;

            if !res.uploads.is_empty() {
-                let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+                self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                    .await;
+                let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
                return Ok((res, idx, opts.expected_bucket_incarnation_id));
            }
        }
@@ -467,7 +473,9 @@ impl ECStore {
            ));
        }

-        let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+        self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+            .await;
+        let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
        Ok((res, idx, opts.expected_bucket_incarnation_id))
    }

@@ -704,13 +712,14 @@ impl ECStore {

    pub(crate) async fn complete_multipart_upload_for_data_movement(
        self: Arc<Self>,
-        target_pool_idx: usize,
+        target: (usize, Option<&ObjectLockDiagGuard>),
        bucket: &str,
        object: &str,
        upload_id: &str,
        uploaded_parts: Vec<CompletePart>,
        opts: &ObjectOptions,
    ) -> Result<ObjectInfo> {
+        let (target_pool_idx, mutation_fence) = target;
        check_complete_multipart_args(bucket, object, upload_id)?;
        if !opts.data_movement {
            return Err(Error::other("targeted multipart completion requires data_movement options"));
@@ -739,6 +748,8 @@ impl ECStore {
            snapshot.add_lock_fences(&mut opts);
            opts.object_lock_config_snapshot = Some(snapshot);
        }
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+            .await;
        #[cfg(test)]
        pause_data_movement_multipart_before_selected_completion(bucket).await;
        let pool = self
@@ -32,12 +32,13 @@ use crate::bucket::metadata_sys::{
 use crate::bucket::object_lock::objectlock_sys::{
    check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
 };
-use crate::bucket::replication::ReplicationObjectBridge;
+use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
+use crate::bucket::versioning::VersioningApi;
 use crate::disk::OldCurrentSize;
 use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot};
 use crate::set_disk::{
-    get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
-    is_lock_optimization_enabled, is_object_lock_diag_enabled,
+    SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
+    is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain,
 };
 use crate::storage_api_contracts::{
    namespace::NamespaceLocking as _,
@@ -352,6 +353,8 @@ impl fmt::Display for ObjectLockDiagMode {

 pub(crate) struct ObjectLockDiagGuard {
    guard: rustfs_lock::NamespaceLockGuard,
+    #[cfg(test)]
+    test_namespace_lock_fence: Option<NamespaceLockFence>,
    enabled: bool,
    op: &'static str,
    bucket: Option<String>,
@@ -373,6 +376,8 @@ impl ObjectLockDiagGuard {
    ) -> Self {
        Self {
            guard,
+            #[cfg(test)]
+            test_namespace_lock_fence: None,
            enabled,
            op,
            bucket,
@@ -393,6 +398,115 @@ impl ObjectLockDiagGuard {
    pub(crate) fn is_lock_lost(&self) -> bool {
        self.guard.is_lock_lost()
    }
+
+    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
+        opts.ensure_namespace_lock_fence();
+        if let Some(signal) = self.lock_lost_signal() {
+            opts.add_namespace_lock_lost_signal(signal);
+        }
+        #[cfg(test)]
+        if let Some(fence) = self.test_namespace_lock_fence.as_ref() {
+            opts.add_namespace_lock_fence_for_test(fence);
+        }
+    }
+}
+
+#[cfg(test)]
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub(crate) enum DecommissionMutationFenceTestPhase {
+    Migration,
+    SourceCleanup,
+}
+
+#[cfg(test)]
+struct DecommissionMutationFenceLossState {
+    bucket: String,
+    object: String,
+    phase: DecommissionMutationFenceTestPhase,
+    fence: NamespaceLockFence,
+    loss_handle: Arc<std::sync::atomic::AtomicBool>,
+}
+
+#[cfg(test)]
+pub(crate) struct DecommissionMutationFenceLossHook {
+    state: Arc<DecommissionMutationFenceLossState>,
+}
+
+#[cfg(test)]
+static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<DecommissionMutationFenceLossState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl DecommissionMutationFenceLossHook {
+    pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self {
+        let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
+        let state = Arc::new(DecommissionMutationFenceLossState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            phase,
+            fence,
+            loss_handle,
+        });
+        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("decommission mutation fence loss hooks should not poison");
+        assert!(slot.is_none(), "decommission mutation fence loss hook must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) fn mark_lost(&self) {
+        self.state.loss_handle.store(true, Ordering::Release);
+    }
+}
+
+#[cfg(test)]
+impl Drop for DecommissionMutationFenceLossHook {
+    fn drop(&mut self) {
+        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("decommission mutation fence loss hooks should not poison");
+        if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+fn decommission_mutation_fence_for_test(
+    bucket: &str,
+    object: &str,
+    phase: DecommissionMutationFenceTestPhase,
+) -> Option<NamespaceLockFence> {
+    DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("decommission mutation fence loss hooks should not poison")
+        .as_ref()
+        .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase)
+        .map(|hook| hook.fence.clone())
+}
+
+pub(crate) struct SourceCleanupMutationFence {
+    guard: ObjectLockDiagGuard,
+    source_lock_covered: bool,
+}
+
+impl SourceCleanupMutationFence {
+    pub(crate) fn source_lock_covered(&self) -> bool {
+        self.source_lock_covered
+    }
+
+    pub(crate) fn is_lock_lost(&self) -> bool {
+        self.guard.is_lock_lost()
+    }
+
+    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
+        self.guard.add_namespace_lock_fence(opts);
+    }
 }

 /// Opaque write-lock guard for the RestoreObject accept path; see
@@ -410,10 +524,7 @@ impl RestoreAcceptGuard {
    }

    pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
-        opts.ensure_namespace_lock_fence();
-        if let Some(signal) = self.0.lock_lost_signal() {
-            opts.add_namespace_lock_lost_signal(signal);
-        }
+        self.0.add_namespace_lock_fence(opts);
    }
 }

@@ -690,16 +801,6 @@ impl SelectObjectSnapshotLockLossWake {
    }
 }

-// LockRegistry clones its canonical client Arc for each endpoint host, so an
-// exact Arc set identifies one distributed namespace-lock quorum domain.
-fn same_distributed_lock_domain(left: &[Arc<dyn rustfs_lock::LockClient>], right: &[Arc<dyn rustfs_lock::LockClient>]) -> bool {
-    left.iter()
-        .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
-        && right
-            .iter()
-            .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
-}
-
 impl AsyncRead for SelectObjectSnapshotReader {
    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
        if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() {
@@ -805,7 +906,7 @@ fn resolve_latest_object_access(
 }

 fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
-    opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
+    (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
 }

 #[cfg(test)]
@@ -813,6 +914,8 @@ struct DeleteAfterObjectLockSnapshotBarrierState {
    bucket: String,
    arrived: tokio::sync::Notify,
    release: tokio::sync::Notify,
+    namespace_pending: tokio::sync::Notify,
+    namespace_acquired: AtomicBool,
 }

 #[cfg(test)]
@@ -832,6 +935,8 @@ impl DeleteAfterObjectLockSnapshotBarrier {
            bucket: bucket.to_string(),
            arrived: tokio::sync::Notify::new(),
            release: tokio::sync::Notify::new(),
+            namespace_pending: tokio::sync::Notify::new(),
+            namespace_acquired: AtomicBool::new(false),
        });
        let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
            .get_or_init(|| std::sync::Mutex::new(None))
@@ -849,6 +954,18 @@ impl DeleteAfterObjectLockSnapshotBarrier {
    pub(crate) fn release(&self) {
        self.state.release.notify_one();
    }
+
+    pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
+        let namespace_pending = self.state.namespace_pending.notified();
+        self.release();
+        tokio::time::timeout(Duration::from_secs(5), namespace_pending)
+            .await
+            .expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
+    }
+
+    pub(crate) fn namespace_acquired(&self) -> bool {
+        self.state.namespace_acquired.load(Ordering::Acquire)
+    }
 }

 #[cfg(test)]
@@ -873,6 +990,97 @@ async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
        .as_ref()
        .filter(|state| state.bucket == bucket)
        .cloned();
+    if let Some(state) = state {
+        state.arrived.notify_one();
+        state.release.notified().await;
+        state.namespace_pending.notify_one();
+    }
+}
+
+#[cfg(test)]
+fn notify_delete_namespace_acquired(bucket: &str) {
+    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("delete snapshot barrier mutex should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket)
+        .cloned();
+    if let Some(state) = state {
+        state.namespace_acquired.store(true, Ordering::Release);
+    }
+}
+
+#[cfg(test)]
+struct VersionedDeleteMarkerCommitBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct VersionedDeleteMarkerCommitBarrier {
+    state: Arc<VersionedDeleteMarkerCommitBarrierState>,
+}
+
+#[cfg(test)]
+static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<VersionedDeleteMarkerCommitBarrierState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl VersionedDeleteMarkerCommitBarrier {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(VersionedDeleteMarkerCommitBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("versioned delete-marker commit barrier mutex should not poison");
+        assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("versioned DELETE should reach the post-marker-commit barrier");
+    }
+
+    pub(crate) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for VersionedDeleteMarkerCommitBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("versioned delete-marker commit barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) {
+    let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("versioned delete-marker commit barrier mutex should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket && state.object == object)
+        .cloned();
    if let Some(state) = state {
        state.arrived.notify_one();
        state.release.notified().await;
@@ -913,6 +1121,160 @@ fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions
    lookup_opts
 }

+fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
+    let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock);
+    lookup_opts.skip_decommissioned = opts.data_movement;
+    lookup_opts
+}
+
+fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
+    pool_count > 0 && (!opts.versioned && !opts.version_suspended || opts.version_id.is_some())
+}
+
+fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
+    if object.version_id.is_some() {
+        return false;
+    }
+
+    let object_name = decode_dir_object(&object.object_name);
+    let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name);
+    versioned || version_suspended
+}
+
+fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option<usize>, pool_idx: usize) -> bool {
+    !creates_latest_marker || marker_target_pool_idx == Some(pool_idx)
+}
+
+#[cfg(test)]
+struct BatchDeletePoolErrorInjectionState {
+    bucket: String,
+    pool_idx: usize,
+    errors: std::collections::HashMap<String, Error>,
+    observed: std::sync::atomic::AtomicUsize,
+}
+
+#[cfg(test)]
+pub(crate) struct BatchDeletePoolErrorInjection {
+    state: Arc<BatchDeletePoolErrorInjectionState>,
+}
+
+#[cfg(test)]
+static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock<std::sync::Mutex<Option<Arc<BatchDeletePoolErrorInjectionState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl BatchDeletePoolErrorInjection {
+    pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self {
+        let state = Arc::new(BatchDeletePoolErrorInjectionState {
+            bucket: bucket.to_string(),
+            pool_idx,
+            errors: errors.into_iter().collect(),
+            observed: std::sync::atomic::AtomicUsize::new(0),
+        });
+        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("batch delete pool error injection mutex should not poison");
+        assert!(slot.is_none(), "batch delete pool error injection must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) fn observed(&self) -> usize {
+        self.state.observed.load(Ordering::Acquire)
+    }
+}
+
+#[cfg(test)]
+impl Drop for BatchDeletePoolErrorInjection {
+    fn drop(&mut self) {
+        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("batch delete pool error injection mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+fn inject_batch_delete_pool_errors(
+    bucket: &str,
+    pool_idx: usize,
+    object_names: &[String],
+    result: &mut (Vec<DeletedObject>, Vec<Option<Error>>),
+) {
+    let state = BATCH_DELETE_POOL_ERROR_INJECTION
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("batch delete pool error injection mutex should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx)
+        .cloned();
+    let Some(state) = state else {
+        return;
+    };
+
+    for (idx, object_name) in object_names.iter().enumerate() {
+        let Some(error) = state.errors.get(object_name) else {
+            continue;
+        };
+        if result.1[idx].is_none() && result.0[idx].found {
+            result.1[idx] = Some(error.clone());
+            state.observed.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+}
+
+fn resolve_batch_delete_pool_results<'a>(
+    initial_error: Option<Error>,
+    pool_results: impl IntoIterator<Item = (&'a DeletedObject, &'a Option<Error>)>,
+) -> (Option<DeletedObject>, Option<Error>, bool) {
+    let mut failure = initial_error.map(|err| (None, err));
+    let mut deleted = None;
+    let mut fallback: Option<(DeletedObject, Option<Error>)> = None;
+    let mut attempted = false;
+
+    for (pool_delete, pool_error) in pool_results {
+        attempted = true;
+        match pool_error {
+            Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => {
+                if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) {
+                    fallback = Some(((*pool_delete).clone(), Some(err.clone())));
+                }
+            }
+            Some(err) => {
+                if failure.is_none() {
+                    failure = Some((Some((*pool_delete).clone()), err.clone()));
+                }
+            }
+            None if pool_delete.found => {
+                if deleted.is_none() {
+                    deleted = Some((*pool_delete).clone());
+                }
+            }
+            None => {
+                if fallback.is_none() {
+                    fallback = Some(((*pool_delete).clone(), None));
+                }
+            }
+        }
+    }
+
+    if let Some((failed_delete, err)) = failure {
+        return (failed_delete, Some(err), attempted);
+    }
+    if let Some(deleted) = deleted {
+        return (Some(deleted), None, attempted);
+    }
+    if let Some((deleted, err)) = fallback {
+        return (Some(deleted), err, attempted);
+    }
+
+    (None, None, attempted)
+}
+
 fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
    let mut lookup_opts = opts.clone();
    lookup_opts.skip_decommissioned = true;
@@ -1541,6 +1903,89 @@ impl ECStore {
        )))
    }

+    pub(crate) async fn acquire_decommission_object_mutation_fence(
+        &self,
+        bucket: &str,
+        object: &str,
+    ) -> Result<ObjectLockDiagGuard> {
+        if self.ctx.lock_manager().is_disabled() {
+            return Err(Error::other("decommission object migration requires namespace locking"));
+        }
+
+        #[cfg(test)]
+        let test_namespace_lock_fence =
+            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration);
+        let object = encode_dir_object(object);
+        let mut opts = ObjectOptions::default();
+        let guard = self
+            .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts)
+            .await?
+            .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?;
+        #[cfg(test)]
+        let guard = {
+            let mut guard = guard;
+            guard.test_namespace_lock_fence = test_namespace_lock_fence;
+            guard
+        };
+        Ok(guard)
+    }
+
+    pub(super) async fn apply_decommission_target_mutation_fence(
+        &self,
+        target_pool_idx: usize,
+        object: &str,
+        opts: &mut ObjectOptions,
+        mutation_fence: Option<&ObjectLockDiagGuard>,
+    ) {
+        let Some(mutation_fence) = mutation_fence else {
+            return;
+        };
+
+        mutation_fence.add_namespace_lock_fence(opts);
+        let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
+        let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
+        opts.no_lock = match (fixed_set, target_set) {
+            (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
+            _ => false,
+        };
+    }
+
+    pub(crate) async fn acquire_decommission_source_cleanup_fence(
+        &self,
+        bucket: &str,
+        object: &str,
+        source_set: &SetDisks,
+    ) -> Result<SourceCleanupMutationFence> {
+        if self.ctx.lock_manager().is_disabled() {
+            return Err(Error::other("decommission source cleanup requires namespace locking"));
+        }
+
+        #[cfg(test)]
+        crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object);
+        #[cfg(test)]
+        let test_namespace_lock_fence =
+            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup);
+        let object = encode_dir_object(object);
+        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
+        let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await;
+        // Lock order: fixed store mutation domain first; source cleanup takes its
+        // hashed source-domain lock second only when this guard does not cover it.
+        let guard = self
+            .acquire_object_write_lock("decommission_source_cleanup", bucket, &object)
+            .await?;
+        #[cfg(test)]
+        let guard = {
+            let mut guard = guard;
+            guard.test_namespace_lock_fence = test_namespace_lock_fence;
+            guard
+        };
+
+        Ok(SourceCleanupMutationFence {
+            guard,
+            source_lock_covered,
+        })
+    }
+
    pub(crate) async fn acquire_all_object_read_locks(
        &self,
        op: &'static str,
@@ -1994,14 +2439,17 @@ impl ECStore {
        object: &str,
        data: &mut PutObjReader,
        opts: &ObjectOptions,
+        mutation_fence: Option<&ObjectLockDiagGuard>,
    ) -> Result<(usize, Result<ObjectInfo>)> {
        if !opts.data_movement {
            return Err(Error::other("data movement PUT requires data_movement options"));
        }
-        let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
+        let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
        let idx = self
            .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
            .await?;
+        self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence)
+            .await;
        let result = self.pools[idx]
            .put_object_with_old_current_size(bucket, &object, data, &opts)
            .await
@@ -2470,6 +2918,10 @@ impl ECStore {
        } else {
            None
        };
+        #[cfg(test)]
+        if _object_lock_guard.is_some() {
+            notify_delete_namespace_acquired(bucket);
+        }
        if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
            let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
            let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
@@ -2503,7 +2955,7 @@ impl ECStore {
            return Ok(ObjectInfo::default());
        }

-        let gopts = writer_pool_lookup_opts(&opts, true);
+        let gopts = delete_pool_lookup_opts(&opts, true);

        if opts.data_movement {
            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
@@ -2608,6 +3060,8 @@ impl ECStore {
            Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
                let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
                let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
+                #[cfg(test)]
+                pause_versioned_delete_marker_after_commit(bucket, object).await;
                obj.name = decode_dir_object(object);
                return Ok(obj);
            }
@@ -2646,7 +3100,7 @@ impl ECStore {
            None
        };

-        if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
+        if should_delete_from_all_pools(&opts, errs.len()) {
            let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await {
                Ok(obj) => obj,
                Err(err) => {
@@ -2670,6 +3124,8 @@ impl ECStore {

            match pool.delete_object(bucket, object, opts.clone()).await {
                Ok(res) => {
+                    #[cfg(test)]
+                    pause_versioned_delete_marker_after_commit(bucket, object).await;
                    if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
                        commit_prepared_tier_delete_journal_entry(api, je).await;
                    }
@@ -2817,32 +3273,104 @@ impl ECStore {
            Ok(guards) => guards,
            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
        };
+        #[cfg(test)]
+        if !_object_lock_guards.is_empty() {
+            notify_delete_namespace_acquired(bucket);
+        }
+
+        let delete_config_snapshot = opts
+            .delete_replication_config_snapshot
+            .as_deref()
+            .expect("batch delete replication config snapshot should be loaded");
+        let latest_marker_objects = objects
+            .iter()
+            .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot))
+            .collect::<Vec<_>>();
+        let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map(
+            |(object, creates_marker)| async move {
+                if *creates_marker {
+                    Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await)
+                } else {
+                    None
+                }
+            },
+        ))
+        .await;
+        let mut marker_target_pool_indices = Vec::with_capacity(objects.len());
+        for (idx, target_result) in marker_target_results.into_iter().enumerate() {
+            match target_result {
+                Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)),
+                Some(Err(err)) => {
+                    del_errs[idx] = Some(err);
+                    marker_target_pool_indices.push(None);
+                }
+                None => marker_target_pool_indices.push(None),
+            }
+        }

        let mut futures = Vec::with_capacity(self.pools.len());
-
        for pool in self.pools.iter() {
            if self.is_pool_rebalancing(pool.pool_idx).await {
                continue;
            }
-            futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone()));
+
+            let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects
+                .iter()
+                .enumerate()
+                .filter(|(idx, _)| {
+                    batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx)
+                })
+                .map(|(idx, object)| (idx, object.clone()))
+                .unzip();
+            if pool_objects.is_empty() {
+                continue;
+            }
+
+            let pool_opts = opts.clone();
+            futures.push(async move {
+                #[cfg(test)]
+                let pool_object_names = pool_objects
+                    .iter()
+                    .map(|object| object.object_name.clone())
+                    .collect::<Vec<_>>();
+                let result = pool.delete_objects(bucket, pool_objects, pool_opts).await;
+                #[cfg(test)]
+                let result = {
+                    let mut result = result;
+                    inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result);
+                    result
+                };
+                (object_indices, result)
+            });
        }

        let results = join_all(futures).await;

        for idx in 0..del_objects.len() {
-            for (dels, errs, pool_accounting) in results.iter() {
-                if errs[idx].is_none() && dels[idx].found {
-                    del_errs[idx] = None;
-                    del_objects[idx] = dels[idx].clone();
-                    accounting[idx] = pool_accounting[idx].clone();
-                    break;
-                }
+            let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| {
+                let pool_object_idx = object_indices.binary_search(&idx).ok()?;
+                Some((&dels[pool_object_idx], &errs[pool_object_idx]))
+            });
+            let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results);
+            if let Some(deleted) = deleted {
+                del_objects[idx] = deleted;
+            }
+            del_errs[idx] = error;

-                if del_errs[idx].is_none() {
-                    del_errs[idx] = errs[idx].clone();
-                    del_objects[idx] = dels[idx].clone();
-                    accounting[idx] = pool_accounting[idx].clone();
-                }
+            if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] {
+                del_objects[idx] = DeletedObject {
+                    object_name: objects[idx].object_name.clone(),
+                    version_id: objects[idx].version_id,
+                    ..Default::default()
+                };
+                del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone()));
+            }
+        }
+
+        #[cfg(test)]
+        for (idx, object) in objects.iter().enumerate() {
+            if del_errs[idx].is_none() && del_objects[idx].delete_marker {
+                pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await;
            }
        }

@@ -3417,6 +3945,80 @@ mod tests {
        assert!(!same_distributed_lock_domain(&[first, second], &[other]));
    }

+    #[tokio::test]
+    async fn decommission_fence_covers_dist_sets_with_same_clients_despite_different_namespaces() {
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
+        let mut second_set = (*original_sets.disk_set[1]).clone();
+        second_set.lockers = original_sets.disk_set[0].lockers.clone();
+        let mut sets = (*original_sets).clone();
+        sets.disk_set[1] = Arc::new(second_set);
+        let sets = Arc::new(sets);
+        ctx.update_erasure_type(SetupType::DistErasure).await;
+
+        assert!(
+            sets.disk_set[0]
+                .lockers
+                .iter()
+                .zip(&sets.disk_set[1].lockers)
+                .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)),
+            "the regression requires identical distributed lock clients"
+        );
+        assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index);
+
+        let pool_config = sets.endpoints.clone();
+        let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
+        let object = (0..1_000)
+            .map(|index| format!("decommission-dist-domain-{index}.bin"))
+            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
+            .expect("a key should hash to the second set namespace");
+        let mutation_fence = store
+            .acquire_decommission_object_mutation_fence("bucket", &object)
+            .await
+            .expect("the fixed distributed mutation fence should be acquired");
+        let target_lock = sets.disk_set[1]
+            .new_ns_lock("bucket", &object)
+            .await
+            .expect("the hashed-set namespace lock should be created");
+        let target_err = target_lock
+            .get_write_lock(Duration::from_millis(50))
+            .await
+            .expect_err("the fixed read fence must conflict through the shared clients");
+        assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. }));
+
+        let mut put_opts = ObjectOptions::default();
+        store
+            .apply_decommission_target_mutation_fence(0, &object, &mut put_opts, Some(&mutation_fence))
+            .await;
+        assert!(put_opts.no_lock, "migration target PUT must reuse the covering fixed fence");
+
+        let mut multipart_opts = ObjectOptions::default();
+        store
+            .apply_decommission_target_mutation_fence(0, &object, &mut multipart_opts, Some(&mutation_fence))
+            .await;
+        assert!(multipart_opts.no_lock, "migration target multipart must reuse the covering fixed fence");
+        drop(mutation_fence);
+
+        let cleanup_object = (0..1_000)
+            .map(|index| format!("decommission-dist-cleanup-{index}.bin"))
+            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
+            .expect("a cleanup key should hash to the second set namespace");
+        let source_fence = store
+            .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref())
+            .await
+            .expect("the fixed distributed cleanup fence should be acquired");
+        assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence");
+        let source_lock = sets.disk_set[1]
+            .new_ns_lock("bucket", &cleanup_object)
+            .await
+            .expect("the source-set namespace lock should be created");
+        let source_err = source_lock
+            .get_read_lock(Duration::from_millis(50))
+            .await
+            .expect_err("the fixed write fence must conflict through the shared clients");
+        assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. }));
+    }
+
    #[test]
    fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() {
        let nil = Uuid::nil();
@@ -4476,6 +5078,159 @@ mod tests {
        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
    }

+    #[test]
+    fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() {
+        let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true);
+
+        assert!(lookup_opts.no_lock);
+        assert!(!lookup_opts.skip_decommissioned);
+        assert!(lookup_opts.skip_rebalancing);
+
+        let explicit_version = delete_pool_lookup_opts(
+            &ObjectOptions {
+                versioned: true,
+                version_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            true,
+        );
+        assert!(!explicit_version.skip_decommissioned);
+    }
+
+    #[test]
+    fn delete_fans_out_for_unversioned_and_explicit_version_mutations() {
+        assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1));
+        assert!(should_delete_from_all_pools(
+            &ObjectOptions {
+                versioned: true,
+                version_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            2,
+        ));
+        assert!(!should_delete_from_all_pools(
+            &ObjectOptions {
+                versioned: true,
+                ..Default::default()
+            },
+            1,
+        ));
+        assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0));
+    }
+
+    #[test]
+    fn batch_delete_identifies_only_latest_versioned_markers() {
+        let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test(
+            s3s::dto::VersioningConfiguration {
+                status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
+                ..Default::default()
+            },
+            None,
+        );
+        let latest = ObjectToDelete {
+            object_name: "latest".to_string(),
+            ..Default::default()
+        };
+        assert!(batch_delete_creates_latest_marker(&latest, &versioned));
+        assert!(!batch_delete_targets_pool(true, Some(1), 0));
+        assert!(batch_delete_targets_pool(true, Some(1), 1));
+        assert!(!batch_delete_targets_pool(true, Some(1), 2));
+
+        let explicit = ObjectToDelete {
+            object_name: "explicit".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            ..Default::default()
+        };
+        assert!(!batch_delete_creates_latest_marker(&explicit, &versioned));
+        assert!(batch_delete_targets_pool(false, Some(1), 0));
+
+        let unversioned = DeleteReplicationConfigSnapshot::default();
+        assert!(!batch_delete_creates_latest_marker(&latest, &unversioned));
+        assert!(batch_delete_targets_pool(false, None, 0));
+    }
+
+    #[test]
+    fn batch_delete_pool_failures_override_success_in_any_pool_order() {
+        let success = DeletedObject {
+            object_name: "object".to_string(),
+            found: true,
+            ..Default::default()
+        };
+        let source_errors = [
+            StorageError::ErasureWriteQuorum,
+            StorageError::NamespaceLockQuorumUnavailable {
+                mode: "delete_objects_commit",
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                required: 1,
+                achieved: 0,
+            },
+        ];
+
+        for source_error in source_errors {
+            for source_first in [true, false] {
+                let failed = (DeletedObject::default(), Some(source_error.clone()));
+                let succeeded = (success.clone(), None);
+                let pool_results = if source_first {
+                    vec![failed, succeeded]
+                } else {
+                    vec![succeeded, failed]
+                };
+
+                let (_, error, attempted) =
+                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
+
+                assert!(attempted);
+                assert_eq!(error, Some(source_error.clone()));
+            }
+        }
+    }
+
+    #[test]
+    fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() {
+        let success = DeletedObject {
+            object_name: "object".to_string(),
+            found: true,
+            ..Default::default()
+        };
+        let missing_errors = [
+            StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
+            StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
+        ];
+
+        for missing_error in missing_errors {
+            let missing = (DeletedObject::default(), Some(missing_error.clone()));
+            for missing_first in [true, false] {
+                let succeeded = (success.clone(), None);
+                let pool_results = if missing_first {
+                    vec![missing.clone(), succeeded]
+                } else {
+                    vec![succeeded, missing.clone()]
+                };
+                let (deleted, error, attempted) =
+                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
+
+                assert!(attempted);
+                let deleted = deleted.expect("successful pool result should be retained");
+                assert!(deleted.found);
+                assert_eq!(deleted.object_name, success.object_name.as_str());
+                assert!(error.is_none());
+            }
+
+            let missing_only = [missing];
+            let (_, error, attempted) =
+                resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error)));
+            assert!(attempted);
+            assert_eq!(error, Some(missing_error));
+        }
+
+        let silent_missing = [(DeletedObject::default(), None)];
+        let (_, error, attempted) =
+            resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error)));
+        assert!(attempted);
+        assert!(error.is_none());
+    }
+
    #[test]
    fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
        let lookup_opts = data_movement_pool_lookup_opts(
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
    use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
    use crate::disk::error::DiskError;
    use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
        }
    }

+    fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
+        ObjectInfo {
+            version_id: Some(version_id),
+            etag,
+            ..object_info_with_mod_time(unix_ts, delete_marker)
+        }
+    }
+
    #[test]
    fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
        let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
    }

    #[test]
-    fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
+    fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
        let candidates = vec![
            LatestObjectInfoCandidate {
                info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
        assert_eq!(idx, 1);
    }

+    #[test]
+    fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 2,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 7,
+                err: None,
+            },
+        ];
+
+        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect("equivalent replicas must resolve deterministically");
+
+        assert_eq!(idx, 7);
+        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("divergent version ids must not silently resolve to the higher pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("divergent etags must not silently resolve to the higher pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("a delete marker tied with a live version must not be masked by the pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
+        let err = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(left),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(right),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect_err("equal-time identity divergence must fail closed");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        let mut data_dir = base.clone();
+        data_dir.data_dir = Some(Uuid::from_u128(2));
+        assert_equal_time_identity_conflict(base.clone(), data_dir);
+
+        let mut size = base.clone();
+        size.size = 1;
+        assert_equal_time_identity_conflict(base.clone(), size);
+
+        let mut actual_size = base.clone();
+        actual_size.actual_size = 1;
+        assert_equal_time_identity_conflict(base.clone(), actual_size);
+
+        let mut checksum = base.clone();
+        checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
+        assert_equal_time_identity_conflict(base.clone(), checksum);
+
+        let mut parts = base.clone();
+        parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+            etag: "part-etag".to_string(),
+            number: 1,
+            size: 1,
+            ..Default::default()
+        }]);
+        assert_equal_time_identity_conflict(base.clone(), parts);
+
+        let mut transition = base;
+        transition.transitioned_object.tier = "tier-a".to_string();
+        assert_equal_time_identity_conflict(
+            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
+            transition,
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut rustfs_alias = base.clone();
+        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+        let mut minio_alias = base.clone();
+        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "X-MINIO-INTERNAL-COMPRESSION".to_string(),
+            "zstd".to_string(),
+        )]));
+
+        let (_, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(rustfs_alias),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(minio_alias),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("same-value internal aliases should resolve");
+        assert_eq!(idx, 1);
+
+        let mut dual_alias = base.clone();
+        dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
+            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
+            ("x-minio-internal-compression".to_string(), "zstd".to_string()),
+        ]));
+        let mut single_alias = base;
+        single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+
+        let (_, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(dual_alias),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(single_alias),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("dual-key and single-key internal metadata should resolve");
+        assert_eq!(idx, 1);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut rustfs_alias = base.clone();
+        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+        let mut minio_alias = base;
+        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-minio-internal-compression".to_string(),
+            "snappy".to_string(),
+        )]));
+
+        assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
+        for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
+            let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+            let mut rustfs_alias = base.clone();
+            rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!(
+                    "X-RUSTFS-INTERNAL-{}{suffix}",
+                    suffix_prefix.to_uppercase(),
+                    suffix = "arn:aws:s3:::Bucket"
+                ),
+                "value".to_string(),
+            )]));
+            let mut minio_alias = base.clone();
+            minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
+                "value".to_string(),
+            )]));
+
+            let (_, idx) = resolve_latest_object_info_candidates(
+                vec![
+                    LatestObjectInfoCandidate {
+                        info: Some(rustfs_alias.clone()),
+                        idx: 0,
+                        err: None,
+                    },
+                    LatestObjectInfoCandidate {
+                        info: Some(minio_alias),
+                        idx: 1,
+                        err: None,
+                    },
+                ],
+                "bucket",
+                "object",
+                &ObjectOptions::default(),
+            )
+            .expect("dynamic internal aliases with the same target should resolve");
+            assert_eq!(idx, 1);
+
+            let mut different_target_case = base;
+            different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
+                "value".to_string(),
+            )]));
+
+            assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
+        }
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut first = base.clone();
+        first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
+            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
+            ("x-minio-internal-compression".to_string(), "snappy".to_string()),
+        ]));
+        let mut second = base;
+        second.user_defined = first.user_defined.clone();
+
+        assert_equal_time_identity_conflict(first, second);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        let mut replication = base.clone();
+        replication.replication_status_internal = Some("PENDING".to_string());
+        replication.replication_status = ReplicationStatusType::Pending;
+        assert_equal_time_identity_conflict(base.clone(), replication);
+
+        let mut purge = base.clone();
+        purge.version_purge_status_internal = Some("PENDING".to_string());
+        purge.version_purge_status = VersionPurgeStatusType::Pending;
+        assert_equal_time_identity_conflict(base.clone(), purge);
+
+        let mut decision = base;
+        decision.replication_decision = "replicate".to_string();
+        assert_equal_time_identity_conflict(
+            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
+            decision,
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
+        let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        without_mod_time.mod_time = None;
+        let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
+        let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
+        let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
+        older.data_dir = Some(Uuid::from_u128(2));
+
+        let (info, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(latest),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(older),
+                    idx: 9,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("older identity divergence must not affect the latest candidate");
+
+        assert_eq!(idx, 0);
+        assert_eq!(
+            info.mod_time,
+            Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: None,
+                idx: 1,
+                err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+            },
+        ];
+
+        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect("not-found pools must not block resolution of found candidates");
+
+        assert_eq!(idx, 0);
+        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
+    }
+
    #[test]
    fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
        let err = resolve_latest_object_info_candidates(
@@ -12,10 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::cmp::Ordering;
+use std::collections::HashMap;

 use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
 use crate::object_api::{ObjectInfo, ObjectOptions};
+use rustfs_utils::http::metadata_compat::{
+    SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
+    strip_internal_prefix_preserving_case,
+};
 use rustfs_utils::path::decode_dir_object;
 use time::OffsetDateTime;

@@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
    object: &str,
 ) -> Result<ObjectInfo> {
    result.map_err(|err| {
-        if err == Error::PreconditionFailed {
+        if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
            err
        } else {
            Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
@@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
 }

 fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
-    if err == Error::PreconditionFailed {
+    if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
        err
    } else {
        Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
    ))
 }

+fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
+    candidate
+        .info
+        .as_ref()
+        .map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
+}
+
+fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    left.transition_version_state == right.transition_version_state
+        && left.transitioned_object.name == right.transitioned_object.name
+        && left.transitioned_object.version_id == right.transitioned_object.version_id
+        && left.transitioned_object.tier == right.transitioned_object.tier
+        && left.transitioned_object.free_version == right.transitioned_object.free_version
+        && left.transitioned_object.status == right.transitioned_object.status
+}
+
+#[derive(PartialEq, Eq)]
+struct LatestUserDefinedIdentity {
+    internal: HashMap<String, String>,
+    other: HashMap<String, String>,
+}
+
+fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
+    let suffix = strip_internal_prefix_preserving_case(key)?;
+
+    for dynamic_prefix in [
+        SUFFIX_REPLICATION_RESET_ARN_PREFIX,
+        SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
+    ] {
+        let prefix_len = dynamic_prefix.len();
+        if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
+            && prefix.eq_ignore_ascii_case(dynamic_prefix)
+        {
+            return Some(format!("{dynamic_prefix}{remainder}"));
+        }
+    }
+
+    Some(suffix.to_lowercase())
+}
+
+fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
+    let mut identity = LatestUserDefinedIdentity {
+        internal: HashMap::with_capacity(user_defined.len()),
+        other: HashMap::with_capacity(user_defined.len()),
+    };
+
+    for (key, value) in user_defined {
+        if let Some(suffix) = normalize_internal_identity_suffix(key) {
+            if identity
+                .internal
+                .insert(suffix, value.clone())
+                .is_some_and(|previous| previous != *value)
+            {
+                return None;
+            }
+        } else {
+            identity.other.insert(key.clone(), value.clone());
+        }
+    }
+
+    Some(identity)
+}
+
+fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    match (
+        normalize_user_defined_identity(&left.user_defined),
+        normalize_user_defined_identity(&right.user_defined),
+    ) {
+        (Some(left), Some(right)) => left == right,
+        _ => false,
+    }
+}
+
+/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
+/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
+/// differ for the same object version while the selected winner still carries
+/// the chosen pool's layout. `put_object_reader` is also intentionally
+/// excluded because it is a transient request handle that `ObjectInfo::clone`
+/// drops. Every other ObjectInfo field is part of the production-visible
+/// identity and must agree before the pool index can provide a deterministic
+/// tie-break.
+fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    left.bucket == right.bucket
+        && left.name == right.name
+        && left.storage_class == right.storage_class
+        && left.mod_time == right.mod_time
+        && left.size == right.size
+        && left.actual_size == right.actual_size
+        && left.is_dir == right.is_dir
+        && same_user_defined_identity(left, right)
+        && left.user_tags == right.user_tags
+        && left.version_id == right.version_id
+        && left.data_dir == right.data_dir
+        && left.delete_marker == right.delete_marker
+        && same_transition_identity(left, right)
+        && left.restore_ongoing == right.restore_ongoing
+        && left.restore_expires == right.restore_expires
+        && left.parts == right.parts
+        && left.is_latest == right.is_latest
+        && left.content_type == right.content_type
+        && left.content_encoding == right.content_encoding
+        && left.expires == right.expires
+        && left.num_versions == right.num_versions
+        && left.successor_mod_time == right.successor_mod_time
+        && left.etag == right.etag
+        && left.inlined == right.inlined
+        && left.metadata_only == right.metadata_only
+        && left.version_only == right.version_only
+        && left.replication_status_internal == right.replication_status_internal
+        && left.replication_status == right.replication_status
+        && left.version_purge_status_internal == right.version_purge_status_internal
+        && left.version_purge_status == right.version_purge_status
+        && left.replication_decision == right.replication_decision
+        && left.checksum == right.checksum
+}
+
 pub(super) fn resolve_latest_object_info_candidates(
-    mut candidates: Vec<LatestObjectInfoCandidate>,
+    candidates: Vec<LatestObjectInfoCandidate>,
    bucket: &str,
    object: &str,
    opts: &ObjectOptions,
 ) -> Result<(ObjectInfo, usize)> {
-    candidates.sort_by(|a, b| {
-        let a_mod = if let Some(info) = &a.info {
-            info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
-        } else {
-            OffsetDateTime::UNIX_EPOCH
+    let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
+
+    if let Some(latest_mod_time) = latest_mod_time {
+        let mut latest_candidates = candidates
+            .into_iter()
+            .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
+            .collect::<Vec<_>>();
+
+        latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
+
+        let Some(winner) = latest_candidates.first() else {
+            return Err(Error::ErasureReadQuorum);
+        };
+        let Some(winner_info) = winner.info.as_ref() else {
+            return Err(Error::ErasureReadQuorum);
        };

-        let b_mod = if let Some(info) = &b.info {
-            info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
-        } else {
-            OffsetDateTime::UNIX_EPOCH
-        };
-
-        if a_mod == b_mod {
-            return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
+        if latest_candidates.iter().skip(1).any(|candidate| {
+            candidate
+                .info
+                .as_ref()
+                .is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
+        }) {
+            return Err(Error::ErasureReadQuorum);
        }

-        b_mod.cmp(&a_mod)
-    });
+        return Ok((winner_info.clone(), winner.idx));
+    }

    for candidate in candidates {
-        if let Some(info) = candidate.info {
-            return Ok((info, candidate.idx));
-        }
-
        if let Some(err) = candidate.err
            && !is_err_object_not_found(&err)
            && !is_err_version_not_found(&err)
@@ -191,6 +316,18 @@ mod tests {
        assert_eq!(err, Error::PreconditionFailed);
    }

+    #[test]
+    fn rebalance_delete_result_preserves_prefix_access_denied() {
+        let err = resolve_rebalance_delete_from_all_pools_result(
+            Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
+            "bucket",
+            "object",
+        )
+        .expect_err("prefix access denial should remain structured");
+
+        assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
+    }
+
    #[test]
    fn rebalance_delete_pool_result_preserves_precondition_failed() {
        let err = resolve_rebalance_delete_from_all_pools_results(
@@ -205,4 +342,19 @@ mod tests {

        assert_eq!(err, Error::PreconditionFailed);
    }
+
+    #[test]
+    fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![RebalanceDeletePoolResult {
+                pool_idx: 0,
+                result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
+            }],
+            "bucket",
+            "object",
+        )
+        .expect_err("prefix access denial should remain structured");
+
+        assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
+    }
 }
@@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
 pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
 pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
 pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";

 const OPERATION_LABEL: &str = "operation";
 const BACKEND_LABEL: &str = "backend";
@@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> {
 #[derive(Serialize, Deserialize, Clone, Default, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct BucketPolicy {
-    #[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
+    // RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
+    #[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
    pub id: ID,
    #[serde(rename = "Version")]
    pub version: String,
@@ -2786,7 +2787,7 @@ mod test {
        let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");

        // Verify empty fields are omitted
-        assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
+        assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");

        let statement = &parsed["Statement"][0];
        assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
@@ -2809,6 +2810,43 @@ mod test {
        assert_eq!(statement["Principal"]["AWS"], "*");
    }

+    #[test]
+    fn test_bucket_policy_deserializes_legacy_id() {
+        let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
+
+        let policy: BucketPolicy =
+            serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
+        assert!(policy.id.is_empty());
+        policy.is_valid().expect("legacy bucket policy should remain valid");
+
+        let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
+            .expect("non-empty legacy ID should deserialize");
+        assert_eq!(policy.id.0, "legacy-policy");
+
+        let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
+        assert_eq!(serialized["Id"], "legacy-policy");
+        assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
+    }
+
+    #[test]
+    fn test_bucket_policy_legacy_id_alias_remains_strict() {
+        let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
+        let error =
+            serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
+        assert!(
+            error.to_string().contains("unknown field `Unexpected`"),
+            "unexpected deserialization error: {error}"
+        );
+
+        let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
+        let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
+            .expect_err("canonical and legacy ID fields should not be accepted together");
+        assert!(
+            error.to_string().contains("duplicate field `Id`"),
+            "unexpected deserialization error: {error}"
+        );
+    }
+
    #[test]
    fn test_existing_object_tag_condition_helpers() {
        let identity_policy = Policy::parse_config(
@@ -29,8 +29,7 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
 pub use rustfs_data_usage::{
    AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
    DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
-    PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, UNKNOWN_TIER, UnknownTierStats,
-    hash_path, prefix_usage_in_cache,
+    PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache,
 };
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use tokio::time::{Duration, Instant, sleep, timeout};
@@ -206,8 +205,7 @@ impl ScannerSizeSummaryExt for SizeSummary {
            self.versions = self.versions.saturating_add(1);
        }

-        let logical_size = size.max(0);
-        let size = usize::try_from(logical_size).unwrap_or(usize::MAX);
+        let size = usize::try_from(size.max(0)).unwrap_or(usize::MAX);
        self.total_size = self.total_size.saturating_add(size);

        if oi.transitioned_object.free_version {
@@ -219,34 +217,12 @@ impl ScannerSizeSummaryExt for SizeSummary {
            tier = oi.transitioned_object.tier.clone();
        }

-        let builtin_tier = tier == storageclass::STANDARD || tier == storageclass::RRS;
-        let tier_registry_is_empty =
-            self.tier_stats.is_empty() || (self.tier_stats.len() == 1 && self.tier_stats.contains_key(UNKNOWN_TIER));
-        let known_tier = tier != UNKNOWN_TIER && (builtin_tier || self.tier_stats.contains_key(&tier));
-
-        // With no configured tier, retain the historical empty-map shape for
-        // ordinary STANDARD/RRS objects. A non-built-in key is still an
-        // observable unknown and must create only the fixed bucket.
-        if tier_registry_is_empty && known_tier {
-            return;
-        }
-
-        let accounting_key = if known_tier { tier.clone() } else { UNKNOWN_TIER.to_string() };
-        let tier_stats = self.tier_stats.entry(accounting_key).or_default();
-        let physical_size = u64::try_from(oi.size.max(0)).unwrap_or(0);
-        *tier_stats = tier_stats.add(&TierStats {
-            total_size: physical_size,
-            num_versions: 1,
-            num_objects: u64::from(oi.is_latest),
-        });
-        if !known_tier {
-            self.unknown_tier_stats.record_dimensions(
-                &tier,
-                u64::try_from(logical_size).unwrap_or(u64::MAX),
-                physical_size,
-                1,
-                u64::from(oi.is_latest),
-            );
+        if let Some(tier_stats) = self.tier_stats.get_mut(&tier) {
+            *tier_stats = tier_stats.add(&TierStats {
+                total_size: u64::try_from(oi.size).unwrap_or(0),
+                num_versions: 1,
+                num_objects: u64::from(oi.is_latest),
+            });
        }
    }
 }
@@ -368,10 +344,6 @@ pub struct DataUsageCacheInfo {
    pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
    #[serde(default)]
    pub cache_key_format: u16,
-    /// Registry generation used for the completed/partial scan. This is
-    /// process-local audit data; older cache writers omit it.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub tier_registry_generation: Option<u64>,
 }

 impl Serialize for DataUsageCacheInfo {
@@ -381,7 +353,7 @@ impl Serialize for DataUsageCacheInfo {
    {
        // Keep this metadata map-encoded so older readers can ignore fields
        // appended by newer scanner versions during rolling upgrades.
-        let mut state = serializer.serialize_map(Some(17))?;
+        let mut state = serializer.serialize_map(Some(16))?;
        state.serialize_entry("name", &self.name)?;
        state.serialize_entry("next_cycle", &self.next_cycle)?;
        state.serialize_entry("leader_epoch", &self.leader_epoch)?;
@@ -398,7 +370,6 @@ impl Serialize for DataUsageCacheInfo {
        state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
        state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
        state.serialize_entry("cache_key_format", &self.cache_key_format)?;
-        state.serialize_entry("tier_registry_generation", &self.tier_registry_generation)?;
        state.end()
    }
 }
@@ -419,17 +390,6 @@ pub(crate) enum DataUsageCachePrepareOutcome {
 }

 impl DataUsageCache {
-    /// Reconcile tier keys loaded from an older cache against the registry
-    /// frozen for this scan. New metadata is already routed through
-    /// `UNKNOWN_TIER`; this pass handles retired keys that predate that rule.
-    pub(crate) fn fold_retired_tiers(&mut self, tier_names: &[String]) {
-        for entry in self.cache.values_mut() {
-            if let Some(tiers) = entry.all_tier_stats.as_mut() {
-                tiers.fold_unknown_tiers(tier_names.iter().map(String::as_str));
-            }
-        }
-    }
-
    /// Prefix-level usage query over this (writer-side) cache; see
    /// [`prefix_usage_in_cache`] for the semantics
    /// (rustfs/backlog#1872).
@@ -915,7 +875,6 @@ impl DataUsageCache {
            delete_markers_total_count: flat.delete_markers as u64,
            objects_total_size: flat.size as u64,
            tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
-            unknown_tier_stats: flat.unknown_tier_stats.filter(|stats| !stats.is_empty()),
            buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
            buckets_usage,
            ..Default::default()
@@ -573,6 +573,7 @@ fn size_summary_add_saturates_all_usage_counters() {
            failed_count: usize::MAX,
        },
    );
+
    let mut increment = SizeSummary {
        total_size: 1,
        versions: 1,
@@ -587,24 +588,6 @@ fn size_summary_add_saturates_all_usage_counters() {
        failed_count: 1,
        ..Default::default()
    };
-    summary.tier_stats.insert(
-        UNKNOWN_TIER.to_string(),
-        TierStats {
-            total_size: u64::MAX,
-            num_versions: u64::MAX,
-            num_objects: u64::MAX,
-        },
-    );
-    increment.tier_stats.insert(
-        UNKNOWN_TIER.to_string(),
-        TierStats {
-            total_size: 1,
-            num_versions: 1,
-            num_objects: 1,
-        },
-    );
-    increment.unknown_tier_stats.unknown_bytes = 1;
-    increment.unknown_tier_stats.unknown_physical_bytes = 1;
    increment.repl_target_stats.insert(
        target.clone(),
        ReplTargetSizeSummary {
@@ -641,8 +624,6 @@ fn size_summary_add_saturates_all_usage_counters() {
    assert_eq!(target_summary.failed_size, i64::MAX);
    assert_eq!(target_summary.pending_count, usize::MAX);
    assert_eq!(target_summary.failed_count, usize::MAX);
-    assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, u64::MAX);
-    assert_eq!(summary.unknown_tier_stats.unknown_bytes, 1);
 }

 #[test]
@@ -692,162 +673,6 @@ fn size_summary_actions_accounting_accumulates_tier_stats() {
    );
 }

-#[test]
-fn unknown_tier_is_bounded_and_accounted() {
-    let mut summary = SizeSummary::new();
-    summary.tier_stats.insert("WARM".to_string(), TierStats::default());
-    let object = ObjectInfo {
-        storage_class: Some("retired-tier".to_string()),
-        size: 11,
-        is_latest: true,
-        ..Default::default()
-    };
-
-    summary.actions_accounting(&object, 11, 11);
-
-    assert_eq!(summary.tier_stats.len(), 2);
-    assert_eq!(summary.tier_stats.get(UNKNOWN_TIER).map(|stats| stats.total_size), Some(11));
-    assert_eq!(summary.unknown_tier_stats.unknown_bytes, 11);
-    assert_eq!(summary.unknown_tier_stats.unknown_physical_bytes, 11);
-    assert_eq!(summary.unknown_tier_stats.unknown_objects, 1);
-    assert!(
-        summary
-            .unknown_tier_stats
-            .diagnostics
-            .iter()
-            .all(|entry| !entry.contains("retired"))
-    );
-}
-
-#[test]
-fn unknown_tier_is_accounted_when_no_remote_tier_is_configured() {
-    let mut summary = SizeSummary::new();
-    let object = ObjectInfo {
-        storage_class: Some("retired-tier".to_string()),
-        size: 3,
-        is_latest: true,
-        ..Default::default()
-    };
-
-    summary.actions_accounting(&object, 9, 9);
-
-    assert_eq!(summary.tier_stats.len(), 1);
-    assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 3);
-    assert_eq!(summary.unknown_tier_stats.unknown_bytes, 9);
-    assert_eq!(summary.unknown_tier_stats.unknown_physical_bytes, 3);
-
-    let standard = ObjectInfo {
-        storage_class: Some(storageclass::STANDARD.to_string()),
-        size: 4,
-        is_latest: true,
-        ..Default::default()
-    };
-    summary.actions_accounting(&standard, 4, 4);
-    assert_eq!(summary.tier_stats.len(), 1, "built-ins preserve the no-tier map shape");
-}
-
-#[test]
-fn million_unique_tier_keys_do_not_grow_stats_map() {
-    let mut summary = SizeSummary::new();
-    summary.tier_stats.insert("WARM".to_string(), TierStats::default());
-    for index in 0..1_000_000_u64 {
-        let object = ObjectInfo {
-            storage_class: Some(format!("untrusted-tier-{index}")),
-            size: 1,
-            ..Default::default()
-        };
-        summary.actions_accounting(&object, 1, 1);
-    }
-
-    assert_eq!(summary.tier_stats.len(), 2);
-    assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 1_000_000);
-    assert_eq!(summary.unknown_tier_stats.unknown_bytes, 1_000_000);
-}
-
-#[test]
-fn unknown_tier_never_triggers_transition() {
-    let mut summary = SizeSummary::new();
-    summary.tier_stats.insert("WARM".to_string(), TierStats::default());
-    let mut object = ObjectInfo {
-        storage_class: Some("removed-tier".to_string()),
-        size: 7,
-        ..Default::default()
-    };
-    object.transitioned_object.status = TRANSITION_COMPLETE.to_string();
-    object.transitioned_object.tier = "removed-tier".to_string();
-
-    summary.actions_accounting(&object, 7, 7);
-
-    assert_eq!(summary.tier_stats.get("removed-tier"), None);
-    assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 7);
-}
-
-#[test]
-fn removed_tier_survives_restart_as_unknown() {
-    let mut summary = SizeSummary::new();
-    summary.tier_stats.insert("COLD".to_string(), TierStats::default());
-    let object = ObjectInfo {
-        storage_class: Some("retired-tier".to_string()),
-        size: 5,
-        ..Default::default()
-    };
-    summary.actions_accounting(&object, 5, 5);
-    let mut entry = DataUsageEntry::default();
-    entry.add_tier_sizes(&summary.tier_stats);
-    entry.add_unknown_tier_stats(&summary.unknown_tier_stats);
-    let encoded = rmp_serde::to_vec(&entry).expect("entry should encode");
-    let restored: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("entry should decode");
-
-    assert_eq!(restored.all_tier_stats.expect("tier stats persisted").tiers[UNKNOWN_TIER].total_size, 5);
-    assert_eq!(restored.unknown_tier_stats.expect("unknown stats persisted").unknown_bytes, 5);
-}
-
-#[test]
-fn tier_registry_refresh_does_not_mix_cycle_generations() {
-    let first = crate::TierRegistrySnapshot {
-        generation: 1,
-        names: Arc::from(["WARM".to_string()]),
-        refresh_failed: false,
-    };
-    let second = crate::TierRegistrySnapshot {
-        generation: 2,
-        names: Arc::from(["COLD".to_string()]),
-        refresh_failed: false,
-    };
-    assert_ne!(first.generation, second.generation);
-    assert_eq!(first.names.as_ref(), ["WARM".to_string()]);
-    assert_eq!(second.names.as_ref(), ["COLD".to_string()]);
-    assert!(first.refreshed(Err(())).refresh_failed);
-    assert!(!second.refreshed(Ok(Arc::from(["HOT".to_string()]))).refresh_failed);
-    assert_eq!(first.refreshed(Err(())).generation, first.generation);
-    assert_eq!(first.refreshed(Err(())).names, first.names);
-}
-
-#[test]
-fn unknown_tier_counter_uses_checked_arithmetic() {
-    let max = TierStats {
-        total_size: u64::MAX,
-        num_versions: u64::MAX,
-        num_objects: u64::MAX,
-    };
-    assert!(max.checked_add(&TierStats::default()).is_some());
-    assert!(
-        max.checked_add(&TierStats {
-            total_size: 1,
-            ..Default::default()
-        })
-        .is_none()
-    );
-
-    let mut unknown = UnknownTierStats {
-        unknown_bytes: u64::MAX,
-        ..Default::default()
-    };
-    unknown.record("overflow", 1, 1, 1);
-    assert_eq!(unknown.unknown_bytes, u64::MAX);
-    assert_eq!(unknown.unknown_objects, 1);
-}
-
 #[test]
 fn test_data_usage_entry_merge_sums_failed_objects() {
    let mut left = DataUsageEntry {
@@ -1101,7 +926,7 @@ const USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
    0xdc, 0x00, 0x20, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
    0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f,
    0x6b, 0x65, 0x79, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63,
-    0x6b, 0x65, 0x74, 0x8c, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10,
+    0x6b, 0x65, 0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10,
    0x00, 0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
    0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f, 0x73,
    0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62, 0x6a, 0x5f,
@@ -1109,8 +934,7 @@ const USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
    0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63,
    0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x02, 0xae,
    0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91, 0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d,
-    0x93, 0xcd, 0x08, 0x00, 0x02, 0x01, 0xb2, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73,
-    0x74, 0x61, 0x74, 0x73, 0xc0,
+    0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
 ];

 #[test]
@@ -94,45 +94,6 @@ static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
 static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0);
 static SCANNER_FOREGROUND_STREAM_READS: AtomicU64 = AtomicU64::new(0);

-/// Immutable tier registry captured at the beginning of a folder scan.
-/// Generation makes it possible to prove that a result was classified against
-/// one registry even when the process-wide TTL cache refreshes concurrently.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct TierRegistrySnapshot {
-    pub(crate) generation: u64,
-    pub(crate) names: Arc<[String]>,
-    /// True when the last refresh attempt failed and `names` is therefore a
-    /// retained last-good snapshot rather than a newly read registry.
-    pub(crate) refresh_failed: bool,
-}
-
-impl TierRegistrySnapshot {
-    /// Apply a refresh only when the registry read succeeds. A failed refresh
-    /// retains the prior generation, preventing a transient config failure
-    /// from classifying the remainder of a scan against an empty registry.
-    pub(crate) fn refreshed(&self, names: Result<Arc<[String]>, ()>) -> Self {
-        match names {
-            Ok(names) => Self {
-                generation: self.generation.saturating_add(1),
-                names,
-                refresh_failed: false,
-            },
-            Err(()) => Self {
-                refresh_failed: true,
-                ..self.clone()
-            },
-        }
-    }
-
-    pub(crate) fn initial(names: Arc<[String]>) -> Self {
-        Self {
-            generation: 1,
-            names,
-            refresh_failed: false,
-        }
-    }
-}
-
 pub fn current_scanner_activity() -> u64 {
    SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed)
 }
@@ -422,48 +383,24 @@ const TIER_NAME_CACHE_TTL: Duration = Duration::from_secs(30);
 /// `TIER_NAME_CACHE_TTL` later; a removed tier can leave an all-zero
 /// `TierStats` seed behind for one cache generation, which merges harmlessly
 /// by key in per-object accounting and disappears on the next refresh.
-static TIER_NAME_CACHE: RwLock<Option<(Instant, TierRegistrySnapshot)>> = RwLock::new(None);
-static TIER_REGISTRY_GENERATION: AtomicU64 = AtomicU64::new(0);
+static TIER_NAME_CACHE: RwLock<Option<(Instant, Arc<[String]>)>> = RwLock::new(None);

-/// Return one immutable registry snapshot for a scanner unit of work.
-pub(crate) async fn runtime_tier_registry() -> TierRegistrySnapshot {
+/// Tier names currently registered in the tier configuration, cached for
+/// `TIER_NAME_CACHE_TTL`.
+pub(crate) async fn runtime_tier_names() -> Arc<[String]> {
    {
        let cached = TIER_NAME_CACHE.read().unwrap_or_else(|err| err.into_inner()).clone();
-        if let Some((refreshed_at, snapshot)) = cached
+        if let Some((refreshed_at, names)) = cached
            && refreshed_at.elapsed() < TIER_NAME_CACHE_TTL
        {
-            return snapshot;
+            return names;
        }
    }

    let tiers = ecstore_get_global_tier_config_mgr().read().await.list_tiers();
    let names: Arc<[String]> = tiers.iter().map(|tier| tier.name.clone()).collect::<Vec<_>>().into();
-    let previous = TIER_NAME_CACHE
-        .read()
-        .unwrap_or_else(|err| err.into_inner())
-        .as_ref()
-        .map(|(_, snapshot)| snapshot.clone());
-    let snapshot = previous
-        .as_ref()
-        .map(|snapshot| {
-            let refreshed = snapshot.refreshed(Ok(Arc::clone(&names)));
-            TierRegistrySnapshot {
-                generation: TIER_REGISTRY_GENERATION.fetch_add(1, Ordering::Relaxed).saturating_add(1),
-                ..refreshed
-            }
-        })
-        .unwrap_or_else(|| TierRegistrySnapshot {
-            generation: TIER_REGISTRY_GENERATION.fetch_add(1, Ordering::Relaxed).saturating_add(1),
-            ..TierRegistrySnapshot::initial(names)
-        });
-    *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = Some((Instant::now(), snapshot.clone()));
-    snapshot
-}
-
-/// Tier names currently registered in the tier configuration, cached for
-/// `TIER_NAME_CACHE_TTL`.
-pub(crate) async fn runtime_tier_names() -> Arc<[String]> {
-    runtime_tier_registry().await.names
+    *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = Some((Instant::now(), Arc::clone(&names)));
+    names
 }

 /// Test-only cache reset; the production cache has no invalidation hook
@@ -55,9 +55,9 @@ use tracing::{debug, error, warn};
 use crate::{
    Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts, ReplicationConfig,
    ReplicationHealObject, ReplicationQueueAdmission, ReplicationStatusType, STORAGE_FORMAT_FILE, ScannerDiskExt as _,
-    ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, TierRegistrySnapshot, apply_expiry_rule,
-    apply_transition_rule, enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
-    path2_bucket_object_with_base_path, queue_replication_heal, runtime_tier_registry, scanner_is_erasure,
+    ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule,
+    enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
+    path2_bucket_object_with_base_path, queue_replication_heal, scanner_is_erasure,
    scanner_replication_config_for_lifecycle_eval,
 };
 use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete};
@@ -626,7 +626,6 @@ fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary)
    }

    into.add_tier_sizes(&summary.tier_stats);
-    into.add_unknown_tier_stats(&summary.unknown_tier_stats);
 }

 fn data_usage_root_has_progress(root: &DataUsageEntry) -> bool {
@@ -671,9 +670,6 @@ pub struct FolderScanner {
    budget: Arc<ScannerCycleBudget>,
    skip_heal: Arc<std::sync::atomic::AtomicBool>,
    local_disk: Arc<Disk>,
-    /// Tier registry frozen for this folder scan. A refresh applies to the
-    /// next scan and cannot mix generations in one aggregate.
-    tier_registry: TierRegistrySnapshot,
    pending_heals_changed: bool,
    #[cfg(test)]
    list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
@@ -1333,11 +1329,7 @@ impl FolderScanner {
                    continue;
                }

-                let sz = match self
-                    .local_disk
-                    .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
-                    .await
-                {
+                let sz = match self.local_disk.get_size(item.clone()).await {
                    Ok(sz) => sz,
                    Err(e) => {
                        let failure_action = classify_get_size_failure(&item, &e);
@@ -2169,10 +2161,6 @@ pub async fn scan_data_folder(

    let failed_object_ttl = rustfs_utils::get_env_u32(ENV_FAILED_OBJECT_TTL_SECS, DEFAULT_FAILED_OBJECT_TTL_SECS) as u64;
    let failed_objects_max = rustfs_utils::get_env_u32(ENV_FAILED_OBJECTS_MAX, DEFAULT_FAILED_OBJECTS_MAX) as usize;
-    let tier_registry = runtime_tier_registry().await;
-    let mut cache = cache;
-    cache.fold_retired_tiers(&tier_registry.names);
-    cache.info.tier_registry_generation = Some(tier_registry.generation);

    // Create folder scanner
    let mut scanner = FolderScanner {
@@ -2201,7 +2189,6 @@ pub async fn scan_data_folder(
        budget: budget.clone(),
        skip_heal,
        local_disk,
-        tier_registry,
        pending_heals_changed: false,
        #[cfg(test)]
        list_path_raw_options_observer: None,
@@ -289,42 +289,11 @@ impl ScannerItem {
        item.object_path()
    }

-    fn effective_tier(oi: &ObjectInfo) -> &str {
-        if oi.transitioned_object.status == crate::TRANSITION_COMPLETE {
-            oi.transitioned_object.tier.as_str()
-        } else {
-            oi.storage_class.as_deref().unwrap_or(crate::storageclass::STANDARD)
-        }
-    }
-
-    fn tier_is_known(oi: &ObjectInfo, tier_names: &[String]) -> bool {
-        let tier = Self::effective_tier(oi);
-        if tier == crate::data_usage_define::UNKNOWN_TIER {
-            return false;
-        }
-        tier == crate::storageclass::STANDARD || tier == crate::storageclass::RRS || tier_names.iter().any(|name| name == tier)
-    }
-
-    fn action_requires_known_tier(action: IlmAction) -> bool {
-        matches!(
-            action,
-            IlmAction::TransitionAction
-                | IlmAction::TransitionVersionAction
-                | IlmAction::DeleteAction
-                | IlmAction::DeleteVersionAction
-                | IlmAction::DeleteRestoredAction
-                | IlmAction::DeleteRestoredVersionAction
-                | IlmAction::DeleteAllVersionsAction
-                | IlmAction::DelMarkerDeleteAllVersionsAction
-        )
-    }
-
    pub async fn apply_actions(
        &mut self,
        object_infos: Vec<ObjectInfo>,
        lock_retention: Option<Arc<ObjectLockConfiguration>>,
        versioning_config: VersioningConfiguration,
-        tier_names: &[String],
        size_summary: &mut SizeSummary,
    ) {
        let object_path = self.object_path();
@@ -456,18 +425,6 @@ impl ScannerItem {
                let mut size = actual_size;
                let mut account_now = true;

-                // A retired/unknown source tier may point at a remote object
-                // that cannot be safely deleted or transitioned. Lifecycle
-                // evaluation is still useful for accounting, but all
-                // side-effecting tier actions fail closed until the registry
-                // recognizes the source again.
-                if !Self::tier_is_known(oi, tier_names) && Self::action_requires_known_tier(event.action) {
-                    size = self.heal_actions(oi, actual_size, size_summary).await;
-                    size_summary.actions_accounting(oi, size, actual_size);
-                    cumulative_size += size;
-                    continue;
-                }
-
                match event.action {
                    IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
                        debug!(
@@ -972,15 +929,4 @@ mod tests {
        assert_eq!(item.object_name, "object");
        assert_eq!(item.object_path(), "object");
    }
-
-    #[test]
-    fn unknown_tier_never_triggers_transition() {
-        let object = ObjectInfo {
-            storage_class: Some("retired-tier".to_string()),
-            ..Default::default()
-        };
-        assert!(!ScannerItem::tier_is_known(&object, &["WARM".to_string()]));
-        assert!(ScannerItem::action_requires_known_tier(IlmAction::TransitionAction));
-        assert!(ScannerItem::action_requires_known_tier(IlmAction::DeleteVersionAction));
-    }
 }
@@ -325,11 +325,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
        budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()),
        skip_heal: Arc::new(AtomicBool::new(false)),
        local_disk: disk,
-        tier_registry: crate::TierRegistrySnapshot {
-            generation: 0,
-            names: Arc::new([]),
-            refresh_failed: false,
-        },
        pending_heals_changed: false,
        list_path_raw_options_observer: None,
    };
@@ -498,9 +498,6 @@ pub trait ScannerIODisk: Send + Sync + Debug + 'static {
    ) -> Result<ScannerDiskScanOutcome>;

    async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary>;
-
-    /// Read one object using a registry snapshot captured at scan start.
-    async fn get_size_with_tier_names(&self, item: ScannerItem, tier_names: &[String]) -> Result<SizeSummary>;
 }

 #[derive(Debug)]
@@ -216,7 +216,6 @@ pub(super) fn completed_data_usage_info(
        delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
        objects_total_size: u64::try_from(total.size).ok()?,
        tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
-        unknown_tier_stats: total.unknown_tier_stats.filter(|stats| !stats.is_empty()),
        buckets_count: u64::try_from(all_buckets.len()).ok()?,
        bucket_sizes,
        buckets_usage,
@@ -13,38 +13,29 @@
 // limitations under the License.
 /// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
 use super::*;
-use crate::UNKNOWN_TIER;

 ///
 /// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
 ///
-/// Preserves the original no-tier shape: with no tiers configured the map
-/// stays completely empty (STANDARD/RRS/UNKNOWN are not seeded either).
-/// Otherwise the standard storage classes and one fixed unknown bucket are
-/// seeded alongside every configured tier so per-object accounting never
-/// inserts an untrusted metadata key.
+/// Preserves the original seeding semantics: with no tiers configured the map
+/// stays completely empty (STANDARD/RRS are not seeded either); otherwise the
+/// standard storage classes are seeded alongside every configured tier so
+/// per-object accounting always finds its tier key.
 pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap<String, TierStats> {
-    let mut tier_stats = HashMap::with_capacity(tier_names.len() + 3);
+    let mut tier_stats = HashMap::with_capacity(tier_names.len() + 2);
    for tier_name in tier_names {
-        if tier_name != UNKNOWN_TIER {
-            tier_stats.insert(tier_name.clone(), TierStats::default());
-        }
+        tier_stats.insert(tier_name.clone(), TierStats::default());
    }
    if !tier_stats.is_empty() {
        tier_stats.insert(storageclass::STANDARD.to_string(), TierStats::default());
        tier_stats.insert(storageclass::RRS.to_string(), TierStats::default());
-        tier_stats.insert(UNKNOWN_TIER.to_string(), TierStats::default());
    }
    tier_stats
 }

 #[async_trait::async_trait]
 impl ScannerIODisk for Disk {
-    async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary> {
-        self.get_size_with_tier_names(item, &runtime_tier_names().await).await
-    }
-
-    async fn get_size_with_tier_names(&self, mut item: ScannerItem, tier_names: &[String]) -> Result<SizeSummary> {
+    async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
        let done_object = Metrics::time(Metric::ScanObject);

        if !is_xl_meta_path(&item.path) {
@@ -116,10 +107,10 @@ impl ScannerIODisk for Disk {

        let mut size_summary = SizeSummary::default();

-        // The caller supplies one registry snapshot for the whole folder scan;
-        // seeding from it prevents a TTL refresh from mixing generations in a
-        // single result.
-        size_summary.tier_stats = tier_stats_template(tier_names);
+        // Tier names come from the process-wide TTL cache; seeding from them
+        // replaces the per-object clone of every full TierConfig.
+        let tier_names = runtime_tier_names().await;
+        size_summary.tier_stats = tier_stats_template(&tier_names);

        let lock_config = object_lock_config_for_scanner_item(&item).await;

@@ -129,7 +120,7 @@ impl ScannerIODisk for Disk {
        // `object_infos`.
        global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);

-        item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary)
+        item.apply_actions(object_infos, lock_config, versioning_config, &mut size_summary)
            .await;

        if !free_version_infos.is_empty() {
@@ -23,7 +23,7 @@ use crate::storage_api::owner::{
 use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
 use crate::{
    DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
-    ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
+    ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
    init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
 };
 use rustfs_filemeta::FileInfo;
@@ -986,8 +986,8 @@ fn is_xl_meta_path_accepts_forward_separator() {
 fn tier_stats_template_seeds_tiers_and_standard_classes() {
    let template = tier_stats_template(&["WARM".to_string(), "COLD".to_string()]);

-    assert_eq!(template.len(), 5);
-    for tier in ["WARM", "COLD", storageclass::STANDARD, storageclass::RRS, UNKNOWN_TIER] {
+    assert_eq!(template.len(), 4);
+    for tier in ["WARM", "COLD", storageclass::STANDARD, storageclass::RRS] {
        assert_eq!(template.get(tier), Some(&TierStats::default()), "missing seed for tier {tier}");
    }
 }
@@ -12,6 +12,7 @@ for later deletion.

 ## Open Items

+- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
 - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
 - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
 - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
@@ -58,7 +58,7 @@
 | heal_erasure_disk_rebuild_test | 4 | 🌙 |
 | inline_fast_path_cluster_test | 16 |  |
 | internode_rpc_signature_e2e_test | 5 |  |
-| kms | 48 |  |
+| kms | 46 |  |
 | leading_slash_key_test | 2 | ✅ |
 | lifecycle_regression_test | 4 |  |
 | list_buckets_auth_test | 1 | ✅ |
@@ -99,4 +99,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |

-**Total listed: 577 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 455 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-21.
+**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
@@ -25,7 +25,7 @@
    },
    {
      "name": "heartbeat",
-      "status": "reserved",
+      "status": "populated",
      "purpose": "Heartbeat payloads, Connect receive time, and freshness window behavior."
    },
    {
@@ -0,0 +1,5 @@
+975c1ca53eefeef6766a6fc0b3d3281f7408255342b0686e5e2aee5ad055414c  duplicate.json
+963529a38a02849c6c2acc6d72668dca9f63218b49c89fae41a451b584850411  overflow.json
+e3adeee1c8a19aa17e70894896fb79c072e3785bea3611b93c11e79f039ed5af  stale.json
+35b9cebd8525389a701e8fe69fbe96407bcb31aa28392fe95babf4a4886985ad  unknown.json
+37941735dbd6ad3d238258a7b2cae6f0b3aa0ecaae1d8817817c3d718d11d633  valid.json
@@ -0,0 +1,9 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "heartbeat",
+  "fixture": "duplicate",
+  "description": "An exact requestId replay returns the first result and creates no second heartbeat.",
+  "first": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
+  "replay": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
+  "expected": {"decision": "DUPLICATE", "heartbeatWrites": 1, "events": 1, "sameResponse": true}
+}
@@ -0,0 +1,11 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "heartbeat",
+  "fixture": "overflow",
+  "description": "Values beyond frozen bounds are rejected before persistence.",
+  "vectors": [
+    {"field": "sequence", "value": 9007199254740992, "maximum": 9007199254740991},
+    {"field": "coarseNodeSummary.total", "value": 4097, "maximum": 4096}
+  ],
+  "expected": {"decision": "REJECT", "httpStatus": 422, "status": "INVALID_ARGUMENT"}
+}
@@ -0,0 +1,9 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "heartbeat",
+  "fixture": "stale",
+  "description": "A lower heartbeat sequence is retained as history and cannot replace the current projection.",
+  "head": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42},
+  "late": {"requestId": "7c4d2e10-9f83-4a5b-b6c7-d8e9f0a1b2c3", "sequence": 9},
+  "expected": {"decision": "ACCEPT_HISTORY", "currentSequence": 42, "historySequence": 9}
+}
@@ -0,0 +1,18 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "heartbeat",
+  "fixture": "unknown",
+  "description": "Unknown optional members and capabilities are accepted, discarded before hashing, and never stored or echoed.",
+  "requestAdditions": {
+    "telemetryProfile": "extended",
+    "authorization": "Bearer non-functional-example",
+    "capabilities": ["heartbeat", "future.capability"],
+    "coarseNodeSummary": {"rackNames": ["customer-rack"]}
+  },
+  "expected": {
+    "decision": "ACCEPT",
+    "storedCapabilities": ["heartbeat"],
+    "discarded": ["authorization", "future.capability", "telemetryProfile", "coarseNodeSummary.rackNames"],
+    "echoed": []
+  }
+}
@@ -0,0 +1,21 @@
+{
+  "protocolVersion": "v1",
+  "fixtureSet": "heartbeat",
+  "fixture": "valid",
+  "description": "A bounded L0 heartbeat. clientTime is advisory; Connect's receivedAt is online authority.",
+  "request": {
+    "protocolVersion": "v1",
+    "requestId": "550e8400-e29b-41d4-a716-446655440000",
+    "agentVersion": "rustfs-agent/1.19.4",
+    "capabilities": ["heartbeat", "inventory"],
+    "sequence": 42,
+    "clientTime": "2026-08-22T01:02:03Z",
+    "coarseNodeSummary": {"total": 8, "healthy": 7, "degraded": 1}
+  },
+  "expected": {
+    "decision": "ACCEPT",
+    "acceptedVersion": "v1",
+    "responseFields": ["serverTime", "acceptedVersion", "capabilityHints"],
+    "onlineAuthority": "serverTime"
+  }
+}
@@ -0,0 +1,173 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::ffi::OsString;
+use std::fs;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use super::{CredentialStore, IdentityStore};
+
+pub const ENV_CONNECT_ENDPOINT: &str = "RUSTFS_CONNECT_ENDPOINT";
+pub const ENV_CONNECT_ROOT_CA_FILE: &str = "RUSTFS_CONNECT_ROOT_CA_FILE";
+pub const ENV_CONNECT_STATE_DIR: &str = "RUSTFS_CONNECT_STATE_DIR";
+
+#[derive(Clone, Copy, Debug)]
+pub struct HeartbeatSchedule {
+    pub cadence: Duration,
+    pub jitter: Duration,
+    pub timeout: Duration,
+    pub initial_backoff: Duration,
+    pub max_backoff: Duration,
+}
+
+impl Default for HeartbeatSchedule {
+    fn default() -> Self {
+        Self {
+            cadence: Duration::from_secs(30),
+            jitter: Duration::from_secs(3),
+            timeout: Duration::from_secs(5),
+            initial_backoff: Duration::from_secs(1),
+            max_backoff: Duration::from_secs(5 * 60),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct HeartbeatConfig {
+    pub endpoint: String,
+    pub root_ca_pem: Vec<u8>,
+    pub identity_store: IdentityStore,
+    pub credential_store: CredentialStore,
+    pub state_path: PathBuf,
+    pub schedule: HeartbeatSchedule,
+}
+
+impl HeartbeatConfig {
+    pub fn new(
+        endpoint: impl Into<String>,
+        root_ca_pem: impl Into<Vec<u8>>,
+        identity_store: IdentityStore,
+        credential_store: CredentialStore,
+        state_path: impl Into<PathBuf>,
+    ) -> Self {
+        Self {
+            endpoint: endpoint.into(),
+            root_ca_pem: root_ca_pem.into(),
+            identity_store,
+            credential_store,
+            state_path: state_path.into(),
+            schedule: HeartbeatSchedule::default(),
+        }
+    }
+
+    pub fn from_env() -> Result<Option<Self>, HeartbeatConfigError> {
+        Self::from_env_values(
+            env::var_os(ENV_CONNECT_ENDPOINT),
+            env::var_os(ENV_CONNECT_ROOT_CA_FILE),
+            env::var_os(ENV_CONNECT_STATE_DIR),
+        )
+    }
+
+    fn from_env_values(
+        endpoint: Option<OsString>,
+        root_ca_file: Option<OsString>,
+        state_dir: Option<OsString>,
+    ) -> Result<Option<Self>, HeartbeatConfigError> {
+        let configured = endpoint.is_some() || root_ca_file.is_some() || state_dir.is_some();
+        if !configured {
+            return Ok(None);
+        }
+        let (Some(endpoint), Some(root_ca_file), Some(state_dir)) = (endpoint, root_ca_file, state_dir) else {
+            return Err(HeartbeatConfigError::Partial);
+        };
+        let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?;
+        let root_ca_file = PathBuf::from(root_ca_file);
+        let state_dir = PathBuf::from(state_dir);
+        if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() || state_dir.as_os_str().is_empty() {
+            return Err(HeartbeatConfigError::Partial);
+        }
+        let root_ca_pem = fs::read(&root_ca_file).map_err(|source| HeartbeatConfigError::RootCertificate {
+            path: root_ca_file,
+            source,
+        })?;
+        Ok(Some(Self::new(
+            endpoint,
+            root_ca_pem,
+            IdentityStore::new(state_dir.join("identity")),
+            CredentialStore::new(state_dir.join("credential")),
+            state_dir.join("heartbeat/state.json"),
+        )))
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum HeartbeatConfigError {
+    #[error(
+        "Connect heartbeat configuration requires RUSTFS_CONNECT_ENDPOINT, RUSTFS_CONNECT_ROOT_CA_FILE, and RUSTFS_CONNECT_STATE_DIR"
+    )]
+    Partial,
+    #[error("RUSTFS_CONNECT_ENDPOINT is not valid UTF-8")]
+    EndpointEncoding,
+    #[error("failed to read the Connect root CA at {path}: {source}")]
+    RootCertificate {
+        path: PathBuf,
+        #[source]
+        source: std::io::Error,
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{HeartbeatConfig, HeartbeatConfigError};
+    use std::ffi::OsString;
+
+    #[test]
+    fn absent_environment_is_disabled_without_side_effects() {
+        assert!(
+            HeartbeatConfig::from_env_values(None, None, None)
+                .expect("absent config")
+                .is_none()
+        );
+    }
+
+    #[test]
+    fn partial_environment_is_rejected() {
+        assert!(matches!(
+            HeartbeatConfig::from_env_values(Some(OsString::from("https://connect.example/agent/")), None, None),
+            Err(HeartbeatConfigError::Partial)
+        ));
+    }
+
+    #[test]
+    fn complete_environment_builds_the_durable_paths() {
+        let temp = tempfile::tempdir().expect("tempdir");
+        let root = temp.path().join("root.pem");
+        std::fs::write(&root, b"root certificate").expect("root CA");
+        let state = temp.path().join("state");
+        let config = HeartbeatConfig::from_env_values(
+            Some(OsString::from("https://connect.example/agent/")),
+            Some(root.into_os_string()),
+            Some(state.clone().into_os_string()),
+        )
+        .expect("complete config")
+        .expect("enabled config");
+
+        assert_eq!(config.endpoint, "https://connect.example/agent/");
+        assert_eq!(config.root_ca_pem, b"root certificate");
+        assert_eq!(config.state_path, state.join("heartbeat/state.json"));
+        assert!(!state.exists(), "parsing configuration must not create state");
+    }
+}
@@ -0,0 +1,585 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fs;
+use std::io::{self, Write as _};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Duration;
+
+use chrono::{DateTime, SecondsFormat, Utc};
+use reqwest::{Client, StatusCode, Url, header};
+use rustls::RootCertStore;
+use rustls::pki_types::{CertificateDer, pem::PemObject as _};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+use zeroize::Zeroizing;
+
+use super::config::HeartbeatConfig;
+use super::credential_store::{CredentialStoreError, DeviceCredential};
+use super::identity::IdentityError;
+use super::identity_store::StoreError;
+use super::registration::{CredentialValidationError, validate_stored_credential};
+
+const PROTOCOL_VERSION: &str = "v1";
+const AGENT_VERSION: &str = concat!("rustfs-agent/", env!("CARGO_PKG_VERSION"));
+const MAX_SEQUENCE: u64 = 9_007_199_254_740_991;
+const MAX_RESPONSE_BYTES: usize = 64 * 1024;
+#[cfg(unix)]
+const FILE_MODE: u32 = 0o600;
+static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+pub struct CoarseNodeSummary {
+    total: u16,
+    healthy: u16,
+    degraded: u16,
+}
+
+impl CoarseNodeSummary {
+    pub fn new(total: u16, healthy: u16, degraded: u16) -> Result<Self, HeartbeatError> {
+        let summary = Self {
+            total,
+            healthy,
+            degraded,
+        };
+        if !summary.is_valid() {
+            return Err(HeartbeatError::NodeSummary);
+        }
+        Ok(summary)
+    }
+
+    fn is_valid(&self) -> bool {
+        self.total != 0
+            && self.total <= 4096
+            && self.healthy <= 4096
+            && self.degraded <= 4096
+            && self.healthy.saturating_add(self.degraded) <= self.total
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum HeartbeatStatus {
+    Starting,
+    Online { server_time: String },
+    BackingOff { delay: Duration },
+    AuthenticationStopped { status: u16, reason: Option<String> },
+    Failed { reason: String },
+    Stopped,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+pub(crate) struct PendingHeartbeat {
+    protocol_version: String,
+    request_id: String,
+    agent_version: String,
+    capabilities: [String; 1],
+    sequence: u64,
+    client_time: String,
+    coarse_node_summary: CoarseNodeSummary,
+}
+
+impl PendingHeartbeat {
+    fn is_valid(&self) -> bool {
+        self.protocol_version == PROTOCOL_VERSION
+            && self.agent_version == AGENT_VERSION
+            && self.capabilities[0] == "heartbeat"
+            && self.sequence <= MAX_SEQUENCE
+            && self.coarse_node_summary.is_valid()
+            && is_exact_utc_seconds(&self.client_time)
+            && Uuid::parse_str(&self.request_id)
+                .is_ok_and(|request_id| request_id.get_version_num() == 4 && request_id.to_string() == self.request_id)
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct HeartbeatResponse {
+    server_time: String,
+    accepted_version: String,
+    #[serde(default)]
+    capability_hints: Vec<String>,
+}
+
+pub(crate) enum Delivery {
+    Accepted { server_time: String },
+    Retry { retry_after: Option<Duration> },
+    AuthenticationStopped { status: u16, reason: Option<String> },
+    Rejected { status: u16, reason: Option<String> },
+}
+
+pub(crate) struct HeartbeatSender {
+    endpoint: Url,
+    root_store: RootCertStore,
+    roots: Vec<CertificateDer<'static>>,
+    config: HeartbeatConfig,
+}
+
+impl HeartbeatSender {
+    pub(crate) fn new(config: HeartbeatConfig) -> Result<Self, HeartbeatError> {
+        let mut endpoint = Url::parse(&config.endpoint).map_err(|_| HeartbeatError::Endpoint)?;
+        if endpoint.scheme() != "https"
+            || endpoint.cannot_be_a_base()
+            || !endpoint.username().is_empty()
+            || endpoint.password().is_some()
+            || endpoint.query().is_some()
+            || endpoint.fragment().is_some()
+        {
+            return Err(HeartbeatError::Endpoint);
+        }
+        if !endpoint.path().ends_with('/') {
+            endpoint.set_path(&format!("{}/", endpoint.path()));
+        }
+        let roots = CertificateDer::pem_slice_iter(&config.root_ca_pem)
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|_| HeartbeatError::RootCertificate)?;
+        if roots.is_empty() {
+            return Err(HeartbeatError::RootCertificate);
+        }
+        let mut root_store = RootCertStore::empty();
+        let (accepted, rejected) = root_store.add_parsable_certificates(roots.clone());
+        if accepted != roots.len() || rejected != 0 {
+            return Err(HeartbeatError::RootCertificate);
+        }
+        let schedule = config.schedule;
+        if schedule.cadence.is_zero()
+            || schedule.timeout.is_zero()
+            || schedule.timeout > Duration::from_secs(5)
+            || schedule.initial_backoff.is_zero()
+            || schedule.max_backoff < schedule.initial_backoff
+            || schedule.max_backoff > Duration::from_secs(5 * 60)
+            || schedule.jitter > schedule.cadence
+        {
+            return Err(HeartbeatError::Schedule);
+        }
+        Ok(Self {
+            endpoint,
+            root_store,
+            roots,
+            config,
+        })
+    }
+
+    pub(crate) async fn send(&self, heartbeat: &PendingHeartbeat) -> Result<Delivery, HeartbeatError> {
+        let (cluster_uid, client) = {
+            let _lock = self.config.credential_store.lock().await?;
+            let credential = self.config.credential_store.load()?.ok_or(HeartbeatError::NotRegistered)?;
+            let identity = self.config.identity_store.load()?.ok_or(HeartbeatError::IdentityMissing)?;
+            validate_stored_credential(&credential, &identity, &self.root_store, &self.roots)?;
+            let now = Utc::now().timestamp();
+            if now < credential.not_before_unix || now >= credential.not_after_unix {
+                return Err(HeartbeatError::CredentialExpired);
+            }
+            let cluster_uid = cluster_uid(&credential)?.to_owned();
+            let client = self.client(&credential, &identity.to_pkcs8_pem()?)?;
+            (cluster_uid, client)
+        };
+        let url = self.endpoint.join(&format!("clusters/{cluster_uid}/heartbeats"))?;
+        let response = match client.post(url).json(heartbeat).send().await {
+            Ok(response) => response,
+            Err(error) if error.is_timeout() || error.is_connect() || error.is_request() => {
+                return Ok(Delivery::Retry { retry_after: None });
+            }
+            Err(error) => return Err(error.into()),
+        };
+        let status = response.status();
+        if status == StatusCode::TOO_MANY_REQUESTS {
+            return Ok(Delivery::Retry {
+                retry_after: retry_after(response.headers(), Utc::now(), self.config.schedule.max_backoff),
+            });
+        }
+        if status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() {
+            return Ok(Delivery::Retry { retry_after: None });
+        }
+        if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) {
+            return Ok(Delivery::AuthenticationStopped {
+                status: status.as_u16(),
+                reason: response_reason(response).await,
+            });
+        }
+        if status != StatusCode::OK {
+            return Ok(Delivery::Rejected {
+                status: status.as_u16(),
+                reason: response_reason(response).await,
+            });
+        }
+        let accepted: HeartbeatResponse =
+            serde_json::from_slice(&bounded_body(response).await?).map_err(|_| HeartbeatError::Response)?;
+        if accepted.accepted_version != PROTOCOL_VERSION
+            || accepted.capability_hints.len() > 32
+            || accepted.capability_hints.iter().any(|hint| hint.len() > 32)
+            || !is_exact_utc_seconds(&accepted.server_time)
+        {
+            return Err(HeartbeatError::Response);
+        }
+        Ok(Delivery::Accepted {
+            server_time: accepted.server_time,
+        })
+    }
+
+    fn client(&self, credential: &DeviceCredential, key: &Zeroizing<String>) -> Result<Client, HeartbeatError> {
+        let mut pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + key.len() + 1));
+        pem.extend_from_slice(credential.certificate_chain.as_bytes());
+        pem.push(b'\n');
+        pem.extend_from_slice(key.as_bytes());
+        let identity = reqwest::Identity::from_pem(&pem).map_err(|_| HeartbeatError::IdentityCertificate)?;
+        let roots = self
+            .roots
+            .iter()
+            .map(|root| reqwest::Certificate::from_der(root.as_ref()))
+            .collect::<Result<Vec<_>, _>>()?;
+        Client::builder()
+            .https_only(true)
+            .redirect(reqwest::redirect::Policy::none())
+            .timeout(self.config.schedule.timeout)
+            .tls_certs_only(roots)
+            .identity(identity)
+            .build()
+            .map_err(Into::into)
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct HeartbeatStateStore {
+    path: PathBuf,
+}
+
+#[derive(Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+struct HeartbeatState {
+    next_sequence: u64,
+    pending: Option<PendingHeartbeat>,
+}
+
+impl HeartbeatStateStore {
+    pub(crate) fn new(path: PathBuf) -> Self {
+        Self { path }
+    }
+
+    pub(crate) fn try_runtime_lock(&self) -> Result<fs::File, HeartbeatError> {
+        let directory = parent(&self.path)?;
+        fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?;
+        let name = filename(&self.path)?;
+        let path = directory.join(format!(".{name}.lock"));
+        let mut options = fs::OpenOptions::new();
+        options.create(true).truncate(false).read(true).write(true);
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::OpenOptionsExt as _;
+            options.mode(FILE_MODE);
+        }
+        let lock = options.open(&path).map_err(|source| state_io(&path, source))?;
+        check_mode(&path)?;
+        lock.try_lock().map_err(|_| HeartbeatError::AlreadyRunning)?;
+        Ok(lock)
+    }
+
+    pub(crate) async fn prepare(
+        &self,
+        summary: CoarseNodeSummary,
+        now: DateTime<Utc>,
+    ) -> Result<PendingHeartbeat, HeartbeatError> {
+        let store = self.clone();
+        tokio::task::spawn_blocking(move || store.prepare_sync(summary, now))
+            .await
+            .map_err(|source| state_io(&self.path, io::Error::other(source)))?
+    }
+
+    pub(crate) async fn mark_accepted(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> {
+        let store = self.clone();
+        let accepted = accepted.clone();
+        tokio::task::spawn_blocking(move || store.mark_accepted_sync(&accepted))
+            .await
+            .map_err(|source| state_io(&self.path, io::Error::other(source)))?
+    }
+
+    fn prepare_sync(&self, summary: CoarseNodeSummary, now: DateTime<Utc>) -> Result<PendingHeartbeat, HeartbeatError> {
+        let mut state = self.read()?;
+        if let Some(pending) = state.pending {
+            return Ok(pending);
+        }
+        if state.next_sequence > MAX_SEQUENCE {
+            return Err(HeartbeatError::SequenceExhausted);
+        }
+        let pending = PendingHeartbeat {
+            protocol_version: PROTOCOL_VERSION.to_owned(),
+            request_id: Uuid::new_v4().to_string(),
+            agent_version: AGENT_VERSION.to_owned(),
+            capabilities: ["heartbeat".to_owned()],
+            sequence: state.next_sequence,
+            client_time: now.to_rfc3339_opts(SecondsFormat::Secs, true),
+            coarse_node_summary: summary,
+        };
+        state.pending = Some(pending.clone());
+        self.write(&state)?;
+        Ok(pending)
+    }
+
+    fn mark_accepted_sync(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> {
+        let mut state = self.read()?;
+        if state.pending.as_ref() != Some(accepted) {
+            return Err(HeartbeatError::StateConflict);
+        }
+        state.next_sequence = accepted.sequence.checked_add(1).ok_or(HeartbeatError::SequenceExhausted)?;
+        state.pending = None;
+        self.write(&state)
+    }
+
+    fn read(&self) -> Result<HeartbeatState, HeartbeatError> {
+        let bytes = match fs::read(&self.path) {
+            Ok(bytes) => bytes,
+            Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(HeartbeatState::default()),
+            Err(source) => return Err(state_io(&self.path, source)),
+        };
+        check_mode(&self.path)?;
+        let state: HeartbeatState = serde_json::from_slice(&bytes).map_err(|source| HeartbeatError::StateInvalid {
+            path: self.path.clone(),
+            source,
+        })?;
+        if state.next_sequence > MAX_SEQUENCE + 1
+            || state
+                .pending
+                .as_ref()
+                .is_some_and(|pending| pending.sequence != state.next_sequence || !pending.is_valid())
+        {
+            return Err(HeartbeatError::StateCorrupt { path: self.path.clone() });
+        }
+        Ok(state)
+    }
+
+    fn write(&self, state: &HeartbeatState) -> Result<(), HeartbeatError> {
+        let bytes = serde_json::to_vec(state).map_err(|source| HeartbeatError::StateInvalid {
+            path: self.path.clone(),
+            source,
+        })?;
+        let directory = parent(&self.path)?;
+        fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?;
+        let temp = stage(directory, &self.path, &bytes)?;
+        let result = fs::rename(&temp, &self.path)
+            .map_err(|source| state_io(&self.path, source))
+            .and_then(|()| fsync_dir(directory).map_err(|source| state_io(directory, source)));
+        if result.is_err() {
+            let _ = fs::remove_file(temp);
+        }
+        result
+    }
+}
+
+fn cluster_uid(credential: &DeviceCredential) -> Result<&str, HeartbeatError> {
+    let mut parts = credential.name.split('/');
+    let valid = parts.next() == Some("organizations");
+    let organization_uid = parts.next();
+    let valid = valid && parts.next() == Some("clusters");
+    let cluster_uid = parts.next();
+    let valid = valid && parts.next() == Some("clusterDevices");
+    let device_uid = parts.next();
+    if !valid
+        || organization_uid.is_none_or(str::is_empty)
+        || cluster_uid.is_none_or(str::is_empty)
+        || device_uid != Some(credential.uid.as_str())
+        || parts.next().is_some()
+    {
+        return Err(HeartbeatError::CredentialName);
+    }
+    cluster_uid.ok_or(HeartbeatError::CredentialName)
+}
+
+fn retry_after(headers: &header::HeaderMap, now: DateTime<Utc>, maximum: Duration) -> Option<Duration> {
+    let value = headers.get(header::RETRY_AFTER)?.to_str().ok()?;
+    let delay = value.parse::<u64>().ok().map(Duration::from_secs).or_else(|| {
+        DateTime::parse_from_rfc2822(value)
+            .ok()
+            .and_then(|at| (at.with_timezone(&Utc) - now).to_std().ok())
+    })?;
+    Some(delay.min(maximum))
+}
+
+fn is_exact_utc_seconds(value: &str) -> bool {
+    DateTime::parse_from_rfc3339(value).is_ok_and(|time| {
+        time.offset().local_minus_utc() == 0
+            && value.ends_with('Z')
+            && time.with_timezone(&Utc).to_rfc3339_opts(SecondsFormat::Secs, true) == value
+    })
+}
+
+async fn response_reason(response: reqwest::Response) -> Option<String> {
+    #[derive(Deserialize)]
+    struct Envelope {
+        #[serde(default)]
+        details: Vec<Detail>,
+    }
+    #[derive(Deserialize)]
+    struct Detail {
+        #[serde(default)]
+        reason: String,
+    }
+
+    serde_json::from_slice::<Envelope>(&bounded_body(response).await.ok()?)
+        .ok()?
+        .details
+        .into_iter()
+        .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason))
+}
+
+async fn bounded_body(mut response: reqwest::Response) -> Result<Vec<u8>, HeartbeatError> {
+    let mut body = Vec::new();
+    while let Some(chunk) = response.chunk().await? {
+        if body.len().saturating_add(chunk.len()) > MAX_RESPONSE_BYTES {
+            return Err(HeartbeatError::ResponseTooLarge);
+        }
+        body.extend_from_slice(&chunk);
+    }
+    Ok(body)
+}
+
+fn parent(path: &Path) -> Result<&Path, HeartbeatError> {
+    path.parent()
+        .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state path has no parent")))
+}
+
+fn filename(path: &Path) -> Result<&str, HeartbeatError> {
+    path.file_name()
+        .and_then(|name| name.to_str())
+        .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state filename is invalid")))
+}
+
+fn stage(directory: &Path, destination: &Path, bytes: &[u8]) -> Result<PathBuf, HeartbeatError> {
+    let name = filename(destination)?;
+    loop {
+        let path = directory.join(format!(
+            ".{name}.{}.{}.tmp",
+            std::process::id(),
+            STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed)
+        ));
+        let mut options = fs::OpenOptions::new();
+        options.write(true).create_new(true);
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::OpenOptionsExt as _;
+            options.mode(FILE_MODE);
+        }
+        let mut file = match options.open(&path) {
+            Ok(file) => file,
+            Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue,
+            Err(source) => return Err(state_io(&path, source)),
+        };
+        if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) {
+            let _ = fs::remove_file(&path);
+            return Err(state_io(&path, source));
+        }
+        return Ok(path);
+    }
+}
+
+fn state_io(path: &Path, source: io::Error) -> HeartbeatError {
+    HeartbeatError::StateIo {
+        path: path.to_path_buf(),
+        source,
+    }
+}
+
+#[cfg(unix)]
+fn check_mode(path: &Path) -> Result<(), HeartbeatError> {
+    use std::os::unix::fs::PermissionsExt as _;
+
+    let mode = fs::metadata(path)
+        .map_err(|source| state_io(path, source))?
+        .permissions()
+        .mode()
+        & 0o7777;
+    if mode != FILE_MODE {
+        return Err(HeartbeatError::StatePermissions {
+            path: path.to_path_buf(),
+            mode,
+            expected: FILE_MODE,
+        });
+    }
+    Ok(())
+}
+
+#[cfg(not(unix))]
+fn check_mode(_path: &Path) -> Result<(), HeartbeatError> {
+    Ok(())
+}
+
+fn fsync_dir(directory: &Path) -> io::Result<()> {
+    #[cfg(unix)]
+    fs::File::open(directory)?.sync_all()?;
+    #[cfg(not(unix))]
+    let _ = directory;
+    Ok(())
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum HeartbeatError {
+    #[error("Connect heartbeat endpoint must be an HTTPS base URL without credentials, query, or fragment")]
+    Endpoint,
+    #[error("Connect heartbeat root CA configuration is invalid")]
+    RootCertificate,
+    #[error("Connect heartbeat schedule is invalid")]
+    Schedule,
+    #[error("RustFS is not registered with Connect")]
+    NotRegistered,
+    #[error("the Connect device private key is missing")]
+    IdentityMissing,
+    #[error("the stored Connect certificate and device private key cannot form a TLS identity")]
+    IdentityCertificate,
+    #[error("the stored Connect credential name is invalid")]
+    CredentialName,
+    #[error("the stored Connect device certificate is not currently valid")]
+    CredentialExpired,
+    #[error("the Connect heartbeat node summary is outside protocol bounds")]
+    NodeSummary,
+    #[error("the Connect heartbeat sequence is exhausted")]
+    SequenceExhausted,
+    #[error("a Connect heartbeat runtime already owns this state")]
+    AlreadyRunning,
+    #[error("the persisted Connect heartbeat changed while delivery was in flight")]
+    StateConflict,
+    #[error("Connect heartbeat state I/O failed at {path}: {source}")]
+    StateIo {
+        path: PathBuf,
+        #[source]
+        source: io::Error,
+    },
+    #[error("Connect heartbeat state at {path} is invalid: {source}")]
+    StateInvalid {
+        path: PathBuf,
+        #[source]
+        source: serde_json::Error,
+    },
+    #[error("Connect heartbeat state at {path} violates the protocol invariants")]
+    StateCorrupt { path: PathBuf },
+    #[cfg(unix)]
+    #[error("Connect heartbeat state at {path} has mode {mode:o}, expected {expected:o}")]
+    StatePermissions { path: PathBuf, mode: u32, expected: u32 },
+    #[error("Connect heartbeat response exceeded 64 KiB")]
+    ResponseTooLarge,
+    #[error("Connect returned an invalid heartbeat response")]
+    Response,
+    #[error(transparent)]
+    Url(#[from] url::ParseError),
+    #[error(transparent)]
+    Transport(#[from] reqwest::Error),
+    #[error(transparent)]
+    Identity(#[from] IdentityError),
+    #[error(transparent)]
+    IdentityStore(#[from] StoreError),
+    #[error(transparent)]
+    CredentialStore(#[from] CredentialStoreError),
+    #[error(transparent)]
+    CredentialValidation(#[from] CredentialValidationError),
+}
@@ -21,20 +21,26 @@
 //! canonical transcript frozen by
 //! `protocol/agent/v1/registration-proof.md`.
 //!
-//! Nothing here contacts the network or starts a task. A deployment that has
-//! not been enrolled into a Connect control plane never calls into it, so an
-//! unconfigured server generates no key and holds no identity.
+//! Enrolled deployments may start the optional outbound heartbeat runtime.
+//! An unconfigured server starts no Connect task, generates no key, and holds
+//! no Connect identity.

 pub mod client;
+pub mod config;
 pub mod credential_store;
+pub mod heartbeat;
 pub mod identity;
 pub mod identity_store;
 pub mod offline;
 pub mod registration;
+pub mod runtime;

 pub use client::{ClientError, ConnectClient, ConnectConfig};
+pub use config::{HeartbeatConfig, HeartbeatConfigError, HeartbeatSchedule};
 pub use credential_store::{CredentialStore, DeviceCredential};
+pub use heartbeat::{CoarseNodeSummary, HeartbeatError, HeartbeatStatus};
 pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript};
 pub use identity_store::{IdentityStore, StoreError};
 pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge};
 pub use registration::{RegistrationToken, TokenError};
+pub use runtime::{HeartbeatRuntime, spawn_heartbeat_runtime};
@@ -0,0 +1,156 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::time::Duration;
+
+use chrono::Utc;
+use rand::RngExt as _;
+use tokio::sync::watch;
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+use super::config::HeartbeatConfig;
+use super::heartbeat::{CoarseNodeSummary, Delivery, HeartbeatError, HeartbeatSender, HeartbeatStateStore, HeartbeatStatus};
+
+pub struct HeartbeatRuntime {
+    shutdown: CancellationToken,
+    status: watch::Receiver<HeartbeatStatus>,
+    task: Option<JoinHandle<()>>,
+}
+
+impl HeartbeatRuntime {
+    pub fn status(&self) -> watch::Receiver<HeartbeatStatus> {
+        self.status.clone()
+    }
+
+    pub async fn shutdown(mut self) {
+        self.shutdown.cancel();
+        if let Some(task) = self.task.take() {
+            let _ = task.await;
+        }
+    }
+}
+
+impl Drop for HeartbeatRuntime {
+    fn drop(&mut self) {
+        self.shutdown.cancel();
+    }
+}
+
+pub fn spawn_heartbeat_runtime<F>(
+    config: Option<HeartbeatConfig>,
+    parent_shutdown: &CancellationToken,
+    sample: F,
+) -> Result<Option<HeartbeatRuntime>, HeartbeatError>
+where
+    F: Fn() -> CoarseNodeSummary + Send + Sync + 'static,
+{
+    let Some(config) = config else {
+        return Ok(None);
+    };
+    let sender = HeartbeatSender::new(config.clone())?;
+    let store = HeartbeatStateStore::new(config.state_path.clone());
+    let lock = store.try_runtime_lock()?;
+    let schedule = config.schedule;
+    let shutdown = parent_shutdown.child_token();
+    let task_shutdown = shutdown.clone();
+    let (status_tx, status_rx) = watch::channel(HeartbeatStatus::Starting);
+    let task = tokio::spawn(async move {
+        let _lock = lock;
+        let mut backoff = schedule.initial_backoff;
+        loop {
+            if task_shutdown.is_cancelled() {
+                break;
+            }
+            let pending = match store.prepare(sample(), Utc::now()).await {
+                Ok(pending) => pending,
+                Err(error) => return failed(&status_tx, error),
+            };
+            let delivery = match cancellable(&task_shutdown, sender.send(&pending)).await {
+                Some(Ok(delivery)) => delivery,
+                Some(Err(error)) => return failed(&status_tx, error),
+                None => break,
+            };
+            let delay = match delivery {
+                Delivery::Accepted { server_time } => {
+                    if let Err(error) = store.mark_accepted(&pending).await {
+                        return failed(&status_tx, error);
+                    }
+                    backoff = schedule.initial_backoff;
+                    let _ = status_tx.send(HeartbeatStatus::Online { server_time });
+                    schedule.cadence.saturating_add(jitter(schedule.jitter))
+                }
+                Delivery::Retry { retry_after } => {
+                    let delay = retry_after
+                        .unwrap_or(backoff)
+                        .clamp(schedule.initial_backoff, schedule.max_backoff);
+                    backoff = backoff.saturating_mul(2).min(schedule.max_backoff);
+                    let _ = status_tx.send(HeartbeatStatus::BackingOff { delay });
+                    delay
+                }
+                Delivery::AuthenticationStopped { status, reason } => {
+                    let _ = status_tx.send(HeartbeatStatus::AuthenticationStopped { status, reason });
+                    return;
+                }
+                Delivery::Rejected { status, reason } => {
+                    let suffix = reason.map_or_else(String::new, |reason| format!("; reason={reason}"));
+                    let _ = status_tx.send(HeartbeatStatus::Failed {
+                        reason: format!("Connect rejected heartbeat with HTTP {status}{suffix}"),
+                    });
+                    return;
+                }
+            };
+            if sleep_or_cancel(&task_shutdown, delay).await {
+                break;
+            }
+        }
+        let _ = status_tx.send(HeartbeatStatus::Stopped);
+    });
+    Ok(Some(HeartbeatRuntime {
+        shutdown,
+        status: status_rx,
+        task: Some(task),
+    }))
+}
+
+fn failed(status: &watch::Sender<HeartbeatStatus>, error: HeartbeatError) {
+    let _ = status.send(HeartbeatStatus::Failed {
+        reason: error.to_string(),
+    });
+}
+
+fn jitter(maximum: Duration) -> Duration {
+    if maximum.is_zero() {
+        Duration::ZERO
+    } else {
+        maximum.mul_f64(rand::rng().random_range(0.0..=1.0))
+    }
+}
+
+async fn cancellable<T>(shutdown: &CancellationToken, future: impl Future<Output = T>) -> Option<T> {
+    tokio::select! {
+        biased;
+        () = shutdown.cancelled() => None,
+        value = future => Some(value),
+    }
+}
+
+async fn sleep_or_cancel(shutdown: &CancellationToken, delay: Duration) -> bool {
+    tokio::select! {
+        biased;
+        () = shutdown.cancelled() => true,
+        () = tokio::time::sleep(delay) => false,
+    }
+}
@@ -20,6 +20,7 @@ use crate::storage_api::server::readiness::contract::admin::StorageAdminApi;
 use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure};
 #[cfg(test)]
 use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints};
+use crate::storage_api::startup::shutdown::mark_get_metadata_read_version_coalescing_service_ready;
 use bytes::Bytes;
 use http::HeaderValue;
 use http::{Request as HttpRequest, Response, StatusCode};
@@ -212,6 +213,9 @@ where
            if readiness_gate_blocks_path(path, &readiness) {
                return Ok(service_not_ready_response(readiness.current_stage()));
            }
+            if !is_probe_path(path) && readiness.is_ready() {
+                mark_get_metadata_read_version_coalescing_service_ready();
+            }
            let resp = inner.call(req).await?;
            // System is ready, forward to the actual S3/RPC handlers
            // Transparently converts any response body into a BoxBody, and then Trace/Cors/Compression continues to work
@@ -232,6 +236,7 @@ pub async fn publish_ready_when_runtime_ready(
        collect_node_readiness,
        |dependency_readiness| {
            readiness.mark_stage(rustfs_common::SystemStage::FullReady);
+            mark_get_metadata_read_version_coalescing_service_ready();
            if let Some(state_manager) = state_manager {
                state_manager.update(ServiceState::Ready);
            }
@@ -128,6 +128,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
    } = lifecycle;
    let StartupServiceRuntime {
        optional_runtimes,
+        heartbeat,
        iam_bootstrap,
        enable_scanner,
    } = service_runtime;
@@ -162,6 +163,9 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
        shutdown_token,
    )
    .await;
+    if let Some(heartbeat) = heartbeat {
+        heartbeat.shutdown().await;
+    }
    if let Err(err) = event_notifier_reconciler.await {
        tracing::warn!(
            target: "rustfs::main::run",
@@ -16,6 +16,7 @@ use crate::site_replication_reconcile::spawn_site_replication_reconcile_task;
 use crate::storage_api::startup::services::{ECStore, EndpointServerPools, ServerContextSlot};
 use crate::{
    config::Config,
+    connect::{CoarseNodeSummary, HeartbeatConfig, HeartbeatRuntime, spawn_heartbeat_runtime},
    init::{init_buffer_profile_system, init_kms_system},
    server::ServiceStateManager,
    startup_audit::init_audit_runtime,
@@ -35,6 +36,7 @@ use tokio_util::sync::CancellationToken;

 pub(crate) struct StartupServiceRuntime {
    pub(crate) optional_runtimes: OptionalRuntimeServices,
+    pub(crate) heartbeat: Option<HeartbeatRuntime>,
    pub(crate) iam_bootstrap: IamBootstrapDisposition,
    pub(crate) enable_scanner: bool,
 }
@@ -73,6 +75,8 @@ pub(crate) async fn init_startup_runtime_services(
    init_kms_system(config).await?;

    let optional_runtimes = init_optional_runtime_services().await?;
+    let heartbeat_config = HeartbeatConfig::from_env().map_err(std::io::Error::other)?;
+    let heartbeat_nodes = heartbeat_config.as_ref().map(|_| endpoint_pools.get_nodes().len());

    init_buffer_profile_system(config);
    init_deadlock_detector_runtime();
@@ -92,10 +96,27 @@ pub(crate) async fn init_startup_runtime_services(
    init_notification_runtime(endpoint_pools, buckets).await?;
    let enable_scanner = init_background_service_runtime(store.clone()).await?;
    init_observability_runtime(store.clone(), ctx.clone()).await;
+    let heartbeat = start_heartbeat_runtime(heartbeat_config, heartbeat_nodes, &ctx)?;

    Ok(StartupServiceRuntime {
        optional_runtimes,
+        heartbeat,
        iam_bootstrap,
        enable_scanner,
    })
 }
+
+fn start_heartbeat_runtime(
+    config: Option<HeartbeatConfig>,
+    node_count: Option<usize>,
+    shutdown: &CancellationToken,
+) -> Result<Option<HeartbeatRuntime>> {
+    let Some(config) = config else {
+        return Ok(None);
+    };
+    let summary = u16::try_from(node_count.unwrap_or_default())
+        .ok()
+        .and_then(|total| CoarseNodeSummary::new(total, 0, 0).ok())
+        .ok_or_else(|| std::io::Error::other("Connect heartbeat node count is outside protocol bounds"))?;
+    spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(std::io::Error::other)
+}
@@ -23,10 +23,12 @@ use bytes::Bytes;
 use rustfs_filemeta::FileInfo;
 use rustfs_io_metrics::internode_metrics::{
    INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST,
-    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL,
-    INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE,
-    INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE,
-    INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION,
+    INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ,
+    INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE,
+    INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_STAGE_READ_VERSION_DISK_READ,
+    INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE,
+    INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
 };
 use rustfs_protos::proto_gen::node_service::*;
 use serde::de::DeserializeOwned;
@@ -242,24 +244,42 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
    }
 }

+fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
+    if let Some(started_at) = started_at {
+        global_internode_metrics().record_stage_duration_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            stage,
+            started_at.elapsed(),
+        );
+    }
+}
+
 fn encode_batch_read_version_response_payloads(
    batch_read_version_resps: &[BatchReadVersionResp],
    request_decoded_from_msgpack: bool,
 ) -> std::result::Result<(Vec<String>, Vec<Bytes>), DiskError> {
+    let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
    let mut batch_read_version_resps_json = Vec::with_capacity(batch_read_version_resps.len());
-    let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len());
-
+    let json_encode_started = internode_stage_timer(attribution_enabled);
    for batch_read_version_resp in batch_read_version_resps {
        batch_read_version_resps_json.push(
            compat_response_json(batch_read_version_resp, request_decoded_from_msgpack)
                .map_err(|err| DiskError::other(format!("encode BatchReadVersionResp json failed: {err}")))?,
        );
+    }
+    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE, json_encode_started);
+
+    let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len());
+    let msgpack_encode_started = internode_stage_timer(attribution_enabled);
+    for batch_read_version_resp in batch_read_version_resps {
        batch_read_version_resps_bin.push(Bytes::from(encode_msgpack_with_capacity(
            batch_read_version_resp,
            "BatchReadVersionResp",
            FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT,
        )?));
    }
+    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, msgpack_encode_started);

    Ok((batch_read_version_resps_json, batch_read_version_resps_bin))
 }
@@ -485,15 +505,37 @@ impl NodeService {
        &self,
        request: Request<BatchReadVersionRequest>,
    ) -> Result<Response<BatchReadVersionResponse>, Status> {
+        let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
        let request = request.into_inner();
+        if attribution_enabled {
+            let metrics = global_internode_metrics();
+            metrics.record_incoming_request_for_operation_and_backend(
+                INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+            );
+            metrics.record_recv_bytes_for_operation_and_backend(
+                INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+                INTERNODE_TRANSPORT_BACKEND_GRPC,
+                request
+                    .disk
+                    .len()
+                    .saturating_add(request.batch_read_version_req.len())
+                    .saturating_add(request.batch_read_version_req_bin.len()),
+            );
+        }
        if let Some(disk) = self.find_disk(&request.disk).await {
+            let decode_started = internode_stage_timer(attribution_enabled);
            let decoded_batch_read_version_req: DecodedRpcPayload<BatchReadVersionReq> = match decode_msgpack_or_json_with_source(
                &request.batch_read_version_req_bin,
                &request.batch_read_version_req,
                "BatchReadVersionReq",
            ) {
-                Ok(batch_read_version_req) => batch_read_version_req,
+                Ok(batch_read_version_req) => {
+                    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started);
+                    batch_read_version_req
+                }
                Err(err) => {
+                    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started);
                    return Ok(Response::new(BatchReadVersionResponse {
                        success: false,
                        batch_read_version_resps: Vec::new(),
@@ -514,8 +556,10 @@ impl NodeService {
                }));
            }

+            let disk_read_started = internode_stage_timer(attribution_enabled);
            match disk.batch_read_version(batch_read_version_req).await {
                Ok(batch_read_version_resps) => {
+                    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started);
                    let (batch_read_version_resps, batch_read_version_resps_bin) =
                        match encode_batch_read_version_response_payloads(&batch_read_version_resps, request_decoded_from_msgpack)
                        {
@@ -537,12 +581,15 @@ impl NodeService {
                        error: None,
                    }))
                }
-                Err(err) => Ok(Response::new(BatchReadVersionResponse {
-                    success: false,
-                    batch_read_version_resps: Vec::new(),
-                    batch_read_version_resps_bin: Vec::new(),
-                    error: Some(err.into()),
-                })),
+                Err(err) => {
+                    record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started);
+                    Ok(Response::new(BatchReadVersionResponse {
+                        success: false,
+                        batch_read_version_resps: Vec::new(),
+                        batch_read_version_resps_bin: Vec::new(),
+                        error: Some(err.into()),
+                    }))
+                }
            }
        } else {
            Ok(Response::new(BatchReadVersionResponse {
@@ -722,9 +769,9 @@ impl NodeService {
        &self,
        request: Request<ReadVersionRequest>,
    ) -> Result<Response<ReadVersionResponse>, Status> {
-        let request = request.into_inner();
        let metrics = global_internode_metrics();
        let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
+        let request = request.into_inner();
        if read_version_attribution_enabled {
            metrics.record_incoming_request_for_operation_and_backend(
                INTERNODE_OPERATION_GRPC_READ_VERSION,
@@ -1635,6 +1682,7 @@ mod tests {
        encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack,
        encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads,
    };
+    use crate::storage::DiskError;
    use crate::storage::rpc::node_service::make_server;
    use crate::storage::storage_api::ReadMultipleResp;
    use crate::storage::storage_api::RenameDataResp;
@@ -2028,7 +2076,8 @@ mod tests {
            path: "object-a".to_string(),
            version_id: "version-a".to_string(),
            success: false,
-            error: "file version not found".to_string(),
+            error: DiskError::FileVersionNotFound.to_string(),
+            error_code: DiskError::FileVersionNotFound.to_u32(),
            ..Default::default()
        }];

@@ -2044,8 +2093,43 @@ mod tests {
            .expect("msgpack batch read version response should decode");

        assert_eq!(json_decoded.index, responses[0].index);
+        assert_eq!(json_decoded.error_code, responses[0].error_code);
        assert_eq!(msgpack_decoded.path, responses[0].path);
        assert_eq!(msgpack_decoded.error, responses[0].error);
+        assert_eq!(msgpack_decoded.error_code, responses[0].error_code);
+    }
+
+    #[test]
+    fn batch_read_version_response_decode_accepts_legacy_payload_without_error_code() {
+        #[derive(Serialize)]
+        struct LegacyBatchReadVersionResp {
+            index: usize,
+            path: String,
+            version_id: String,
+            success: bool,
+            file_info: FileInfo,
+            error: String,
+        }
+
+        let legacy = LegacyBatchReadVersionResp {
+            index: 2,
+            path: "object-legacy".to_string(),
+            version_id: "version-legacy".to_string(),
+            success: false,
+            file_info: FileInfo::default(),
+            error: "legacy error".to_string(),
+        };
+        let legacy_json = serde_json::to_string(&legacy).expect("legacy json should encode");
+        let legacy_msgpack = encode_msgpack(&legacy, "LegacyBatchReadVersionResp").expect("legacy msgpack should encode");
+
+        let json_decoded: BatchReadVersionResp =
+            decode_msgpack_or_json(&[], &legacy_json, "BatchReadVersionResp").expect("legacy json should decode");
+        let msgpack_decoded: BatchReadVersionResp =
+            decode_msgpack_or_json(&legacy_msgpack, "", "BatchReadVersionResp").expect("legacy msgpack should decode");
+
+        assert_eq!(json_decoded.error_code, 0);
+        assert_eq!(msgpack_decoded.error_code, 0);
+        assert_eq!(msgpack_decoded.error, legacy.error);
    }

    #[test]
@@ -1100,6 +1100,10 @@ pub(crate) fn shutdown_background_monitors() {
    rustfs_ecstore::shutdown_background_monitors();
 }

+pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
+    rustfs_ecstore::mark_get_metadata_read_version_coalescing_service_ready();
+}
+
 pub(crate) fn set_global_rustfs_port(value: u16) {
    ecstore_global::set_global_rustfs_port(value);
 }
@@ -284,7 +284,8 @@ pub(crate) mod startup {

    pub(crate) mod shutdown {
        pub(crate) use crate::storage::storage_api::{
-            shutdown_background_monitors, shutdown_background_services, store_compression_total_in_backend,
+            mark_get_metadata_read_version_coalescing_service_ready, shutdown_background_monitors, shutdown_background_services,
+            store_compression_total_in_backend,
        };
    }

@@ -0,0 +1,687 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::VecDeque;
+use std::fs;
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use bytes::Bytes;
+use http_body_util::{BodyExt as _, Full};
+use hyper::service::service_fn;
+use hyper::{Request, Response, StatusCode};
+use hyper_util::rt::TokioIo;
+use rcgen::{
+    BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair,
+    KeyUsagePurpose, SanType,
+};
+use rustfs::connect::{
+    CoarseNodeSummary, CredentialStore, DeviceCredential, HeartbeatConfig, HeartbeatSchedule, HeartbeatStatus, IdentityStore,
+    spawn_heartbeat_runtime,
+};
+use rustls::RootCertStore;
+use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
+use rustls::server::WebPkiClientVerifier;
+use serde_json::{Value, json};
+use time::OffsetDateTime;
+use tokio::net::TcpListener;
+use tokio::sync::watch;
+use tokio_rustls::TlsAcceptor;
+use tokio_util::sync::CancellationToken;
+
+const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70";
+const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81";
+const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92";
+
+struct TestPki {
+    root_params: CertificateParams,
+    root_key: KeyPair,
+    root_der: CertificateDer<'static>,
+    root_pem: String,
+    server_der: CertificateDer<'static>,
+    server_key: PrivatePkcs8KeyDer<'static>,
+}
+
+impl TestPki {
+    fn new() -> Self {
+        let now = OffsetDateTime::now_utc();
+        let root_key = KeyPair::generate().expect("generate root key");
+        let mut root_params = CertificateParams::default();
+        root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+        root_params.not_before = now - time::Duration::days(30);
+        root_params.not_after = now + time::Duration::days(30);
+        root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature];
+        let root = root_params.self_signed(&root_key).expect("sign root");
+
+        let server_key = KeyPair::generate().expect("generate server key");
+        let mut server_params = CertificateParams::default();
+        server_params.not_before = now - time::Duration::hours(1);
+        server_params.not_after = now + time::Duration::days(2);
+        server_params
+            .subject_alt_names
+            .push(SanType::DnsName("localhost".try_into().expect("valid DNS name")));
+        server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
+        let server = server_params
+            .signed_by(&server_key, &Issuer::from_params(&root_params, &root_key))
+            .expect("sign server certificate");
+        Self {
+            root_params,
+            root_key,
+            root_der: root.der().clone(),
+            root_pem: root.pem(),
+            server_der: server.der().clone(),
+            server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()),
+        }
+    }
+
+    fn server_config(&self) -> rustls::ServerConfig {
+        let mut roots = RootCertStore::empty();
+        roots.add(self.root_der.clone()).expect("add client root");
+        let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
+            .build()
+            .expect("client verifier");
+        rustls::ServerConfig::builder()
+            .with_client_cert_verifier(verifier)
+            .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key()))
+            .expect("server TLS")
+    }
+
+    fn stores(&self, temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) {
+        let now = OffsetDateTime::now_utc();
+        self.stores_with_certificate(temp, now - time::Duration::hours(1), now + time::Duration::hours(23), true)
+    }
+
+    fn stores_with_certificate(
+        &self,
+        temp: &tempfile::TempDir,
+        not_before: OffsetDateTime,
+        not_after: OffsetDateTime,
+        bind_identity: bool,
+    ) -> (IdentityStore, CredentialStore) {
+        let identity_store = IdentityStore::new(temp.path().join("identity"));
+        let identity = identity_store.load_or_create().expect("create identity");
+        let private_key = PrivatePkcs8KeyDer::from(identity.to_pkcs8_der().expect("serialize key").to_vec());
+        let device_key = if bind_identity {
+            KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("device key")
+        } else {
+            KeyPair::generate().expect("mismatched device key")
+        };
+        let mut params = CertificateParams::default();
+        params.not_before = not_before;
+        params.not_after = not_after;
+        params.serial_number = Some(vec![1; 16].into());
+        params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
+        params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
+        params.distinguished_name = DistinguishedName::new();
+        params.distinguished_name.push(DnType::CommonName, DEVICE_UID);
+        params.subject_alt_names.push(SanType::URI(
+            format!("urn:rustfs:connect:device:{DEVICE_UID}")
+                .try_into()
+                .expect("device URI"),
+        ));
+        let certificate = params
+            .signed_by(&device_key, &Issuer::from_params(&self.root_params, &self.root_key))
+            .expect("device certificate");
+        let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}");
+        let credential = DeviceCredential {
+            name: format!("{cluster}/clusterDevices/{DEVICE_UID}"),
+            uid: DEVICE_UID.to_owned(),
+            protocol_version: "v1".to_owned(),
+            key_id: format!("x509-{}", "01".repeat(16)),
+            certificate_serial: "01".repeat(16),
+            certificate: certificate.pem(),
+            certificate_chain: certificate.pem(),
+            not_before_unix: not_before.unix_timestamp(),
+            not_after_unix: not_after.unix_timestamp(),
+        };
+        let directory = temp.path().join("credential");
+        fs::create_dir_all(&directory).expect("credential directory");
+        let path = directory.join("device.crt.json");
+        fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("write credential");
+        private_mode(&path);
+        (identity_store, CredentialStore::new(directory))
+    }
+}
+
+#[derive(Clone)]
+struct Reply {
+    status: StatusCode,
+    body: Value,
+    retry_after: Option<&'static str>,
+    delay: Duration,
+}
+
+impl Reply {
+    fn ok(time: &str) -> Self {
+        Self {
+            status: StatusCode::OK,
+            body: json!({
+                "serverTime": time,
+                "acceptedVersion": "v1",
+                "capabilityHints": [],
+                "futureField": true
+            }),
+            retry_after: None,
+            delay: Duration::ZERO,
+        }
+    }
+
+    fn error(status: StatusCode) -> Self {
+        Self {
+            status,
+            body: json!({"details": []}),
+            retry_after: None,
+            delay: Duration::ZERO,
+        }
+    }
+}
+
+struct TestServer {
+    endpoint: String,
+    seen: Arc<Mutex<Vec<Value>>>,
+    task: tokio::task::JoinHandle<()>,
+}
+
+impl Drop for TestServer {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
+async fn server(pki: &TestPki, replies: Vec<Reply>) -> TestServer {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind server");
+    let address = listener.local_addr().expect("server address");
+    let acceptor = TlsAcceptor::from(Arc::new(pki.server_config()));
+    let replies = Arc::new(Mutex::new(VecDeque::from(replies)));
+    let seen = Arc::new(Mutex::new(Vec::new()));
+    let captured = seen.clone();
+    let task = tokio::spawn(async move {
+        while let Ok((stream, _)) = listener.accept().await {
+            let acceptor = acceptor.clone();
+            let replies = replies.clone();
+            let seen = captured.clone();
+            tokio::spawn(async move {
+                let Ok(stream) = acceptor.accept(stream).await else { return };
+                let service = service_fn(move |request: Request<hyper::body::Incoming>| {
+                    let replies = replies.clone();
+                    let seen = seen.clone();
+                    async move {
+                        assert_eq!(request.uri().path(), format!("/agent/clusters/{CLUSTER_UID}/heartbeats"));
+                        let body = request.into_body().collect().await.expect("request body").to_bytes();
+                        seen.lock()
+                            .expect("seen lock")
+                            .push(serde_json::from_slice(&body).expect("request JSON"));
+                        let reply = replies
+                            .lock()
+                            .expect("reply lock")
+                            .pop_front()
+                            .unwrap_or_else(|| Reply::error(StatusCode::SERVICE_UNAVAILABLE));
+                        if !reply.delay.is_zero() {
+                            tokio::time::sleep(reply.delay).await;
+                        }
+                        let mut builder = Response::builder()
+                            .status(reply.status)
+                            .header("content-type", "application/json");
+                        if let Some(value) = reply.retry_after {
+                            builder = builder.header("retry-after", value);
+                        }
+                        Ok::<_, hyper::Error>(
+                            builder
+                                .body(Full::new(Bytes::from(serde_json::to_vec(&reply.body).expect("reply JSON"))))
+                                .expect("reply"),
+                        )
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    TestServer {
+        endpoint: format!("https://localhost:{}/agent/", address.port()),
+        seen,
+        task,
+    }
+}
+
+fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> HeartbeatConfig {
+    let (identity_store, credential_store) = pki.stores(temp);
+    config_with_stores(temp, pki, server, identity_store, credential_store)
+}
+
+fn config_with_stores(
+    temp: &tempfile::TempDir,
+    pki: &TestPki,
+    server: &TestServer,
+    identity_store: IdentityStore,
+    credential_store: CredentialStore,
+) -> HeartbeatConfig {
+    HeartbeatConfig {
+        endpoint: server.endpoint.clone(),
+        root_ca_pem: pki.root_pem.as_bytes().to_vec(),
+        identity_store,
+        credential_store,
+        state_path: temp.path().join("heartbeat/state.json"),
+        schedule: HeartbeatSchedule {
+            cadence: Duration::from_millis(40),
+            jitter: Duration::ZERO,
+            timeout: Duration::from_millis(200),
+            initial_backoff: Duration::from_millis(20),
+            max_backoff: Duration::from_millis(80),
+        },
+    }
+}
+
+fn rewrite_credential(temp: &tempfile::TempDir, update: impl FnOnce(&mut DeviceCredential)) {
+    let path = temp.path().join("credential/device.crt.json");
+    let mut credential: DeviceCredential =
+        serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("parse credential");
+    update(&mut credential);
+    fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("rewrite credential");
+    private_mode(&path);
+}
+
+fn summary() -> CoarseNodeSummary {
+    CoarseNodeSummary::new(8, 7, 1).expect("node summary")
+}
+
+async fn wait_for(
+    status: &mut watch::Receiver<HeartbeatStatus>,
+    predicate: impl Fn(&HeartbeatStatus) -> bool,
+) -> HeartbeatStatus {
+    tokio::time::timeout(Duration::from_secs(3), async {
+        loop {
+            let current = status.borrow_and_update().clone();
+            if predicate(&current) {
+                return current;
+            }
+            status.changed().await.expect("status channel");
+        }
+    })
+    .await
+    .expect("heartbeat status timeout")
+}
+
+async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer, expected: &str) {
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+    assert!(matches!(
+        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+        HeartbeatStatus::Failed { reason } if reason.contains(expected)
+    ));
+    assert!(server.seen.lock().expect("seen lock").is_empty());
+    runtime.shutdown().await;
+}
+
+#[tokio::test]
+async fn connect_config_absent_starts_no_task() {
+    let shutdown = CancellationToken::new();
+    let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let sampled = calls.clone();
+    let runtime = spawn_heartbeat_runtime(None, &shutdown, move || {
+        sampled.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        summary()
+    })
+    .expect("absent config");
+
+    assert!(runtime.is_none());
+    tokio::task::yield_now().await;
+    assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 0);
+}
+
+#[tokio::test]
+async fn duplicate_runtime_is_rejected_without_a_second_task() {
+    let pki = TestPki::new();
+    let mut reply = Reply::ok("2026-08-22T01:02:03Z");
+    reply.delay = Duration::from_secs(5);
+    let server = server(&pki, vec![reply]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let config = config(&temp, &pki, &server);
+    let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary)
+        .expect("first runtime")
+        .expect("configured runtime");
+
+    assert!(matches!(
+        spawn_heartbeat_runtime(Some(config), &shutdown, summary),
+        Err(rustfs::connect::HeartbeatError::AlreadyRunning)
+    ));
+    runtime.shutdown().await;
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn dropped_runtime_keeps_the_lock_until_its_task_stops() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let config = config(&temp, &pki, &server);
+    let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary)
+        .expect("first runtime")
+        .expect("configured runtime");
+
+    drop(runtime);
+    assert!(matches!(
+        spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary),
+        Err(rustfs::connect::HeartbeatError::AlreadyRunning)
+    ));
+
+    let replacement = tokio::time::timeout(Duration::from_secs(3), async {
+        loop {
+            match spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) {
+                Ok(Some(runtime)) => break runtime,
+                Err(rustfs::connect::HeartbeatError::AlreadyRunning) => tokio::task::yield_now().await,
+                Ok(None) => panic!("configured replacement returned no runtime"),
+                Err(error) => panic!("unexpected replacement error: {error}"),
+            }
+        }
+    })
+    .await
+    .expect("dropped runtime releases its lock after stopping");
+    replacement.shutdown().await;
+}
+
+#[tokio::test]
+async fn corrupt_persisted_state_is_rejected_before_network_delivery() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let config = config(&temp, &pki, &server);
+    let directory = config.state_path.parent().expect("state directory");
+    fs::create_dir_all(directory).expect("create state directory");
+    fs::write(
+        &config.state_path,
+        br#"{"nextSequence":0,"pending":{"protocolVersion":"v1","requestId":"550e8400-e29b-41d4-a716-446655440000","agentVersion":"rustfs-agent/1.0.0-rc.3","capabilities":["heartbeat"],"sequence":0,"clientTime":"2026-08-22T01:02:03Z","coarseNodeSummary":{"total":0,"healthy":0,"degraded":0}}}"#,
+    )
+    .expect("write corrupt state");
+    private_mode(&config.state_path);
+    let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+
+    assert!(matches!(
+        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await,
+        HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants")
+    ));
+    assert!(server.seen.lock().expect("seen lock").is_empty());
+    runtime.shutdown().await;
+}
+
+#[tokio::test]
+async fn invalid_stored_resource_name_is_rejected_before_network_delivery() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let config = config(&temp, &pki, &server);
+    rewrite_credential(&temp, |credential| {
+        credential.name = format!("organizations/{ORGANIZATION_UID}/clusters/not-a-uuid/clusterDevices/{DEVICE_UID}");
+    });
+
+    assert_credential_failure(config, &server, "wrong device identity").await;
+}
+
+#[tokio::test]
+async fn invalid_stored_protocol_is_rejected_before_network_delivery() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let config = config(&temp, &pki, &server);
+    rewrite_credential(&temp, |credential| credential.protocol_version = "v2".to_owned());
+
+    assert_credential_failure(config, &server, "wrong device identity").await;
+}
+
+#[tokio::test]
+async fn stored_certificate_key_mismatch_is_rejected_before_network_delivery() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let now = OffsetDateTime::now_utc();
+    let (identity_store, credential_store) =
+        pki.stores_with_certificate(&temp, now - time::Duration::hours(1), now + time::Duration::hours(23), false);
+    let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store);
+
+    assert_credential_failure(config, &server, "different device key").await;
+}
+
+#[tokio::test]
+async fn expired_stored_certificate_is_rejected_before_network_delivery() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let now = OffsetDateTime::now_utc();
+    let (identity_store, credential_store) =
+        pki.stores_with_certificate(&temp, now - time::Duration::days(2), now - time::Duration::days(1), true);
+    let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store);
+
+    assert_credential_failure(config, &server, "not currently valid").await;
+}
+
+#[tokio::test]
+async fn sends_only_l0_fields_and_accepts_additive_response_fields() {
+    let pki = TestPki::new();
+    let server = server(&pki, vec![Reply::ok("2038-01-19T03:14:07Z")]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+
+    assert_eq!(
+        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await,
+        HeartbeatStatus::Online {
+            server_time: "2038-01-19T03:14:07Z".to_owned()
+        }
+    );
+    runtime.shutdown().await;
+    let seen = server.seen.lock().expect("seen lock");
+    let request = &seen[0];
+    let mut keys = request
+        .as_object()
+        .expect("heartbeat object")
+        .keys()
+        .map(String::as_str)
+        .collect::<Vec<_>>();
+    keys.sort_unstable();
+    assert_eq!(
+        keys,
+        [
+            "agentVersion",
+            "capabilities",
+            "clientTime",
+            "coarseNodeSummary",
+            "protocolVersion",
+            "requestId",
+            "sequence"
+        ]
+    );
+    assert_eq!(request["capabilities"], json!(["heartbeat"]));
+    assert_eq!(request["coarseNodeSummary"], json!({"total": 8, "healthy": 7, "degraded": 1}));
+    assert_ne!(request["clientTime"], "2038-01-19T03:14:07Z");
+    assert!(request.get("authorization").is_none());
+}
+
+#[tokio::test]
+async fn restart_replays_pending_request_then_advances_sequence() {
+    let pki = TestPki::new();
+    let first_server = server(&pki, vec![Reply::error(StatusCode::SERVICE_UNAVAILABLE)]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let first_config = config(&temp, &pki, &first_server);
+    let runtime = spawn_heartbeat_runtime(Some(first_config.clone()), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+    wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await;
+    runtime.shutdown().await;
+    let first = first_server.seen.lock().expect("seen lock")[0].clone();
+    drop(first_server);
+
+    let second_server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z"), Reply::ok("2026-08-22T01:02:04Z")]).await;
+    let mut second_config = first_config;
+    second_config.endpoint = second_server.endpoint.clone();
+    let runtime = spawn_heartbeat_runtime(Some(second_config), &shutdown, summary)
+        .expect("restart runtime")
+        .expect("configured runtime");
+    tokio::time::timeout(Duration::from_secs(3), async {
+        while second_server.seen.lock().expect("seen lock").len() < 2 {
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("two heartbeats");
+    runtime.shutdown().await;
+
+    let seen = second_server.seen.lock().expect("seen lock");
+    assert_eq!(seen[0]["requestId"], first["requestId"]);
+    assert_eq!(seen[0]["sequence"], first["sequence"]);
+    assert_ne!(seen[1]["requestId"], seen[0]["requestId"]);
+    assert_eq!(seen[1]["sequence"].as_u64(), seen[0]["sequence"].as_u64().map(|value| value + 1));
+}
+
+#[tokio::test]
+async fn retry_after_is_respected_with_the_local_upper_bound() {
+    let pki = TestPki::new();
+    let mut reply = Reply::error(StatusCode::TOO_MANY_REQUESTS);
+    reply.retry_after = Some("300");
+    let server = server(&pki, vec![reply]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+    assert_eq!(
+        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await,
+        HeartbeatStatus::BackingOff {
+            delay: Duration::from_millis(80)
+        }
+    );
+    runtime.shutdown().await;
+}
+
+#[tokio::test]
+async fn disconnects_use_exponential_backoff_with_a_cap() {
+    let pki = TestPki::new();
+    let server = server(
+        &pki,
+        vec![
+            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
+            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
+            Reply::error(StatusCode::SERVICE_UNAVAILABLE),
+        ],
+    )
+    .await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+    for delay in [20, 40, 80] {
+        assert_eq!(
+            wait_for(&mut status, |status| {
+                matches!(status, HeartbeatStatus::BackingOff { delay: observed } if *observed == Duration::from_millis(delay))
+            })
+            .await,
+            HeartbeatStatus::BackingOff {
+                delay: Duration::from_millis(delay)
+            }
+        );
+    }
+    runtime.shutdown().await;
+}
+
+#[tokio::test]
+async fn revoked_credential_stops_and_exposes_local_status() {
+    let pki = TestPki::new();
+    let mut reply = Reply::error(StatusCode::UNAUTHORIZED);
+    reply.body = json!({"details": [{"reason": "CREDENTIAL_REVOKED"}]});
+    let server = server(&pki, vec![reply]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    let mut status = runtime.status();
+    assert_eq!(
+        wait_for(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await,
+        HeartbeatStatus::AuthenticationStopped {
+            status: 401,
+            reason: Some("CREDENTIAL_REVOKED".to_owned())
+        }
+    );
+    tokio::time::sleep(Duration::from_millis(100)).await;
+    assert_eq!(server.seen.lock().expect("seen lock").len(), 1);
+    runtime.shutdown().await;
+}
+
+#[tokio::test]
+async fn shutdown_cancels_an_in_flight_request() {
+    let pki = TestPki::new();
+    let mut reply = Reply::ok("2026-08-22T01:02:03Z");
+    reply.delay = Duration::from_secs(5);
+    let server = server(&pki, vec![reply]).await;
+    let temp = tempfile::tempdir().expect("tempdir");
+    let shutdown = CancellationToken::new();
+    let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary)
+        .expect("start runtime")
+        .expect("configured runtime");
+    tokio::time::timeout(Duration::from_secs(3), async {
+        while server.seen.lock().expect("seen lock").is_empty() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("request reached server");
+    tokio::time::timeout(Duration::from_millis(250), runtime.shutdown())
+        .await
+        .expect("cancellable shutdown");
+}
+
+#[test]
+fn consumes_the_frozen_heartbeat_fixtures() {
+    let registry: Value =
+        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/fixture-sets.json")).expect("fixture registry");
+    let heartbeat = registry["sets"]
+        .as_array()
+        .expect("fixture sets")
+        .iter()
+        .find(|set| set["name"] == "heartbeat")
+        .expect("heartbeat fixture set");
+    assert_eq!(heartbeat["status"], "populated");
+    let valid: Value =
+        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/valid.json")).expect("valid fixture");
+    assert_eq!(valid["request"]["protocolVersion"], "v1");
+    let overflow: Value =
+        serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/overflow.json")).expect("overflow fixture");
+    assert_eq!(overflow["expected"]["httpStatus"], 422);
+}
+
+#[cfg(unix)]
+fn private_mode(path: &Path) {
+    use std::os::unix::fs::PermissionsExt as _;
+    fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("private mode");
+}
+
+#[cfg(not(unix))]
+fn private_mode(_path: &Path) {}
@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""Fail when a critical scheduled validation has not started recently."""
+
+from __future__ import annotations
+
+import argparse
+from datetime import datetime, timedelta, timezone
+import json
+import os
+from pathlib import Path
+import re
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from urllib.parse import quote, urlencode
+from urllib.request import Request, urlopen
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load_validations(path: Path) -> list[tuple[str, int]]:
+    data = json.loads(path.read_text())
+    if not isinstance(data, list) or not data:
+        raise ValueError("scheduled validation config must be a non-empty list")
+
+    validations: list[tuple[str, int]] = []
+    seen: set[str] = set()
+    for item in data:
+        if not isinstance(item, dict):
+            raise ValueError("scheduled validation entries must be objects")
+        workflow = item.get("workflow")
+        max_age_hours = item.get("max_age_hours")
+        if not isinstance(workflow, str) or not re.fullmatch(
+            r"\.github/workflows/[a-z0-9-]+\.yml", workflow
+        ):
+            raise ValueError(f"invalid scheduled validation workflow: {workflow!r}")
+        if workflow in seen:
+            raise ValueError(f"duplicate scheduled validation workflow: {workflow}")
+        if (
+            not isinstance(max_age_hours, int)
+            or isinstance(max_age_hours, bool)
+            or max_age_hours <= 0
+        ):
+            raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}")
+        seen.add(workflow)
+        validations.append((workflow, max_age_hours))
+    return validations
+
+
+def parse_timestamp(value: object) -> datetime:
+    if not isinstance(value, str):
+        raise ValueError(f"invalid run timestamp: {value!r}")
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        raise ValueError(f"run timestamp has no timezone: {value!r}")
+    return parsed.astimezone(timezone.utc)
+
+
+def stale_reason(
+    run: dict[str, object] | None, now: datetime, max_age_hours: int
+) -> str | None:
+    if run is None:
+        return "no scheduled run has been recorded"
+    created_at = parse_timestamp(run.get("created_at"))
+    age = now - created_at
+    if age > timedelta(hours=max_age_hours):
+        return f"last scheduled run is {age.total_seconds() / 3600:.1f}h old"
+    return None
+
+
+def fetch_latest_scheduled_run(
+    repository: str, workflow: str, token: str, api_url: str
+) -> dict[str, object] | None:
+    owner, repo = repository.split("/", 1)
+    workflow_name = Path(workflow).name
+    endpoint = (
+        f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}"
+        f"/actions/workflows/{quote(workflow_name, safe='')}/runs?"
+        + urlencode({"event": "schedule", "per_page": 1})
+    )
+    request = Request(
+        endpoint,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    with urlopen(request, timeout=30) as response:
+        payload = json.load(response)
+    runs = payload.get("workflow_runs")
+    if not isinstance(runs, list):
+        raise ValueError(f"GitHub returned no workflow_runs list for {workflow}")
+    if not runs:
+        return None
+    if not isinstance(runs[0], dict):
+        raise ValueError(f"GitHub returned an invalid workflow run for {workflow}")
+    return runs[0]
+
+
+def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None:
+    lines = ["## Scheduled validation freshness"]
+    if not failures:
+        lines.append("")
+        lines.append("All critical scheduled validations have a recent scheduled run.")
+    else:
+        lines.extend(
+            [
+                "",
+                "The following critical validations are stale or could not be inspected:",
+                "",
+                "| Workflow | Limit | Result | Last run |",
+                "| --- | ---: | --- | --- |",
+            ]
+        )
+        for workflow, max_age_hours, reason, run_url in failures:
+            link = f"[open]({run_url})" if run_url else "—"
+            lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |")
+    path.write_text("\n".join(lines) + "\n")
+
+
+def check_freshness(
+    config: Path, report: Path, repository: str, token: str, api_url: str
+) -> int:
+    now = datetime.now(timezone.utc)
+    failures: list[tuple[str, int, str, str]] = []
+    for workflow, max_age_hours in load_validations(config):
+        try:
+            run = fetch_latest_scheduled_run(repository, workflow, token, api_url)
+            reason = stale_reason(run, now, max_age_hours)
+            if reason is not None:
+                run_url = str(run.get("html_url", "")) if run else ""
+                failures.append((workflow, max_age_hours, reason, run_url))
+        except Exception as error:
+            failures.append(
+                (workflow, max_age_hours, f"inspection failed: {error}", "")
+            )
+    write_report(report, failures)
+    return 1 if failures else 0
+
+
+class SelfTests(unittest.TestCase):
+    NOW = datetime(2026, 8, 22, 12, tzinfo=timezone.utc)
+
+    def test_freshness_boundaries(self) -> None:
+        at_limit = {"created_at": "2026-08-21T00:00:00Z"}
+        past_limit = {"created_at": "2026-08-20T23:59:59Z"}
+        self.assertIsNone(stale_reason(at_limit, self.NOW, 36))
+        self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36))
+        self.assertIsNotNone(stale_reason(None, self.NOW, 36))
+
+    def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "validations.json"
+            path.write_text(
+                json.dumps(
+                    [
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0},
+                    ]
+                )
+            )
+            with self.assertRaises(ValueError):
+                load_validations(path)
+            path.write_text(
+                json.dumps(
+                    [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}]
+                )
+            )
+            with self.assertRaises(ValueError):
+                load_validations(path)
+
+    def test_check_reports_missing_runs(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            config = root / "validations.json"
+            report = root / "report.md"
+            config.write_text(
+                json.dumps(
+                    [
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36},
+                    ]
+                )
+            )
+            with mock.patch(
+                __name__ + ".fetch_latest_scheduled_run",
+                side_effect=[
+                    {"created_at": "2999-01-01T00:00:00Z"},
+                    None,
+                    RuntimeError("API unavailable"),
+                ],
+            ):
+                self.assertEqual(
+                    check_freshness(
+                        config,
+                        report,
+                        "rustfs/rustfs",
+                        "token",
+                        "https://api.github.test",
+                    ),
+                    1,
+                )
+            contents = report.read_text()
+            self.assertIn(".github/workflows/fuzz.yml", contents)
+            self.assertIn("inspection failed: API unavailable", contents)
+            self.assertNotIn(".github/workflows/ci.yml`", contents)
+
+            config.write_text(
+                json.dumps(
+                    [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}]
+                )
+            )
+            with mock.patch(
+                __name__ + ".fetch_latest_scheduled_run",
+                return_value={"created_at": "2999-01-01T00:00:00Z"},
+            ):
+                self.assertEqual(
+                    check_freshness(
+                        config,
+                        report,
+                        "rustfs/rustfs",
+                        "token",
+                        "https://api.github.test",
+                    ),
+                    0,
+                )
+            self.assertIn("All critical scheduled validations", report.read_text())
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--config", type=Path, default=ROOT / ".github/scheduled-validations.json"
+    )
+    parser.add_argument("--report", type=Path)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        load_validations(args.config)
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
+        return (
+            0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
+        )
+    if args.report is None:
+        parser.error("--report is required unless --self-test is used")
+
+    repository = os.environ.get("GITHUB_REPOSITORY", "")
+    token = os.environ.get("GH_TOKEN", "")
+    api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com")
+    if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository):
+        parser.error("GITHUB_REPOSITORY must be owner/repository")
+    if not token:
+        parser.error("GH_TOKEN is required")
+    return check_freshness(args.config, args.report, repository, token, api_url)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -10,11 +10,17 @@ import sys
 import tempfile
 import tomllib
 import unittest
+from datetime import datetime, timezone
 from unittest import mock
 from pathlib import Path
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError


 ROOT = Path(__file__).resolve().parents[1]
+SCHEDULED_ALERT_WORKFLOWS = tuple(
+    item["workflow"]
+    for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
+)


 def words(value: str) -> set[str]:
@@ -252,6 +258,292 @@ def check_profile_definitions(root: Path) -> list[str]:
    return errors


+def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None:
+    try:
+        start = lines.index(f"{' ' * indent}{key}:") + 1
+    except ValueError:
+        return None
+    end = next(
+        (
+            index
+            for index in range(start, len(lines))
+            if lines[index].strip()
+            and not lines[index].lstrip().startswith("#")
+            and len(lines[index]) - len(lines[index].lstrip()) <= indent
+        ),
+        len(lines),
+    )
+    return lines[start:end]
+
+
+def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None:
+    uses_index = next(
+        (
+            index
+            for index, line in enumerate(job_lines)
+            if (
+                line.split("#", 1)[0].strip() == f"- uses: {action}"
+                and len(line) - len(line.lstrip()) == 6
+            )
+            or (
+                line.split("#", 1)[0].strip() == f"uses: {action}"
+                and len(line) - len(line.lstrip()) == 8
+            )
+        ),
+        None,
+    )
+    if uses_index is None:
+        return None
+    start = next(
+        (
+            index
+            for index in range(uses_index, -1, -1)
+            if job_lines[index].lstrip().startswith("- ")
+        ),
+        uses_index,
+    )
+    indent = len(job_lines[start]) - len(job_lines[start].lstrip())
+    end = next(
+        (
+            index
+            for index in range(start + 1, len(job_lines))
+            if len(job_lines[index]) - len(job_lines[index].lstrip()) == indent
+            and job_lines[index].lstrip().startswith("- ")
+        ),
+        len(job_lines),
+    )
+    return start, job_lines[start:end]
+
+
+def alert_step_errors(
+    job_lines: list[str],
+    expected_action_if: str | None,
+    required_permissions: tuple[str, ...],
+    required_action_tokens: tuple[str, ...],
+) -> list[str]:
+    checkout = workflow_step_block(job_lines, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0")
+    action = workflow_step_block(job_lines, "./.github/actions/schedule-failure-issue")
+    errors: list[str] = []
+    permissions = yaml_block(job_lines, "permissions", 4)
+    permission_text = "\n".join(line.split("#", 1)[0] for line in permissions or [])
+    missing_permissions = [token for token in required_permissions if token not in permission_text]
+    if missing_permissions:
+        errors.append("alert job permissions missing " + ", ".join(missing_permissions))
+    if checkout is None:
+        errors.append("checkout step is missing")
+    if action is None:
+        errors.append("local alert action step is missing")
+    if checkout is None or action is None:
+        return errors
+
+    if checkout[0] >= action[0]:
+        errors.append("checkout must run before the local alert action")
+    checkout_ifs = [line.strip() for line in checkout[1] if line.strip().startswith("if:")]
+    if checkout_ifs:
+        errors.append("checkout step must not be conditional")
+    action_ifs = [line.strip() for line in action[1] if line.strip().startswith("if:")]
+    expected_ifs = [] if expected_action_if is None else [expected_action_if]
+    if action_ifs != expected_ifs:
+        errors.append("alert action has an invalid step condition")
+    action_text = "\n".join(line.split("#", 1)[0] for line in action[1])
+    missing_action_tokens = [token for token in required_action_tokens if token not in action_text]
+    if missing_action_tokens:
+        errors.append("alert action inputs missing " + ", ".join(missing_action_tokens))
+    return errors
+
+
+def schedule_utc_slots(hour: int, minute: int, timezone_name: str | None) -> set[tuple[int, int]]:
+    if timezone_name is None:
+        return {(hour, minute)}
+    zone = ZoneInfo(timezone_name)
+    return {
+        (utc.hour, utc.minute)
+        for year in (2025, 2026)
+        for month in range(1, 13)
+        for utc in [datetime(year, month, 1, hour, minute, tzinfo=zone).astimezone(timezone.utc)]
+    }
+
+
+def check_scheduled_alerts(root: Path) -> list[str]:
+    errors: list[str] = []
+    schedule_slots: dict[tuple[int, int], list[str]] = {}
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        try:
+            lines = path.read_text().splitlines()
+        except FileNotFoundError:
+            errors.append(f"{relative}: missing scheduled validation workflow")
+            continue
+
+        on_block = yaml_block(lines, "on", 0)
+        schedule_block = yaml_block(on_block or [], "schedule", 2)
+        schedule_lines = schedule_block or []
+        cron_indices = [index for index, line in enumerate(schedule_lines) if re.match(r"^\s*-\s+cron:", line)]
+        if not cron_indices:
+            errors.append(f"{relative}: missing simple numeric schedule")
+        else:
+            for position, cron_index in enumerate(cron_indices):
+                cron_line = schedule_lines[cron_index]
+                schedule = re.match(r"^\s*-\s+cron:\s*[\"']?(\d+)\s+(\d+)\s+", cron_line)
+                if not schedule:
+                    errors.append(f"{relative}: missing simple numeric schedule")
+                    continue
+                minute, hour = map(int, schedule.groups())
+                if minute == 0:
+                    errors.append(f"{relative}: scheduled validation must avoid minute zero")
+                entry_end = cron_indices[position + 1] if position + 1 < len(cron_indices) else len(schedule_lines)
+                entry = "\n".join(schedule_lines[cron_index + 1 : entry_end])
+                timezone_match = re.search(r"^\s*timezone:\s*[\"']?([^\"'\s]+)", entry, re.MULTILINE)
+                timezone_name = timezone_match.group(1) if timezone_match else None
+                try:
+                    utc_slots = schedule_utc_slots(hour, minute, timezone_name)
+                except ZoneInfoNotFoundError:
+                    errors.append(f"{relative}: unknown schedule timezone {timezone_name}")
+                    continue
+                for slot in utc_slots:
+                    schedule_slots.setdefault(slot, []).append(relative)
+
+        job_lines = yaml_block(lines, "alert-on-failure", 2)
+        if job_lines is None:
+            errors.append(f"{relative}: missing alert-on-failure job")
+            continue
+        job = "\n".join(line.split("#", 1)[0] for line in job_lines)
+        required = (
+            "always()",
+            "github.event_name == 'schedule'",
+            "contains(needs.*.result, 'failure')",
+            "issues: write",
+            "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+            "uses: ./.github/actions/schedule-failure-issue",
+            "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        )
+        missing = [token for token in required if token not in job]
+        if missing:
+            errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}")
+        else:
+            errors.extend(
+                f"{relative}: {error}"
+                for error in alert_step_errors(job_lines, None, ("issues: write",), ("github-token: ${{ secrets.GITHUB_TOKEN }}",))
+            )
+
+    for (hour, minute), workflows in schedule_slots.items():
+        if len(workflows) > 1:
+            errors.append(
+                f"scheduled validations share {hour:02d}:{minute:02d} UTC: {', '.join(workflows)}"
+            )
+
+    watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml"
+    try:
+        watchdog_lines = watchdog_path.read_text().splitlines()
+    except FileNotFoundError:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog")
+        return errors
+    watchdog_on = yaml_block(watchdog_lines, "on", 0)
+    watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2)
+    watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4)
+    if watchdog_workflows is None:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows")
+        return errors
+    watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows)
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        if not path.is_file():
+            continue
+        source = path.read_text()
+        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
+        if not match:
+            errors.append(f"{relative}: missing workflow name")
+        elif f'- "{match.group(1).strip()}"' not in watchdog_sources:
+            errors.append(f"{relative}: missing from scheduled completion watchdog")
+    watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2)
+    if watchdog_job_lines is None:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job")
+        return errors
+    watchdog_job = "\n".join(line.split("#", 1)[0] for line in watchdog_job_lines)
+    required = (
+        "github.event.workflow_run.event == 'schedule'",
+        "github.event.workflow_run.conclusion != 'success'",
+        "github.event.workflow_run.conclusion != 'failure'",
+        "actions: read",
+        "issues: write",
+        "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+        "uses: ./.github/actions/schedule-failure-issue",
+        "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        "workflow-name: ${{ github.event.workflow_run.name }}",
+        "source-run-id: ${{ github.event.workflow_run.id }}",
+        "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
+        "source-event: ${{ github.event.workflow_run.event }}",
+        "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+        "source-sha: ${{ github.event.workflow_run.head_sha }}",
+    )
+    missing = [token for token in required if token not in watchdog_job]
+    if missing:
+        errors.append(
+            ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing)
+        )
+    else:
+        errors.extend(
+            ".github/workflows/scheduled-validation-watchdog.yml: " + error
+            for error in alert_step_errors(
+                watchdog_job_lines,
+                None,
+                ("actions: read", "issues: write"),
+                (
+                    "github-token: ${{ secrets.GITHUB_TOKEN }}",
+                    "workflow-name: ${{ github.event.workflow_run.name }}",
+                    "source-run-id: ${{ github.event.workflow_run.id }}",
+                    "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
+                    "source-event: ${{ github.event.workflow_run.event }}",
+                    "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+                    "source-sha: ${{ github.event.workflow_run.head_sha }}",
+                ),
+            )
+        )
+
+    freshness_path = root / ".github/workflows/scheduled-validation-freshness.yml"
+    try:
+        freshness_lines = freshness_path.read_text().splitlines()
+    except FileNotFoundError:
+        errors.append(".github/workflows/scheduled-validation-freshness.yml: missing freshness check")
+        return errors
+    freshness_job_lines = yaml_block(freshness_lines, "check-freshness", 2)
+    if freshness_job_lines is None:
+        errors.append(".github/workflows/scheduled-validation-freshness.yml: missing check-freshness job")
+        return errors
+    freshness_job = "\n".join(line.split("#", 1)[0] for line in freshness_job_lines)
+    required = (
+        "python3 scripts/check_scheduled_validation_freshness.py",
+        "actions: read",
+        "issues: write",
+        "if: failure()",
+        "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+        "uses: ./.github/actions/schedule-failure-issue",
+        "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md",
+    )
+    missing = [token for token in required if token not in freshness_job]
+    if missing:
+        errors.append(
+            ".github/workflows/scheduled-validation-freshness.yml: missing " + ", ".join(missing)
+        )
+    else:
+        errors.extend(
+            ".github/workflows/scheduled-validation-freshness.yml: " + error
+            for error in alert_step_errors(
+                freshness_job_lines,
+                "if: failure()",
+                ("actions: read", "issues: write"),
+                (
+                    "github-token: ${{ secrets.GITHUB_TOKEN }}",
+                    "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md",
+                ),
+            )
+        )
+    if not (root / "scripts/check_scheduled_validation_freshness.py").is_file():
+        errors.append("scripts/check_scheduled_validation_freshness.py: missing freshness checker")
+    return errors
+
+
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
    try:
        expected_digest = profile_selection(root, profile)
@@ -281,6 +573,7 @@ def validate(root: Path) -> list[str]:
    errors.extend(check_runner_selection(root))
    errors.extend(check_s3_tests_runner(root))
    errors.extend(check_profile_definitions(root))
+    errors.extend(check_scheduled_alerts(root))
    return errors


@@ -363,6 +656,7 @@ class SelfTests(unittest.TestCase):
                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
+                mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
            ):
                self.assertEqual(len(validate(root)), 1)

@@ -413,6 +707,272 @@ class SelfTests(unittest.TestCase):
            with mock.patch.object(sys, "platform", "linux"):
                self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)

+    def test_scheduled_alerts_require_completion_watchdog(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            alert = (
+                "  alert-on-failure:\n"
+                "    if: always() && github.event_name == 'schedule' && "
+                "contains(needs.*.result, 'failure')\n"
+                "    permissions:\n"
+                "      issues: write\n"
+                "    steps:\n"
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+            )
+            names: list[str] = []
+            for index, relative in enumerate(SCHEDULED_ALERT_WORKFLOWS, start=1):
+                path = root / relative
+                path.parent.mkdir(parents=True, exist_ok=True)
+                names.append(path.stem)
+                path.write_text(
+                    f'name: "{path.stem}"\n'
+                    f'on:\n  schedule:\n    - cron: "{index} {index} * * *"\n'
+                    f'jobs:\n{alert}'
+                )
+            watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
+            watchdog.write_text(
+                "on:\n  workflow_run:\n    workflows:\n"
+                + "\n".join(f'      - "{name}"' for name in names)
+                + "\njobs:\n"
+                + "  alert-on-incomplete-run:\n"
+                + "    github.event.workflow_run.event == 'schedule'\n"
+                + "    github.event.workflow_run.conclusion != 'success'\n"
+                + "    github.event.workflow_run.conclusion != 'failure'\n"
+                + "    permissions:\n"
+                + "      actions: read\n"
+                + "      issues: write\n"
+                + "    steps:\n"
+                + "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                + "      - uses: ./.github/actions/schedule-failure-issue\n"
+                + "        with:\n"
+                + "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                + "          workflow-name: ${{ github.event.workflow_run.name }}\n"
+                + "          source-run-id: ${{ github.event.workflow_run.id }}\n"
+                + "          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n"
+                + "          source-event: ${{ github.event.workflow_run.event }}\n"
+                + "          source-ref-name: ${{ github.event.workflow_run.head_branch }}\n"
+                + "          source-sha: ${{ github.event.workflow_run.head_sha }}\n"
+            )
+            freshness = root / ".github/workflows/scheduled-validation-freshness.yml"
+            freshness.write_text(
+                "jobs:\n"
+                "  check-freshness:\n"
+                "    permissions:\n"
+                "      actions: read\n"
+                "      issues: write\n"
+                "    steps:\n"
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - run: python3 scripts/check_scheduled_validation_freshness.py\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        if: failure()\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                "          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md\n"
+            )
+            checker = root / "scripts/check_scheduled_validation_freshness.py"
+            checker.parent.mkdir()
+            checker.write_text("")
+            self.assertEqual(check_scheduled_alerts(root), [])
+
+            first = root / SCHEDULED_ALERT_WORKFLOWS[0]
+            mutations = (
+                ("contains(needs.*.result, 'failure')", "false"),
+                ("issues: write", "issues: read"),
+                (
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                ),
+                (
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                (
+                    "      - uses: ./.github/actions/schedule-failure-issue\n",
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+            )
+            for required, replacement in mutations:
+                original = first.read_text()
+                first.write_text(original.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+                first.write_text(original)
+
+            first_original = first.read_text()
+            real_steps = (
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+            )
+            first.write_text(
+                first_original.replace(
+                    real_steps,
+                    "      - run: |\n"
+                    "          : <<'MARKER'\n"
+                    "          uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "          MARKER\n"
+                    "      - run: |\n"
+                    "          : <<'MARKER'\n"
+                    "          uses: ./.github/actions/schedule-failure-issue\n"
+                    "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                    "          MARKER\n",
+                )
+            )
+            self.assertTrue(check_scheduled_alerts(root))
+            first.write_text(
+                first_original.replace(
+                    real_steps,
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        with:\n"
+                    "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            watchdog_mutations = (
+                ("actions: read", "actions: none"),
+                ("issues: write", "issues: read"),
+                (
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                ),
+                (
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                (
+                    "      - uses: ./.github/actions/schedule-failure-issue\n",
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+                ("source-event: ${{ github.event.workflow_run.event }}", "source-event: watchdog"),
+                (
+                    "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+                    "source-ref-name: main",
+                ),
+                ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: missing"),
+            )
+            for required, replacement in watchdog_mutations:
+                original = watchdog.read_text()
+                watchdog.write_text(original.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+                watchdog.write_text(original)
+
+            watchdog_original = watchdog.read_text()
+            watchdog.write_text(
+                watchdog_original.replace("issues: write", "issues: read")
+                + "  decoy:\n    permissions:\n      issues: write\n"
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+
+            first_original = first.read_text()
+            first.write_text(
+                first_original.replace('  schedule:\n    - cron: "1 1 * * *"\n', "")
+                + '  decoy:\n    strategy:\n      matrix:\n        cron:\n          - "1 1 * * *"\n'
+                + '    runs-on: ubuntu-latest\n    steps:\n      - run: true\n'
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "1 1 * * *"\n    - cron: "0 5 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "1 1 * * *"\n    - cron: "2 2 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            watchdog.write_text(
+                watchdog_original.replace(f'      - "{names[0]}"\n', "")
+                + f'  decoy:\n    strategy:\n      matrix:\n        workflow:\n          - "{names[0]}"\n'
+                + '    runs-on: ubuntu-latest\n    steps:\n      - run: true\n'
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+
+            watchdog.write_text(watchdog_original.replace(f'      - "{names[0]}"\n', ""))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+            original = first.read_text()
+            first.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "0 0', original, count=1))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(original)
+
+            second = root / SCHEDULED_ALERT_WORKFLOWS[1]
+            second_original = second.read_text()
+            second.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "1 1', second_original, count=1))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            second.write_text(second_original)
+
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "7 0 * * *"\n      timezone: "Asia/Shanghai"\n',
+                )
+            )
+            second.write_text(
+                second_original.replace(
+                    '    - cron: "2 2 * * *"\n',
+                    '    - cron: "2 2 * * *"\n    - cron: "7 16 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+            second.write_text(second_original)
+
+            freshness_original = freshness.read_text()
+            freshness.write_text(freshness_original.replace("details-file:", "report-file:"))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing")
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace(
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace(
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("if: failure()", "if: github.event_name == 'workflow_dispatch'")
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("issues: write", "issues: read")
+                + "  decoy:\n    permissions:\n      issues: write\n"
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+
 def main() -> int:
    if sys.argv[1:] == ["--self-test"]:
        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
@@ -436,7 +996,7 @@ def main() -> int:
        for error in errors:
            print(f"ERROR: {error}", file=sys.stderr)
        return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired")
    return 0


--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
overtrue	acc37b49c8	test(ecstore): cover final sweep cancel fence	2026-08-23 03:21:15 +08:00
overtrue	2379bfb5a3	fix(ecstore): fence final decommission sweep	2026-08-23 03:15:33 +08:00
overtrue	4527b003cc	fix(ecstore): remove duplicate decommission error helper	2026-08-23 03:00:41 +08:00
overtrue	c83aa7f974	Merge remote-tracking branch 'origin/main' into overtrue/fix-1911-ilm-metadata-decommission # Conflicts: # crates/ecstore/src/core/pools.rs	2026-08-23 02:58:32 +08:00
Zhengchao An	2d7120460b	test(e2e): remove fake KMS suite results (#6401 )	2026-08-23 01:45:15 +08:00
Zhengchao An	b91845c98c	ci: align cache writer and reader keys (#6398 )	2026-08-23 01:44:50 +08:00
Zhengchao An	7ba6f8cb33	test(e2e): fix cluster nightly oracles (#6397 )	2026-08-23 01:44:35 +08:00
Zhengchao An	f44b30c61a	ci(perf): fix nightly regression baseline (#6389 )	2026-08-23 01:43:52 +08:00
Zhengchao An	c6590182ed	ci(mint): pin manual image default (#6387 )	2026-08-23 01:43:27 +08:00
Zhengchao An	6d85a9c6a8	ci(s3tests): stabilize HAProxy request handling (#6386 )	2026-08-23 01:42:58 +08:00
Zhengchao An	26e6508b64	fix(ecstore): reject equal-time latest identity conflicts before index fallback (#6374 )	2026-08-23 01:42:11 +08:00
GatewayJ	51e369be6c	fix(policy): accept legacy bucket policy ID field (#6362 )	2026-08-23 01:40:44 +08:00
Zhengchao An	ddc4120c82	ci: detect incomplete and stale scheduled validations (#6357 )	2026-08-23 01:40:28 +08:00
cxymds	87235ffd28	perf(ecstore): bound decommission entry workers (#6360 )	2026-08-23 01:40:03 +08:00
cxymds	62c465ecef	fix(heal): supervise scheduler task panics (#6351 )	2026-08-23 01:39:06 +08:00
houseme	f1b92af4a3	feat(ecstore): coalesce GET ReadVersion RPCs (#6395 )	2026-08-23 01:15:38 +08:00
Zhengchao An	7c1a76dfd9	fix(ecstore): fence deletes against decommission commits (#6363 ) * fix(ecstore): fence deletes against decommission commits * fix(ecstore): preserve decommission target write locks * fix(ecstore): preserve delete markers during source cleanup * fix(ecstore): route batch delete markers to active pools * fix(ecstore): preserve batch delete pool errors * fix(ecstore): retain source-set lock during cleanup * test(ecstore): exercise decommission delete fences * test(ecstore): finish decommission delete fence scenario * fix(ecstore): reuse fixed fence for reverse decommission * fix(ecstore): fence decommission commit loss * fix(ecstore): annotate batch delete fallback * test(ecstore): fix decommission fence fixtures * fix(ecstore): unblock decommission delete fences * fix(ecstore): preserve distributed decommission set locks * fix(ecstore): match decommission lock backend domain * test(ecstore): align decommission fence barriers * fix(ecstore): satisfy delete fence lint checks * fix(rebalance): preserve access-denied delete errors	2026-08-23 00:40:38 +08:00
Zhengchao An	3ddf1a81ac	test(kms): replace 33 hard-coded startup sleeps with readiness probe (#6349 ) * refactor(e2e/kms): replace fixed startup sleeps with KMS readiness probe Replace 33 hard-coded sleep(3s) / sleep(2s) startup waits in KMS e2e tests with an active readiness probe (wait_for_kms_ready) that polls the KMS status endpoint with exponential backoff (200ms→1s, 5s budget). This cuts per-test startup latency from a fixed 3s to ~200-500ms while remaining robust against slow CI machines. Non-startup sleeps (ILM polling loops, fault-recovery detection delays, test-runner inter-test pauses) are left untouched. * style: cargo fmt * fix(kms): use .expect() instead of ? in test functions that return () 7 call sites of wait_for_kms_ready() used ? in async test functions that return () instead of Result. Changed to .expect("KMS ready"). * fix(kms): enforce readiness probe deadline * fix(kms): validate readiness backend status	2026-08-22 16:35:57 +00:00
Zhengchao An	cc412914d5	feat(connect): emit durable heartbeats (#6383 )	2026-08-22 16:03:23 +00:00
overtrue	b6f135f0fe	fix(ecstore): drop removed decommission test import	2026-08-22 23:28:14 +08:00
overtrue	b07f0a92a8	merge: sync main into ILM decommission fix	2026-08-22 22:55:14 +08:00
overtrue	34a52a1e7d	fix(ecstore): restore decommission test imports	2026-08-22 19:13:21 +08:00
overtrue	fe2d166cd3	fix(ecstore): distinguish active ILM target cleanup	2026-08-22 18:43:05 +08:00
overtrue	c687f95260	fix(ecstore): preserve active ILM source journals	2026-08-22 18:43:05 +08:00
overtrue	fe66aa509e	test(ecstore): isolate durable ILM scenario stack	2026-08-22 18:43:05 +08:00
overtrue	918686bd4e	test(ecstore): compile multi-source ILM recovery	2026-08-22 18:43:05 +08:00
overtrue	a9cbf82943	test(ecstore): serialize multi-source ILM recovery	2026-08-22 18:43:05 +08:00
overtrue	ee7a82ec1f	test(ecstore): cover durable ILM recovery boundaries	2026-08-22 18:43:05 +08:00
overtrue	3229ce05e3	fix(ecstore): repair durable ILM receipt recovery	2026-08-22 18:43:04 +08:00
overtrue	7428f3138c	fix(ecstore): harden durable ILM cursor receipts	2026-08-22 18:43:04 +08:00
overtrue	cf0a78ce11	fix(ecstore): avoid terminal receipt shadowing	2026-08-22 18:43:04 +08:00
overtrue	7bda7f4825	fix(ecstore): re-export durable ILM checkpoint	2026-08-22 18:43:04 +08:00
overtrue	20a8c443af	fix(ecstore): anchor decommission ILM receipts	2026-08-22 18:43:04 +08:00
overtrue	948ee1fc52	fix(ecstore): close ILM receipt recovery gaps	2026-08-22 18:43:04 +08:00
overtrue	ecd779b6f3	fix(ecstore): track ILM recovery across decommission	2026-08-22 18:43:04 +08:00
overtrue	619fae2512	fix(ecstore): verify ILM metadata before decommission	2026-08-22 18:43:04 +08:00
overtrue	4f9d22bda7	fix(ecstore): migrate ILM metadata during decommission	2026-08-22 18:43:04 +08:00