fix(heal): fence format repair during pool transitions (#6342)

* fix(heal): fence format repair during pool transitions

* fix(heal): fence format writes during transitions

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
+sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
-sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
+sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6

.config/make/tests.mak

@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test

.github/actions/schedule-failure-issue/action.yml

@@ -14,9 +14,10 @@
 
 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
@@ -38,6 +39,30 @@ inputs:
       Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
+  details-file:
+    description: "Optional Markdown file appended to the issue body."
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -48,17 +73,22 @@ runs:
         GH_TOKEN: ${{ inputs.github-token }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
+        DETAILS_FILE: ${{ inputs.details-file }}
       run: |
         set -euo pipefail
 
         title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
 
-        # Failed job names for this run attempt. The alert job runs while the
+        # Inspect the reported run attempt. It can be the current in-workflow
-        # run as a whole is still in progress, so inspect the jobs that have
+        # failure or a completed run observed by the external watchdog.
-        # already completed with a non-success conclusion.
         failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
                 | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
           failed_jobs="- (failed job not recorded yet — see the run page)"
         fi
 
+        details=""
+        if [ -n "${DETAILS_FILE}" ]; then
+          if [ -f "${DETAILS_FILE}" ]; then
+            details="$(cat "${DETAILS_FILE}")"
+          else
+            details="Details file was not available: \`${DETAILS_FILE}\`"
+          fi
+        fi
+
         body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Run of **${WORKFLOW_NAME}** did not complete successfully.
 
-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
+        - Event: \`${SOURCE_EVENT}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
 
-        Failed jobs:
+        Non-success jobs:
         ${failed_jobs}
+
+        ${details}
         EOF
         )"
 

.github/scheduled-validations.json

@@ -0,0 +1,14 @@
+[
+  { "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+]

.github/workflows/audit.yml

@@ -46,7 +46,7 @@ on:
     # advisory could sit unnoticed for seven days. The check list is unchanged —
     # splitting it into a light daily advisories-only run and a weekly full run
     # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '23 3 * * *' # Daily 03:23 UTC
   workflow_dispatch:
 
 permissions:

.github/workflows/build.yml

@@ -52,7 +52,7 @@ on:
       - ".dockerignore"
       - "flake.lock"
   schedule:
-    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
+    - cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
   workflow_dispatch:
     inputs:
       build_docker:
@@ -1032,3 +1032,23 @@ jobs:
 
           echo "🎉 Released $TAG successfully!"
           echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cache-warm.yml

@@ -94,6 +94,9 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
+  # ci.yml or the writer and readers use disjoint cache keys.
+  RUST_BACKTRACE: 1
 
 jobs:
   # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
@@ -101,7 +104,7 @@ jobs:
   warm-ci-dev:
     name: Warm ci-dev
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -191,7 +194,7 @@ jobs:
   warm-ci-feat-rio:
     name: Warm ci-feat-rio
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -219,7 +222,7 @@ jobs:
   warm-ci-feat-proto:
     name: Warm ci-feat-proto
     runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:

.github/workflows/ci-docs-only.yml

@@ -126,7 +126,10 @@ jobs:
         run: ./scripts/check_embedded_secrets.sh
 
       - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py
 
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh

.github/workflows/ci.yml

@@ -59,7 +59,7 @@ on:
   merge_group:
     types: [ checks_requested ]
   schedule:
-    - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
+    - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
   workflow_dispatch:
 
 permissions:
@@ -161,7 +161,10 @@ jobs:
         run: ./scripts/check_embedded_secrets.sh
 
       - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py
 
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
@@ -1032,3 +1035,37 @@ jobs:
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/coverage.yml

@@ -37,7 +37,7 @@ on:
     # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
     # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
     # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
+    - cron: "43 7 * * 0"
 
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.

.github/workflows/e2e-replication-nightly.yml

@@ -40,7 +40,7 @@ on:
   schedule:
     # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
     # stale (01:30) and performance-ab (06:00).
-    - cron: "0 4 * * *"
+    - cron: "29 4 * * *"
 
 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -196,6 +196,9 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Verify protocol socket oracle
+        run: ss -tn state CLOSE-WAIT >/dev/null
+
       # The suite owns fixed protocol ports and serializes its internal cases.
       - name: Verify protocol e2e membership
         env:

.github/workflows/e2e-s3tests.yml

@@ -90,10 +90,14 @@ on:
         description: "Optional pytest -m expression"
         required: false
         default: ""
+      testexpr:
+        description: "Optional pytest -k expression"
+        required: false
+        default: ""
   schedule:
     # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
     # single-node and the 4-node distributed topologies (matrix below).
-    - cron: "0 2 * * 0"
+    - cron: "19 2 * * 0"
 
 env:
   # main user
@@ -116,6 +120,7 @@ env:
   XDIST: ${{ github.event.inputs.xdist || '4' }}
   MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
   MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
+  TESTEXPR: ${{ github.event.inputs.testexpr || '' }}
   S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
   TEST_TIMEOUT: "300"
 
@@ -269,14 +274,20 @@ jobs:
           EOF
 
           cat > haproxy.cfg <<'EOF'
+          global
+            log stdout format raw local0 info
+
           defaults
             mode http
+            log global
+            log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP'
             timeout connect 5s
             timeout client  30s
             timeout server  30s
 
           frontend fe_s3
             bind *:9000
+            option http-buffer-request
             default_backend be_s3
 
           backend be_s3
@@ -314,6 +325,7 @@ jobs:
           XDIST="${XDIST}" \
           MAXFAIL="${MAXFAIL}" \
           MARKEXPR="${MARKEXPR}" \
+          TESTEXPR="${TESTEXPR}" \
           ./scripts/s3-tests/run.sh
 
       - name: Publish compatibility report

.github/workflows/fuzz.yml

@@ -30,7 +30,7 @@ on:
       - "Cargo.lock"
       - ".github/workflows/fuzz.yml"
   schedule:
-    - cron: "0 2 * * *"
+    - cron: "17 2 * * *"
   workflow_dispatch:
     inputs:
       profile:

.github/workflows/minio-interop.yml

@@ -121,3 +121,21 @@ jobs:
           cargo nextest run --run-ignored ignored-only --no-tests=fail \
             -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
             -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/mint.yml

@@ -45,13 +45,6 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: mint
 
 on:
@@ -70,13 +63,13 @@ on:
           - core
           - full
       mint-image:
-        description: "Mint image reference"
+        description: "Mint image reference (empty = pinned default)"
         required: false
-        default: "minio/mint:edge"
+        default: ""
   schedule:
     # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
     # 3h) has finished, so the two never contend for the same runner pool.
-    - cron: "0 6 * * 0"
+    - cron: "41 6 * * 0"
 
 env:
   S3_ACCESS_KEY: rustfsadmin-ci

.github/workflows/nightly-gnu.yml

@@ -16,7 +16,7 @@ name: Nightly GNU Build
 
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "7 0 * * *"
       timezone: "Asia/Shanghai"
   workflow_dispatch:
 
@@ -194,3 +194,23 @@ jobs:
 
       - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
         run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/performance-ab.yml

@@ -22,22 +22,15 @@
 # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
 # blocked (rustfs/backlog#935 correction 1).
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Performance A/B
 
 on:
   schedule:
-    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
+    - cron: "31 6 * * *" # 06:31 UTC nightly, against main
   workflow_dispatch:
     inputs:
       duration:
-        description: "warp duration per round (short by default to fit the double-build budget)"
+        description: "warp duration per round"
         required: false
         default: "12s"
         type: string
@@ -46,12 +39,8 @@ on:
         required: false
         default: false
         type: boolean
-  push:
-    # Every main commit pre-builds and caches its release binary (perf-3) so the
-    # nightly A/B restores a ready baseline instead of paying the double build.
-    branches: [main]
-
 permissions:
+  actions: read
   contents: read
 
 env:
@@ -59,83 +48,19 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # perf-3: on every push to main, build the release binary once and cache it
-  # keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
-  # restore this instead of paying the ~32min-per-side source
-  # build. That double build is what pushed the expanded 24-cell nightly past its
-  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
-  # builds off the shared cargo cache keep each push cheap, and building on the
-  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
-  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
-  # are cancelled ~7/8 of the time and are not a reliable baseline.
-  build-baseline-cache:
-    name: Build + cache baseline binary
-    if: github.event_name == 'push'
-    runs-on: sm-standard-2
-    # Latest-wins: consumers only ever restore the binary for the *current*
-    # origin/main tip, so when pushes land faster than the ~65min build, a
-    # superseded build's output is dead weight — cancel it instead of stacking
-    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
-    # most costs one same-commit self-heal in the A/B job.
-    concurrency:
-      group: perf-baseline-build-main
-      cancel-in-progress: true
-    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
-    # single release build past 60min on this runner — every cache build on
-    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
-    # time of 1h0m0s") and the cache never populated. The measured binary must
-    # keep the production profile, so the budget absorbs the build instead.
-    timeout-minutes: 100
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Build release rustfs
-        run: cargo build --release --bin rustfs
-
-      - name: Stage binary for cache
-        run: |
-          set -euo pipefail
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-
-      - name: Cache baseline binary by SHA
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ github.sha }}
-
   warp-ab:
     name: Warp A/B budget gate
-    # Always run on schedule / manual dispatch. Never on push — that event only
-    # feeds build-baseline-cache above.
-    if: >-
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch'
     runs-on: sm-standard-2
-    # With perf-3's cached baseline binary the common (cache-hit) nightly is
+    # A normal nightly restores the last successful binary and builds only the
-    # measurement-only and finishes well under 50min. This ceiling stays
+    # candidate; daily access keeps that cache warm. A cache miss may build both
-    # generous only to absorb the same-commit cache-miss self-heal (~65min
+    # and needs room for the A/B run plus artifact and cache publication.
-    # single build with the post-#4806 LTO profile + measurement). A timeout
+    timeout-minutes: 180
-    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
-    # not just failure). perf-6 recalibrates the budget once the noise study
-    # lands.
-    timeout-minutes: 120
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          fetch-depth: 0 # baseline is built from origin/main
+          fetch-depth: 0 # baseline may be an earlier successful scheduled head
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -163,24 +88,55 @@ jobs:
           fi
           echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
 
-      # perf-3: resolve the commits so the cache can be keyed by SHA. The
+      # A failed regression run must keep comparing against the last known-good
-      # baseline is origin/main; the candidate is the checked-out ref. On the
+      # scheduled head. Otherwise the next nightly would absorb the regression
-      # nightly (checkout == main) they are the same commit, so one cached binary
+      # into its baseline and turn green without a fix.
-      # serves both phases and the run does zero source builds.
+      - name: Find last successful scheduled baseline
+        id: scheduled_baseline
+        if: github.event_name == 'schedule'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        with:
+          result-encoding: string
+          script: |
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "performance-ab.yml",
+              event: "schedule",
+              status: "success",
+              per_page: 1,
+            });
+            return data.workflow_runs[0]?.head_sha ?? "";
+
+      # Manual runs compare a selected ref with current main. Scheduled runs
+      # compare current main with the last successful scheduled head. With no
+      # history, the first run measures the candidate against itself and seeds
+      # that head only if the complete rig succeeds.
       - name: Resolve baseline / candidate commits
         id: commits
+        env:
+          SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
         run: |
           set -euo pipefail
-          baseline_sha="$(git rev-parse origin/main)"
           candidate_sha="$(git rev-parse HEAD)"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
+            if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+              echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
+              exit 1
+            fi
+          else
+            baseline_sha="$(git rev-parse origin/main)"
+          fi
+          git cat-file -e "${baseline_sha}^{commit}"
           echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
           echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
           echo "baseline commit: $baseline_sha"
           echo "candidate commit: $candidate_sha"
 
-      # Exact-key restore of the baseline binary built by build-baseline-cache
+      # Exact-key restore of the candidate binary saved by its successful
-      # when origin/main last landed. A miss (binary evicted or not built yet)
+      # scheduled run. A miss leaves cache-hit unset and falls back to a source
-      # leaves cache-hit unset and the rig falls back to a source build.
+      # build of that known-good head.
       - name: Restore cached baseline binary
         id: baseline_cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
@@ -270,11 +226,11 @@ jobs:
             elif [[ "$selfheal_built" == "true" ]]; then
               base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
             else
-              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
+              base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
             fi
             if [[ "$candidate_sha" == "$baseline_sha" ]]; then
-              # Nightly on main: the candidate is the same commit as the baseline,
+              # No commits landed since the last successful baseline, so reuse
-              # so reuse the one binary for both phases and skip all builds.
+              # the one binary for both phases and measure only rig drift.
               args+=(--candidate-bin "$base_bin")
               cand_src="same binary as baseline (same commit)"
             elif [[ "$candidate_built" == "true" ]]; then
@@ -362,6 +318,23 @@ jobs:
             fi
           } >> "$GITHUB_STEP_SUMMARY"
 
+      - name: Stage successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        run: |
+          set -euo pipefail
+          cp candidate-bin/rustfs baseline-bin/rustfs
+
+      - name: Cache successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
+
       # Scheduled failure alerting is handled by the alert-on-failure job below
       # (perf-2 consuming ci-8's schedule-failure-issue composite action).
 

.github/workflows/runner-hygiene.yml

@@ -30,7 +30,7 @@ name: Runner Hygiene
 
 on:
   schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+    - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
   workflow_dispatch:
 
 permissions:

.github/workflows/scheduled-validation-freshness.yml

@@ -0,0 +1,57 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Freshness
+
+on:
+  schedule:
+    - cron: "47 23 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scheduled-validation-freshness
+  cancel-in-progress: false
+
+jobs:
+  check-freshness:
+    name: Check scheduled validation freshness
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check latest scheduled runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set +e
+          python3 scripts/check_scheduled_validation_freshness.py \
+            --report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
+          status=$?
+          cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
+          exit "${status}"
+      - name: Open or update freshness issue
+        if: failure()
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md

.github/workflows/scheduled-validation-watchdog.yml

@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}

Cargo.lock

@@ -12688,6 +12688,7 @@ dependencies = [
  "js-sys",
  "rand 0.10.2",
  "serde_core",
+ "sha1_smol",
  "wasm-bindgen",
 ]
 

crates/common/src/metrics.rs

@@ -901,6 +901,10 @@ pub struct Metrics {
     scanner_cycle_max_duration_millis: AtomicU64,
     scanner_cycle_max_objects: AtomicU64,
     scanner_cycle_max_directories: AtomicU64,
+    scanner_cycle_timeout_total: AtomicU64,
+    scanner_cycle_recovery_required_total: AtomicU64,
+    scanner_cycle_last_progress_age_seconds: AtomicU64,
+    scanner_leader_lease_without_progress: AtomicBool,
     scanner_bitrot_cycle_enabled: AtomicBool,
     scanner_bitrot_cycle_millis: AtomicU64,
     scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport {
     #[serde(default)]
     pub cycle_max_directories: u64,
     #[serde(default)]
+    pub cycle_timeout_total: u64,
+    #[serde(default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(default)]
+    pub leader_lease_without_progress: bool,
+    #[serde(default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(default)]
     pub bitrot_cycle_seconds: f64,
@@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
+const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
+const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
 
 fn scan_cycle_result_label(result: u8) -> &'static str {
     match result {
@@ -1913,6 +1928,10 @@ impl Metrics {
             scanner_cycle_max_duration_millis: AtomicU64::new(0),
             scanner_cycle_max_objects: AtomicU64::new(0),
             scanner_cycle_max_directories: AtomicU64::new(0),
+            scanner_cycle_timeout_total: AtomicU64::new(0),
+            scanner_cycle_recovery_required_total: AtomicU64::new(0),
+            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
+            scanner_leader_lease_without_progress: AtomicBool::new(false),
             scanner_bitrot_cycle_enabled: AtomicBool::new(false),
             scanner_bitrot_cycle_millis: AtomicU64::new(0),
             scanner_checkpoint: Mutex::new(None),
@@ -2412,12 +2431,29 @@ impl Metrics {
             .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
         self.scanner_cycle_max_directories
             .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
+        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
         self.scanner_bitrot_cycle_enabled
             .store(bitrot_cycle.is_some(), Ordering::Relaxed);
         self.scanner_bitrot_cycle_millis
             .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
     }
 
+    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
+        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
+        if recovery_required {
+            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
+        }
+        self.scanner_cycle_last_progress_age_seconds
+            .store(progress_age.as_secs(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
+        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
+    }
+
     pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
         if let Some(concurrency_limit) = concurrency_limit {
             self.scanner_set_scan_concurrency_limit
@@ -3265,6 +3301,10 @@ impl Metrics {
         m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
         m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
+        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
+        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
+        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
+        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
         m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
         m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
         m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4926,4 +4966,20 @@ mod tests {
         assert!(!report.bitrot_cycle_enabled);
         assert_eq!(report.bitrot_cycle_seconds, 0.0);
     }
+
+    #[tokio::test]
+    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
+        let metrics = Metrics::new();
+        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
+        let timed_out = metrics.report().await;
+        assert_eq!(timed_out.cycle_timeout_total, 1);
+        assert_eq!(timed_out.cycle_last_progress_age, 17);
+        assert!(timed_out.leader_lease_without_progress);
+
+        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
+        let current = metrics.report().await;
+        assert_eq!(current.cycle_timeout_total, 1);
+        assert_eq!(current.cycle_last_progress_age, 0);
+        assert!(!current.leader_lease_without_progress);
+    }
 }

crates/config/README.md

@@ -84,6 +84,12 @@ Current guidance:
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
 
+Scanner cycle budget controls:
+
+- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
+- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
+- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
+
 ## Mmap read environment aliases
 
 - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)

crates/config/src/constants/scanner.rs

@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";
 
-/// Default scanner cycle runtime budget.
+/// Default scanner cycle runtime budget when no override is configured.
-/// `0` keeps the existing unbounded per-cycle behavior.
+///
-pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
+/// An explicit `0` remains the compatibility escape hatch for an unbounded
+/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
+/// operation from holding the leader lease forever.
+pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
 
 /// Default scanner per-cycle object budget.
 /// `0` keeps the existing unbounded per-cycle behavior.

crates/data-usage/src/data_usage.rs

@@ -585,9 +585,12 @@ impl VersionsHistogram {
     }
 }
 
-/// Replication statistics for a single target
+/// Replication statistics for a single target.
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+///
-pub struct ReplicationStats {
+/// Renamed from `ReplicationStats`; serde field names are preserved
+/// byte-identically to maintain wire compatibility with existing snapshots.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ReplicationTargetUsage {
     pub pending_size: u64,
     pub replicated_size: u64,
     pub failed_size: u64,
@@ -600,7 +603,7 @@ pub struct ReplicationStats {
     pub replicated_count: u64,
 }
 
-impl ReplicationStats {
+impl ReplicationTargetUsage {
     pub fn is_empty(&self) -> bool {
         let Self {
             pending_size,
@@ -636,7 +639,7 @@ impl ReplicationStats {
 /// Replication statistics for all targets
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct ReplicationAllStats {
-    pub targets: HashMap<String, ReplicationStats>,
+    pub targets: HashMap<String, ReplicationTargetUsage>,
     pub replica_size: u64,
     pub replica_count: u64,
 }
@@ -649,7 +652,7 @@ impl ReplicationAllStats {
             targets,
         } = self;
 
-        *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
+        *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
     }
 
     #[deprecated(note = "use is_empty instead")]
@@ -2466,7 +2469,7 @@ mod tests {
 
     #[test]
     fn replication_stats_empty_checks_every_field() {
-        type SetField = fn(&mut ReplicationStats);
+        type SetField = fn(&mut ReplicationTargetUsage);
 
         let cases: [(&str, SetField); 10] = [
             ("pending_size", |stats| stats.pending_size = 1),
@@ -2481,9 +2484,9 @@ mod tests {
             ("replicated_count", |stats| stats.replicated_count = 1),
         ];
 
-        assert!(ReplicationStats::default().is_empty());
+        assert!(ReplicationTargetUsage::default().is_empty());
         for (field, set_nonzero) in cases {
-            let mut stats = ReplicationStats::default();
+            let mut stats = ReplicationTargetUsage::default();
             set_nonzero(&mut stats);
             assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
         }
@@ -2514,17 +2517,17 @@ mod tests {
         }
 
         let empty_targets = ReplicationAllStats {
-            targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
+            targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
             ..Default::default()
         };
         assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
 
         let stats = ReplicationAllStats {
             targets: HashMap::from([
-                ("arn:test:empty".to_string(), ReplicationStats::default()),
+                ("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
                 (
                     "arn:test:non-empty".to_string(),
-                    ReplicationStats {
+                    ReplicationTargetUsage {
                         pending_count: 1,
                         ..Default::default()
                     },
@@ -2565,7 +2568,7 @@ mod tests {
                 replication_stats: Some(ReplicationAllStats {
                     targets: HashMap::from([(
                         "arn:test:pending".to_string(),
-                        ReplicationStats {
+                        ReplicationTargetUsage {
                             pending_count: 1,
                             ..Default::default()
                         },
@@ -2714,7 +2717,7 @@ mod tests {
                 targets: HashMap::from([
                     (
                         "arn:self-only".to_string(),
-                        ReplicationStats {
+                        ReplicationTargetUsage {
                             pending_size: 7,
                             pending_count: 1,
                             ..Default::default()
@@ -2722,7 +2725,7 @@ mod tests {
                     ),
                     (
                         "arn:shared".to_string(),
-                        ReplicationStats {
+                        ReplicationTargetUsage {
                             failed_size: 3,
                             failed_count: 1,
                             missed_threshold_size: 2,
@@ -2741,7 +2744,7 @@ mod tests {
                 targets: HashMap::from([
                     (
                         "arn:shared".to_string(),
-                        ReplicationStats {
+                        ReplicationTargetUsage {
                             failed_size: 5,
                             failed_count: 2,
                             after_threshold_size: 4,
@@ -2751,7 +2754,7 @@ mod tests {
                     ),
                     (
                         "arn:other-only".to_string(),
-                        ReplicationStats {
+                        ReplicationTargetUsage {
                             replicated_size: 11,
                             replicated_count: 3,
                             ..Default::default()
@@ -2993,7 +2996,9 @@ mod tests {
     fn replication_target_deserialization_preserves_large_historical_maps() {
         let mut stats = ReplicationAllStats::default();
         for index in 0..=1024 {
-            stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
+            stats
+                .targets
+                .insert(format!("target-{index}"), ReplicationTargetUsage::default());
         }
         let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
         let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
@@ -3002,6 +3007,47 @@ mod tests {
         assert_eq!(decoded.targets.len(), stats.targets.len());
     }
 
+    /// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
+    /// must produce the exact same value.  This guards against accidental serde
+    /// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
+    /// rename.  Wire-level field names are the serialized Rust field identifiers,
+    /// which must remain byte-identical.
+    #[test]
+    fn replication_target_usage_rmp_round_trip() {
+        let original = ReplicationTargetUsage {
+            pending_size: 100,
+            replicated_size: 2_000,
+            failed_size: 50,
+            failed_count: 3,
+            pending_count: 7,
+            missed_threshold_size: 11,
+            after_threshold_size: 22,
+            missed_threshold_count: 1,
+            after_threshold_count: 2,
+            replicated_count: 99,
+        };
+
+        let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
+        let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
+        assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
+
+        // Also verify that encoding as an unnamed sequence and then decoding
+        // with named fields produces the correct mapping (this catches reordering).
+        let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
+        // Spot-check that known field names appear in the named encoding.
+        let named_str = String::from_utf8_lossy(&named_buf);
+        assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
+        assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
+        assert!(
+            named_str.contains("missed_threshold_size"),
+            "field 'missed_threshold_size' must survive the rename"
+        );
+        assert!(
+            named_str.contains("after_threshold_count"),
+            "field 'after_threshold_count' must survive the rename"
+        );
+    }
+
     #[test]
     fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
         let mut entry = DataUsageEntry {

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -380,10 +380,24 @@ mod tests {
         cluster.start_node(1).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+        let mut recovered = serde_json::Value::Null;
-        assert!(
+        for _ in 0..60 {
-            !status_body.contains("MissingContentLength"),
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-            "background heal status should not fail without an explicit Content-Length: {status_body}"
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            recovered = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
+                break;
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        assert_eq!(
+            recovered["clusterStatusComplete"],
+            serde_json::Value::Bool(true),
+            "cluster heal status should recover before root heal starts: {recovered}"
         );
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;

crates/e2e_test/src/kms/bucket_default_encryption_test.rs

@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/common.rs

@@ -189,34 +189,121 @@ pub async fn wait_for_kms_ready(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
+    wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
+}
+
+async fn wait_for_kms_ready_with_timeout(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    total_deadline: Duration,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let start = tokio::time::Instant::now();
+    let deadline = start + total_deadline;
     let mut backoff = Duration::from_millis(200);
     let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;
 
     loop {
-        if !first_attempt {
+        match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
-            if start.elapsed() >= total_deadline {
+            Ok(Ok(status)) => {
-                return Err("KMS failed to become ready within 5 seconds".into());
+                let backend_status = serde_json::from_str::<serde_json::Value>(&status)
-            }
+                    .ok()
-            sleep(backoff).await;
+                    .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
-            backoff = (backoff * 2).min(max_backoff);
+                if backend_status.as_deref() == Some("healthy") {
-        }
+                    info!("KMS is ready (status: {})", status);
-        first_attempt = false;
+                    return Ok(());
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
                 }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
+                warn!(
+                    backend_status = backend_status.as_deref().unwrap_or("missing"),
+                    elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
+                    "KMS not ready yet, retrying…"
+                );
             }
+            Ok(Err(e)) => {
+                let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+                warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
+            }
+            Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
         }
+
+        let now = tokio::time::Instant::now();
+        if now >= deadline {
+            return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
+        }
+        sleep((now + backoff).min(deadline) - now).await;
+        backoff = (backoff * 2).min(max_backoff);
+    }
+}
+
+#[cfg(test)]
+mod readiness_tests {
+    use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    #[tokio::test]
+    async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let requests = Arc::new(AtomicUsize::new(0));
+        let server_requests = Arc::clone(&requests);
+        let server = tokio::spawn(async move {
+            for backend_status in ["error", "healthy"] {
+                let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+                let mut request = Vec::new();
+                let mut chunk = [0_u8; 1024];
+                while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    let read = socket.read(&mut chunk).await.expect("read readiness request");
+                    if read == 0 {
+                        break;
+                    }
+                    request.extend_from_slice(&chunk[..read]);
+                }
+                server_requests.fetch_add(1, Ordering::SeqCst);
+
+                let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                socket.write_all(response.as_bytes()).await.expect("write readiness response");
+            }
+        });
+
+        wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
+            .await
+            .expect("KMS should become ready after the healthy response");
+
+        let observed_requests = requests.load(Ordering::SeqCst);
+        server.abort();
+        assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
+    }
+
+    #[tokio::test]
+    async fn kms_readiness_deadline_covers_a_stalled_status_request() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+            let mut request = [0_u8; 1024];
+            let _ = socket.read(&mut request).await.expect("read readiness request");
+            std::future::pending::<()>().await;
+        });
+
+        let result = tokio::time::timeout(
+            Duration::from_secs(1),
+            wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
+        )
+        .await
+        .expect("readiness helper must enforce its own deadline");
+
+        server.abort();
+        assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
     }
 }
 

crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs

@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     let bucket = "copy-object-self-copy-bucket-default-sse-test";

crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs

@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
         )
         .await
         .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     let client = kms_env.base_env.create_s3_client();
     let bucket = "copy-object-version-restore-sse-test";

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -24,7 +24,6 @@ use super::common::{
     test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::info;
 
 /// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = Arc::new(kms_env.base_env.create_s3_client());
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_fault_recovery_test.rs

@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/kms_local_test.rs

@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 
@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
         .expect("Failed to start RustFS with Local KMS");
 
     // Wait for KMS initialization
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");
 
     info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
 

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,7 +19,6 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
@@ -45,8 +44,8 @@ impl VaultKmsTestContext {
 
         start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
-        // Allow Vault to finish initialising token auth and transit engine.
+        // Wait for KMS to finish initialising.
-        sleep(Duration::from_secs(2)).await;
+        super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
         Ok(Self { env })
     }

crates/e2e_test/src/kms/mod.rs

@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
 #[cfg(test)]
 mod kms_fault_recovery_test;
 
-#[cfg(test)]
-mod test_runner;
-
 #[cfg(test)]
 mod bucket_default_encryption_test;
 

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
 
     let mut kms_env = LocalKMSTestEnvironment::new().await?;
     let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;
 
     let s3_client = kms_env.base_env.create_s3_client();
     kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

crates/e2e_test/src/kms/test_runner.rs

@@ -1,499 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-#![allow(dead_code)]
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Unified KMS test suite runner
-//!
-//! This module provides a unified interface for running KMS tests with categorization,
-//! filtering, and comprehensive reporting capabilities.
-
-use crate::common::init_logging;
-use std::time::Instant;
-use tokio::time::{Duration, sleep};
-use tracing::{debug, error, info, warn};
-
-/// Test category for organization and filtering
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TestCategory {
-    CoreFunctionality,
-    MultipartEncryption,
-    EdgeCases,
-    FaultRecovery,
-    Comprehensive,
-    Performance,
-}
-
-impl TestCategory {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            TestCategory::CoreFunctionality => "core-functionality",
-            TestCategory::MultipartEncryption => "multipart-encryption",
-            TestCategory::EdgeCases => "edge-cases",
-            TestCategory::FaultRecovery => "fault-recovery",
-            TestCategory::Comprehensive => "comprehensive",
-            TestCategory::Performance => "performance",
-        }
-    }
-}
-
-/// Test definition with metadata
-#[derive(Debug, Clone)]
-pub struct TestDefinition {
-    pub name: String,
-    pub description: String,
-    pub category: TestCategory,
-    pub estimated_duration: Duration,
-    pub is_critical: bool,
-}
-
-impl TestDefinition {
-    pub fn new(
-        name: impl Into<String>,
-        description: impl Into<String>,
-        category: TestCategory,
-        estimated_duration: Duration,
-        is_critical: bool,
-    ) -> Self {
-        Self {
-            name: name.into(),
-            description: description.into(),
-            category,
-            estimated_duration,
-            is_critical,
-        }
-    }
-}
-
-/// Test execution result
-#[derive(Debug, Clone)]
-pub struct TestResult {
-    pub test_name: String,
-    pub category: TestCategory,
-    pub success: bool,
-    pub duration: Duration,
-    pub error_message: Option<String>,
-}
-
-impl TestResult {
-    pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
-        Self {
-            test_name,
-            category,
-            success: true,
-            duration,
-            error_message: None,
-        }
-    }
-
-    pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
-        Self {
-            test_name,
-            category,
-            success: false,
-            duration,
-            error_message: Some(error),
-        }
-    }
-}
-
-/// Comprehensive test suite configuration
-#[derive(Debug, Clone)]
-pub struct TestSuiteConfig {
-    pub categories: Vec<TestCategory>,
-    pub include_critical_only: bool,
-    pub max_duration: Option<Duration>,
-    pub parallel_execution: bool,
-}
-
-impl Default for TestSuiteConfig {
-    fn default() -> Self {
-        Self {
-            categories: vec![
-                TestCategory::CoreFunctionality,
-                TestCategory::MultipartEncryption,
-                TestCategory::EdgeCases,
-                TestCategory::FaultRecovery,
-                TestCategory::Comprehensive,
-            ],
-            include_critical_only: false,
-            max_duration: None,
-            parallel_execution: false,
-        }
-    }
-}
-
-/// Unified KMS test suite runner
-pub struct KMSTestSuite {
-    tests: Vec<TestDefinition>,
-    config: TestSuiteConfig,
-}
-
-impl KMSTestSuite {
-    /// Create a new test suite with default configuration
-    pub fn new() -> Self {
-        let tests = vec![
-            // Core Functionality Tests
-            TestDefinition::new(
-                "test_local_kms_end_to_end",
-                "End-to-end KMS test with all encryption types",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_local_kms_key_isolation",
-                "Test KMS key isolation and security",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(45),
-                true,
-            ),
-            // Multipart Encryption Tests
-            TestDefinition::new(
-                "test_local_kms_multipart_upload",
-                "Test large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step1_basic_single_file_encryption",
-                "Basic single file encryption test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step2_basic_multipart_upload_without_encryption",
-                "Basic multipart upload without encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step3_multipart_upload_with_sse_s3",
-                "Multipart upload with SSE-S3 encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step4_large_multipart_upload_with_encryption",
-                "Large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(90),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step5_all_encryption_types_multipart",
-                "All encryption types multipart test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            // Edge Cases Tests
-            TestDefinition::new(
-                "test_kms_zero_byte_file_encryption",
-                "Test encryption of zero-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_single_byte_file_encryption",
-                "Test encryption of single-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_boundary_conditions",
-                "Test multipart upload boundary conditions",
-                TestCategory::EdgeCases,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_invalid_key_scenarios",
-                "Test invalid key scenarios",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_concurrent_encryption",
-                "Test concurrent encryption operations",
-                TestCategory::EdgeCases,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_key_validation_security",
-                "Test key validation security",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            // Fault Recovery Tests
-            TestDefinition::new(
-                "test_kms_key_directory_unavailable",
-                "Test KMS when key directory is unavailable",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_corrupted_key_files",
-                "Test KMS with corrupted key files",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_upload_interruption",
-                "Test multipart upload interruption recovery",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_resource_constraints",
-                "Test KMS under resource constraints",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(90),
-                false,
-            ),
-            // Comprehensive Tests
-            TestDefinition::new(
-                "test_comprehensive_kms_full_workflow",
-                "Full KMS workflow comprehensive test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(300),
-                true,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_stress_test",
-                "KMS stress test with large datasets",
-                TestCategory::Comprehensive,
-                Duration::from_secs(400),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_key_isolation",
-                "Comprehensive key isolation test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(180),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_concurrent_operations",
-                "Comprehensive concurrent operations test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(240),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_performance_benchmark",
-                "KMS performance benchmark test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(360),
-                false,
-            ),
-        ];
-
-        Self {
-            tests,
-            config: TestSuiteConfig::default(),
-        }
-    }
-
-    /// Configure the test suite
-    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
-        self.config = config;
-        self
-    }
-
-    /// Filter tests based on category
-    pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| &test.category == category).collect()
-    }
-
-    /// Filter tests based on criticality
-    pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| test.is_critical).collect()
-    }
-
-    /// Get test summary by category
-    pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
-        let mut summary = std::collections::HashMap::new();
-        for test in &self.tests {
-            summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
-        }
-        summary
-    }
-
-    /// Run the complete test suite
-    pub async fn run_test_suite(&self) -> Vec<TestResult> {
-        init_logging();
-        info!("🚀 Starting unified KMS test suite");
-
-        let start_time = Instant::now();
-        let mut results = Vec::new();
-
-        // Filter tests based on configuration
-        let tests_to_run: Vec<&TestDefinition> = self
-            .tests
-            .iter()
-            .filter(|test| self.config.categories.contains(&test.category))
-            .filter(|test| !self.config.include_critical_only || test.is_critical)
-            .collect();
-
-        info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
-        for (i, test) in tests_to_run.iter().enumerate() {
-            info!("  {}. {} ({})", i + 1, test.name, test.category.as_str());
-        }
-
-        // Execute tests
-        for (i, test_def) in tests_to_run.iter().enumerate() {
-            info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
-            info!("   📝 Description: {}", test_def.description);
-            info!("   🏷️  Category: {}", test_def.category.as_str());
-            info!("   ⏱️  Estimated duration: {:?}", test_def.estimated_duration);
-
-            let test_start = Instant::now();
-            let result = self.run_single_test(test_def).await;
-            let test_duration = test_start.elapsed();
-
-            match result {
-                Ok(_) => {
-                    info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
-                    results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
-                }
-                Err(e) => {
-                    error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
-                    results.push(TestResult::failure(
-                        test_def.name.clone(),
-                        test_def.category.clone(),
-                        test_duration,
-                        e.to_string(),
-                    ));
-                }
-            }
-
-            // Add delay between tests to avoid resource conflicts
-            if i < tests_to_run.len() - 1 {
-                debug!("⏸️  Waiting two seconds before the next test...");
-                sleep(Duration::from_secs(2)).await;
-            }
-        }
-
-        let total_duration = start_time.elapsed();
-        self.print_test_summary(&results, total_duration);
-
-        results
-    }
-
-    /// Run a single test by dispatching to the appropriate test function
-    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        // This is a placeholder for test dispatch logic
-        // In a real implementation, this would dispatch to actual test functions
-        warn!("⚠️  Test '{}' is not implemented in the unified runner; skipping", test_def.name);
-        Ok(())
-    }
-
-    /// Print comprehensive test summary
-    fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
-        info!("📊 KMS test suite summary");
-        info!("⏱️  Total duration: {:.2} seconds", total_duration.as_secs_f64());
-        info!("📈 Total tests: {}", results.len());
-
-        let passed = results.iter().filter(|r| r.success).count();
-        let failed = results.iter().filter(|r| !r.success).count();
-
-        info!("✅ Passed: {}", passed);
-        info!("❌ Failed: {}", failed);
-        info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
-
-        // Summary by category
-        let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
-        for result in results {
-            let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
-            *total += 1;
-            if result.success {
-                *passed_count += 1;
-            }
-        }
-
-        info!("📊 Category summary:");
-        for (category, (total, passed_count)) in category_summary {
-            info!(
-                "  🏷️  {}: {}/{} ({:.1}%)",
-                category.as_str(),
-                passed_count,
-                total,
-                (passed_count as f64 / total as f64) * 100.0
-            );
-        }
-
-        // List failed tests
-        if failed > 0 {
-            warn!("❌ Failing tests:");
-            for result in results.iter().filter(|r| !r.success) {
-                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
-            }
-        }
-    }
-}
-
-/// Quick test suite for critical tests only
-#[tokio::test]
-async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let config = TestSuiteConfig {
-        categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
-        include_critical_only: true,
-        max_duration: Some(Duration::from_secs(600)), // 10 minutes max
-        parallel_execution: false,
-    };
-
-    let suite = KMSTestSuite::new().with_config(config);
-    let results = suite.run_test_suite().await;
-
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
-    }
-
-    info!("✅ All critical tests passed");
-    Ok(())
-}
-
-/// Full comprehensive test suite
-#[tokio::test]
-async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let suite = KMSTestSuite::new();
-    let results = suite.run_test_suite().await;
-
-    let total_tests = results.len();
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
-
-    info!("📊 Full suite success rate: {:.1}%", success_rate);
-
-    // Allow up to 10% failure rate for non-critical tests
-    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
-    }
-
-    info!("✅ Full test suite succeeded");
-    Ok(())
-}

crates/e2e_test/src/object_lambda_test.rs

@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
 use rustfs_signer::pre_sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let bucket = "object-lambda-e2e-disabled-target";
     let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
         .send()
         .await?;
 
-    configure_webhook_target_with_key_values(
+    let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
-        &env,
+    tokio::fs::create_dir_all(&queue_dir).await?;
-        "transformer",
+    let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
-        vec![
+    let directive = format!(
-            ("endpoint", "http://127.0.0.1:9/transform".to_string()),
+        "notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
-            ("auth_token", "secret-token".to_string()),
+    );
-            ("enable", "off".to_string()),
+    let disable_response = signed_request(
-        ],
+        http::Method::PUT,
+        &config_url,
+        &env.access_key,
+        &env.secret_key,
+        Some(directive.into_bytes()),
+        Some("text/plain"),
     )
     .await?;
-    wait_for_target_visibility(&env, "transformer").await?;
+    let disable_status = disable_response.status();
+    let disable_body = disable_response.text().await?;
+    assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
 
     let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
     let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let bucket = "object-lambda-e2e-invalid-endpoint";
 
@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;
 
     let response = send_configure_webhook_target_request(
         &env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
     init_logging();
 
     let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.set_env(ENV_NOTIFY_ENABLE, "true");
+    cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
     cluster.start().await?;
 
     let bucket = "listen-notification-cluster";

crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs

@@ -15,7 +15,6 @@
 use crate::common::{RustFSTestClusterEnvironment, init_logging};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::CompletedMultipartUpload;
 use tokio::time::{Duration, sleep};
 use tracing::info;
 use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
     }
 }
 
-async fn complete_reports_missing_upload(
+async fn multipart_listing_reports_missing_upload(
     client: &aws_sdk_s3::Client,
     bucket: &str,
     key: &str,
     upload_id: &str,
 ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
-    let result = client
+    let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
-        .complete_multipart_upload()
+
-        .bucket(bucket)
+    Ok(!result
-        .key(key)
+        .uploads()
-        .upload_id(upload_id)
+        .iter()
-        .multipart_upload(CompletedMultipartUpload::builder().build())
+        .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
-        .send()
-        .await;
-    match result {
-        Ok(_) => Ok(false),
-        Err(SdkError::ServiceError(err)) => {
-            let code = err.err().meta().code().unwrap_or("");
-            if code == "NoSuchUpload" {
-                Ok(true)
-            } else {
-                Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
-            }
-        }
-        Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
-    }
 }
 
 async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
         let mut all_cleaned = true;
         for (idx, client) in clients.iter().enumerate() {
             let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
-            let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
+            let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
-            if !(list_parts_missing && complete_missing) {
+            if !(list_parts_missing && listing_missing) {
                 info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
                 all_cleaned = false;
                 break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
         1,
         "multipart upload should be visible before background cleanup"
     );
+    assert!(
+        !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
+        "multipart upload listing should contain the upload before background cleanup"
+    );
 
     wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
 

crates/ecstore/src/api/mod.rs

@@ -317,8 +317,6 @@ pub mod config {
 }
 
 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +328,8 @@ pub mod data_usage {
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,
     };
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }
 
 pub mod disk {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -3425,6 +3425,44 @@ mod tests {
         assert!(mutexes.contains_key("second"));
     }
 
+    #[tokio::test]
+    async fn update_all_targets_publishes_disable_proxy_on_target_client() {
+        // The read-proxy selector (replication_proxy::get_proxy_targets) skips
+        // targets whose TargetClient carries disable_proxy — the persisted
+        // per-target opt-out must survive client publication.
+        let sys = BucketTargetSys::default();
+        let target = |arn: &str, disable_proxy: bool| BucketTarget {
+            arn: arn.to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            disable_proxy,
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets {
+            targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
+        };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let proxied = sys
+            .get_remote_target_client("bucket", "arn:proxied")
+            .await
+            .expect("client should be published");
+        assert!(!proxied.disable_proxy);
+        let opted_out = sys
+            .get_remote_target_client("bucket", "arn:opted-out")
+            .await
+            .expect("client should be published");
+        assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn target_updates_serialize_client_build_through_publication_per_bucket() {
         let sys = Arc::new(BucketTargetSys::default());

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
         .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size().unwrap_or_default();
+    let asz = oi.get_actual_size_or_physical();
     let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
     let checksum = if ssec { oi.checksum.clone() } else { None };
 

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     };
     let mut replication_state = oi.replication_state();
     replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size().unwrap_or_default();
+    let actual_size = oi.get_actual_size_or_physical();
 
     Ok(ReplicateObjectInfo {
         name: oi.name.clone(),

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
             .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
         version_id: object_info.version_id.map(|version_id| version_id.to_string()),
         etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size().unwrap_or_default(),
+        actual_size: object_info.get_actual_size_or_physical(),
         delete_marker: object_info.delete_marker,
         content_type: object_info.content_type.as_deref(),
         content_encoding: object_info.content_encoding.as_deref(),
@@ -542,6 +542,20 @@ mod tests {
         assert!(replication_target_head_is_newer_null_version(&source, &target));
     }
 
+    #[test]
+    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
+        let source = ObjectInfo {
+            size: 128,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(replication_source_object(&source).actual_size, 128);
+    }
+
     #[test]
     fn replication_target_head_content_matches_compare_etag_only() {
         let source = ObjectInfo {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -42,8 +42,9 @@ use futures::lock::Mutex;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_io_metrics::internode_metrics::{
-    INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
+    INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
-    INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+    INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
+    INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
 };
 use rustfs_protos::ChannelClass;
 use rustfs_protos::evict_failed_connection;
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
 const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
 const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
 const BATCH_METADATA_RPC_OFF: &str = "off";
 const BATCH_METADATA_RPC_AUTO: &str = "auto";
 const BATCH_METADATA_RPC_ON: &str = "on";
@@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
 }
 
 fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
-    rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
+    rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
+        .or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
         .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
         .as_deref()
         .map(parse_batch_metadata_rpc_mode)
@@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
     }
 }
 
+fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
+    if let Some(started_at) = started_at {
+        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
+    }
+}
+
 /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
 /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
 /// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items(
     Ok(batch_read_version_resps)
 }
 
+fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
+    req.items
+        .iter()
+        .fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
+            total
+                .saturating_add(item.org_volume.len())
+                .saturating_add(item.volume.len())
+                .saturating_add(item.path.len())
+                .saturating_add(item.version_id.len())
+        })
+}
+
+fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
+    response
+        .batch_read_version_resps
+        .iter()
+        .map(String::len)
+        .sum::<usize>()
+        .saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
+}
+
 fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
     file_info.validate_for_metadata_read().map_err(Into::into)
 }
@@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
             state = "started",
             "Remote disk RPC started"
         );
+        let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
+        let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
         let batch_read_version_req = compat_json(&req)?;
         let batch_read_version_req_bin = encode_msgpack(&req)?;
-
+        record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
+        let request_payload_bytes = batch_read_version_attribution_enabled
+            .then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
         let batch_result = self
             .execute_with_timeout_for_op(
                 "batch_read_version",
                 move || async move {
                     let disk = self.disk_ref().await;
+                    let disk_len = disk.len();
                     let mut client = self
                         .get_bulk_client()
                         .await
@@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
                         batch_read_version_req_bin: batch_read_version_req_bin.into(),
                     });
 
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
+                    if let Some(request_payload_bytes) = request_payload_bytes {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
+                            request_payload_bytes.saturating_add(disk_len),
+                        );
+                    }
+                    let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
                     let response = match client.batch_read_version(request).await {
-                        Ok(response) => response.into_inner(),
+                        Ok(response) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            response.into_inner()
+                        }
                         Err(status) if status.code() == Code::Unimplemented => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                             if mode.should_fallback_on_unimplemented() {
                                 record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
                                 warn!(
@@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
                             }
 
                             record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                             warn!(
                                 event = EVENT_REMOTE_DISK_RPC,
                                 component = LOG_COMPONENT_ECSTORE,
@@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
                             );
                             return Err(Error::from(status));
                         }
-                        Err(status) => return Err(Error::from(status)),
+                        Err(status) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            return Err(Error::from(status));
+                        }
                     };
 
                     if !response.success {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                         return Err(response.error.unwrap_or_default().into());
                     }
 
-                    decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
+                        batch_read_version_response_payload_len(&response),
+                    );
+                    let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
+                    match decode_batch_read_version_response_items(response, &self.endpoint) {
+                        Ok(batch_read_version_resps) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            Ok(Some(batch_read_version_resps))
+                        }
+                        Err(err) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            Err(err)
+                        }
+                    }
                 },
                 get_max_timeout_duration(),
             )
@@ -4621,6 +4687,7 @@ mod tests {
             } else {
                 "file version not found".to_string()
             },
+            error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
         }
     }
 
@@ -4740,6 +4807,7 @@ mod tests {
     fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
         temp_env::with_vars(
             [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                 (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
                 (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
             ],
@@ -4749,10 +4817,25 @@ mod tests {
         );
     }
 
+    #[test]
+    fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
+                (ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
+                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
+            ],
+            || {
+                assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
+            },
+        );
+    }
+
     #[test]
     fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
         temp_env::with_vars(
             [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                 (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
                 (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
             ],

crates/ecstore/src/cluster/rpc/runtime_sources.rs

@@ -14,9 +14,10 @@
 
 use rustfs_io_metrics::internode_metrics::{
     INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
-    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
-    INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
-    INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
+    INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+    global_internode_metrics,
 };
 use std::time::Duration;
 
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
     );
 }
 
+pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_outgoing_request_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_stage_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        stage,
+        duration,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics()
+        .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_sent_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_recv_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+    record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
+}
+
 pub(crate) fn record_remote_disk_grpc_read_version_error() {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;

crates/ecstore/src/config/storageclass.rs

@@ -248,10 +248,13 @@ impl Config {
         let shard_size = shard_size as usize;
         // Keep the historical two-data-shard object budget while preventing
         // wider EC layouts from multiplying the maximum inline object size.
+        // Use div_ceil to match the shard_file_size calculation (which also uses
+        // div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
+        // for objects right at the threshold.
         let inline_block = if self.initialized && self.inline_block_explicit {
             self.inline_block
         } else {
-            (DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
+            DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
         };
 
         if versioned {

crates/ecstore/src/core/sets.rs

@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
     multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
-    object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::{
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
     }
 }
 
+fn apply_delete_accounting_results(
+    accounting: &mut [Option<DeleteAccounting>],
+    set_objects: &[DelObj],
+    set_accounting: &[Option<DeleteAccounting>],
+) {
+    for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
+        accounting[obj.orig_idx] = value.clone();
+    }
+}
+
+impl Sets {
+    pub(crate) async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut del_errs = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
+        let mut set_obj_map = HashMap::new();
+
+        for (i, obj) in objects.iter().enumerate() {
+            let idx = self.get_hashed_set_index(obj.object_name.as_str());
+            set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
+                orig_idx: i,
+                obj: obj.clone(),
+            });
+        }
+
+        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
+        let mut futures = FuturesUnordered::new();
+        let bucket = bucket.to_owned();
+
+        for (set_index, set_objects) in set_obj_map {
+            let disks = self.get_disks(set_index);
+            let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
+            let bucket = bucket.clone();
+            let opts = opts.clone();
+            let semaphore = semaphore.clone();
+            futures.push(async move {
+                let _permit = semaphore
+                    .acquire_owned()
+                    .await
+                    .expect("delete_objects semaphore should remain open");
+                let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
+                (set_objects, deleted, errors, accounting)
+            });
+        }
+
+        while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
+            apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
+            apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
+        }
+
+        (del_objects, del_errs, accounting)
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for Sets {
     type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        // Default return value
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
-        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        (deleted, errors)
-
-        let mut del_errs = Vec::with_capacity(objects.len());
-        for _ in 0..objects.len() {
-            del_errs.push(None)
-        }
-
-        let mut set_obj_map = HashMap::new();
-
-        // hash key
-        for (i, obj) in objects.iter().enumerate() {
-            let idx = self.get_hashed_set_index(obj.object_name.as_str());
-
-            if !set_obj_map.contains_key(&idx) {
-                set_obj_map.insert(
-                    idx,
-                    vec![DelObj {
-                        // set_idx: idx,
-                        orig_idx: i,
-                        obj: obj.clone(),
-                    }],
-                );
-            } else if let Some(val) = set_obj_map.get_mut(&idx) {
-                val.push(DelObj {
-                    // set_idx: idx,
-                    orig_idx: i,
-                    obj: obj.clone(),
-                });
-            }
-        }
-
-        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
-        let mut futures = FuturesUnordered::new();
-        let bucket = bucket.to_string();
-
-        for (k, v) in set_obj_map {
-            let disks = self.get_disks(k);
-            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let bucket = bucket.clone();
-            let opts = opts.clone();
-            let semaphore = semaphore.clone();
-
-            futures.push(async move {
-                let _permit = semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("delete_objects semaphore should remain open");
-                let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
-                (v, dobjects, errs)
-            });
-        }
-
-        while let Some((v, dobjects, errs)) = futures.next().await {
-            apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
-        }
-
-        (del_objects, del_errs)
     }
 
     #[tracing::instrument(skip(self))]
@@ -985,14 +988,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
     }
 }
 
-#[async_trait::async_trait]
+impl Sets {
-impl crate::storage_api_contracts::heal::HealOperations for Sets {
+    pub(crate) async fn heal_format_with_fence<F>(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option<Error>)>
-    type Error = Error;
+    where
-    type HealResultItem = HealResultItem;
+        F: Fn() -> bool + Send + Sync,
-    type HealOptions = HealOpts;
+    {
-
-    #[tracing::instrument(skip(self))]
-    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         let (disks, init_errs) = init_storage_disks_with_errors(
             &self.endpoints.endpoints,
             &DiskOption {
@@ -1065,6 +1065,9 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
             // Save new formats `format.json` on unformatted disks.
             for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() {
                 if fm.is_some() && disk.is_some() {
+                    if fence_lost() {
+                        return Ok((res, Some(StorageError::SlowDown)));
+                    }
                     if let Err(err) = save_format_file(disk, fm).await {
                         if let Some(disk) = disk.as_ref() {
                             let _ = disk.close().await;
@@ -1098,6 +1101,18 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
         }
         Ok((res, None))
     }
+}
+
+#[async_trait::async_trait]
+impl crate::storage_api_contracts::heal::HealOperations for Sets {
+    type Error = Error;
+    type HealResultItem = HealResultItem;
+    type HealOptions = HealOpts;
+
+    #[tracing::instrument(skip(self))]
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        self.heal_format_with_fence(dry_run, || false).await
+    }
     #[tracing::instrument(skip(self))]
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         let mut result = HealResultItem {

crates/ecstore/src/data_movement/mod.rs

@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::ECStore;
+use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
 fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
     is_unversioned_data_movement_object(source)
         && is_unversioned_data_movement_object(target)
-        && !target.delete_marker
         && source
             .mod_time
             .zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
 pub(crate) struct SourceCleanupBucketFence<'a> {
     pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
     pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
 }
 
 fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
 struct SourceCleanupDeleteBarrierState {
     bucket: String,
     object: String,
+    fence_pending: tokio::sync::Notify,
     arrived: tokio::sync::Notify,
+    is_paused: AtomicBool,
     release: tokio::sync::Notify,
 }
 
@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
 }
 
 #[cfg(test)]
-static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
+static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
     std::sync::OnceLock::new();
 
 #[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
         let state = Arc::new(SourceCleanupDeleteBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
+            fence_pending: tokio::sync::Notify::new(),
             arrived: tokio::sync::Notify::new(),
+            is_paused: AtomicBool::new(false),
             release: tokio::sync::Notify::new(),
         });
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
-            .get_or_init(|| std::sync::Mutex::new(None))
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
             .lock()
             .expect("source cleanup delete barrier mutex should not poison");
-        assert!(slot.is_none(), "source cleanup delete barrier must be unique");
+        assert!(
-        *slot = Some(Arc::clone(&state));
+            !barriers
+                .iter()
+                .any(|barrier| barrier.bucket == bucket && barrier.object == object),
+            "source cleanup delete barrier must be unique per object"
+        );
+        barriers.push(Arc::clone(&state));
         Self { state }
     }
 
@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
             .expect("source cleanup should reach the pre-delete barrier");
     }
 
+    pub(crate) async fn wait_until_fence_pending(&self) {
+        tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
+            .await
+            .expect("source cleanup should attempt the fixed mutation fence");
+    }
+
+    pub(crate) fn is_paused(&self) -> bool {
+        self.state.is_paused.load(Ordering::Acquire)
+    }
+
     pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
 }
 
+#[cfg(test)]
+pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("source cleanup delete barrier mutex should not poison")
+        .iter()
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.fence_pending.notify_one();
+    }
+}
+
 #[cfg(test)]
 impl Drop for SourceCleanupDeleteBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
-            .get_or_init(|| std::sync::Mutex::new(None))
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
             .lock()
             .expect("source cleanup delete barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+        barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
-            *slot = None;
-        }
     }
 }
 
 #[cfg(test)]
 async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
-    let barrier = SOURCE_CLEANUP_DELETE_BARRIER
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
-        .get_or_init(|| std::sync::Mutex::new(None))
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
         .lock()
         .expect("source cleanup delete barrier mutex should not poison")
-        .as_ref()
+        .iter()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
         .cloned();
     if let Some(barrier) = barrier {
+        barrier.is_paused.store(true, Ordering::Release);
         barrier.arrived.notify_one();
         barrier.release.notified().await;
     }
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
     op_label: &str,
 ) -> std::result::Result<ObjectInfo, SourceCleanupError> {
     let cleanup_key = encode_dir_object(object);
-    let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
+    let source_guard = if bucket_fence
-    let _guard = ns_lock
+        .object_mutation_fence
-        .get_write_lock(get_lock_acquire_timeout())
+        .is_some_and(SourceCleanupMutationFence::source_lock_covered)
-        .await
+    {
-        .map_err(Error::from)?;
+        None
+    } else {
+        let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
+        Some(
+            ns_lock
+                .get_write_lock(get_lock_acquire_timeout())
+                .await
+                .map_err(Error::from)?,
+        )
+    };
 
     if bucket_fence
         .lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
             "{op_label}: bucket incarnation fence was lost before source cleanup"
         ))));
     }
+    if bucket_fence
+        .object_mutation_fence
+        .is_some_and(SourceCleanupMutationFence::is_lock_lost)
+    {
+        return Err(SourceCleanupError::Storage(Error::other(format!(
+            "{op_label}: object mutation fence was lost before source cleanup"
+        ))));
+    }
 
     ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
 
@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
         expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
         ..Default::default()
     };
-    opts.add_namespace_lock_guard(&_guard);
+    if let Some(source_guard) = source_guard.as_ref() {
+        opts.add_namespace_lock_guard(source_guard);
+    }
+    if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
+        object_mutation_fence.add_namespace_lock_fence(&mut opts);
+    }
     if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
         opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
     }
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
     }
 }
 
+pub(crate) async fn migrate_decommission_object(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+) -> Result<()> {
+    let source = rd.object_info.clone();
+    let _mutation_fence = store
+        .acquire_decommission_object_mutation_fence(&bucket, &source.name)
+        .await?;
+    let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
+        .await?
+        .ok_or(Error::FileNotFound)?;
+    if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
+        return Err(Error::FileNotFound);
+    }
+
+    migrate_object_inner(
+        store,
+        pool_idx,
+        bucket,
+        rd,
+        source_bucket_incarnation_id,
+        op_label,
+        Some(&_mutation_fence),
+    )
+    .await
+}
+
 pub(crate) async fn migrate_object(
     store: Arc<ECStore>,
     pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
     rd: GetObjectReader,
     source_bucket_incarnation_id: Option<uuid::Uuid>,
     op_label: &str,
+) -> Result<()> {
+    migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
+}
+
+async fn migrate_object_inner(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+    mutation_fence: Option<&ObjectLockDiagGuard>,
 ) -> Result<()> {
     let object_info = rd.object_info.clone();
     let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
         let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
         new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
+            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
             .await
         {
             Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
             if let Err(err) = store
                 .clone()
                 .complete_multipart_upload_for_data_movement(
-                    target_pool_idx,
+                    (target_pool_idx, mutation_fence),
                     &bucket,
                     &object_info.name,
                     &res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
     let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
     put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
     let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
+        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
         .await
         .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
     if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
     }
 
     #[test]
-    fn test_precondition_conflict_rejects_newer_delete_marker() {
+    fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
-        let source = ObjectInfo {
+        for version_id in [None, Some(Uuid::nil())] {
-            size: 128,
+            let source = ObjectInfo {
-            etag: Some("etag-source".to_string()),
+                version_id,
-            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+                size: 128,
-            ..Default::default()
+                etag: Some("etag-source".to_string()),
-        };
+                mod_time: Some(OffsetDateTime::UNIX_EPOCH),
-        let target = ObjectInfo {
+                ..Default::default()
-            delete_marker: true,
+            };
-            etag: None,
+            let target = ObjectInfo {
-            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+                delete_marker: true,
-            ..source.clone()
+                etag: None,
-        };
+                mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+                ..source.clone()
+            };
 
-        let should_resume =
+            assert!(
-            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
+                resolve_data_movement_overwrite_resume_result(
-                .expect("delete marker conflict should be evaluated");
+                    &Error::PreconditionFailed,
+                    Ok(Some(target.clone())),
+                    &source,
+                    0,
+                    1,
+                )
+                .expect("newer null delete marker should be evaluated")
+            );
 
-        assert!(!should_resume);
+            let mut same_time = target.clone();
+            same_time.mod_time = source.mod_time;
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
+                    .expect("same-generation null delete marker should be rejected")
+            );
+
+            let mut versioned = target;
+            versioned.version_id = Some(Uuid::new_v4());
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
+                    .expect("a UUID delete marker must not erase a null source version")
+            );
+        }
     }
 
     #[test]

crates/ecstore/src/data_usage/mod.rs

@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
 }
 
 pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
-    let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
+    // A compressed object may carry -1 while the transformed size is unknown
+    // (legacy streaming sentinel). In that case the persisted physical size
+    // is still a valid accounting floor; every other negative value is corrupt.
+    // An explicit negative `actual-size` metadata value is corrupt, however:
+    // the sentinel is only valid in the in-memory/object-part field written by
+    // the legacy streaming path, not as a persisted declared size.
+    let compressed = object.is_compressed();
+    if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    if object
+        .parts
+        .iter()
+        .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
+        .filter(|value| !value.is_empty());
+    if declared_actual_size
+        .as_deref()
+        .and_then(|value| value.parse::<i64>().ok())
+        .is_some_and(|size| size < 0)
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let logical_size = match object.get_actual_size().map_err(Error::other)? {
+        size if size == -1 && compressed && declared_actual_size.is_none() => None,
+        size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
+        _ => return Err(Error::PartMissingOrCorrupt),
+    };
     let persisted_part_size = if object.parts.is_empty() {
         u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
     } else {
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             // Compressed streaming objects persist -1 when the transformed
             // part size is unknown. The physical part size remains a valid
             // quota floor; reject only non-negative values that overflow.
-            let actual_size = if part.actual_size < 0 {
+            let actual_size = if part.actual_size == -1 {
-                if object.is_compressed() {
+                0
-                    0
-                } else {
-                    return Err(Error::PartMissingOrCorrupt);
-                }
             } else {
                 u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
             };
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
             total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
         })?
     };
-    Ok(logical_size.max(persisted_part_size))
+    Ok(logical_size.unwrap_or(0).max(persisted_part_size))
 }
 
 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3320,6 +3346,80 @@ mod tests {
         );
     }
 
+    #[test]
+    fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 400,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
+    }
+
+    #[test]
+    fn quota_object_size_rejects_compressed_part_sum_overflow() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 1,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: i64::MAX,
+                    ..Default::default()
+                },
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: 1,
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        };
+
+        assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
+    }
+
+    #[test]
+    fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let corrupt_object = ObjectInfo {
+            size: 400,
+            actual_size: -2,
+            user_defined: Arc::new(metadata.clone()),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
+
+        let corrupt_part = ObjectInfo {
+            size: 400,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+                size: 400,
+                actual_size: -2,
+                ..Default::default()
+            }]),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
+    }
+
     #[tokio::test]
     #[serial]
     async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {

crates/ecstore/src/disk/local.rs

@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
 #[cfg(target_os = "linux")]
 const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
 const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
+const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
 const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
 const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
 const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
@@ -6106,6 +6107,43 @@ impl LocalDisk {
         Ok((bytes, modtime))
     }
 
+    async fn write_missing_delete_marker(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        object_dir: &Path,
+        xl_path: &Path,
+        rollback_dir: Option<Uuid>,
+    ) -> Result<()> {
+        if let Some(rollback_dir) = rollback_dir {
+            let rollback_path = object_dir.join(rollback_dir.to_string());
+            fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
+            fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
+                .await
+                .map_err(to_file_error)?;
+        }
+        if let Err(err) = self.write_metadata("", volume, path, fi).await {
+            if let Some(rollback_dir) = rollback_dir
+                && let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
+            {
+                warn!(
+                    event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
+                    result = "failed",
+                    volume,
+                    path,
+                    rollback_dir = %rollback_dir,
+                    error = ?restore_err,
+                    "Disk local delete rollback failed"
+                );
+            }
+            return Err(err);
+        }
+        Ok(())
+    }
+
     async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
         let volume_dir = self.io_get_bucket_path(volume)?;
         let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
@@ -6123,7 +6161,20 @@ impl LocalDisk {
             return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
         }
 
-        let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
+        let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
+            Ok(data) => data,
+            Err(DiskError::FileNotFound) => {
+                // `deleted` alone can be an explicit marker purge; only
+                // `mark_deleted` may create metadata that was not present.
+                let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
+                    return Err(DiskError::FileNotFound);
+                };
+                return self
+                    .write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
+                    .await;
+            }
+            Err(err) => return Err(err),
+        };
 
         if data.is_empty() {
             return Err(DiskError::FileNotFound);
@@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk {
                 }
 
                 if fi.deleted && force_del_marker {
-                    if let Some(rollback_dir) = rollback_dir {
+                    return self
-                        let rollback_path = file_path.join(rollback_dir.to_string());
+                        .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
-                        fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
+                        .await;
-                        fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
-                            .await
-                            .map_err(to_file_error)?;
-                    }
-                    if let Err(err) = self.write_metadata("", volume, path, fi).await {
-                        if let Some(rollback_dir) = rollback_dir
-                            && let Err(restore_err) =
-                                restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
-                        {
-                            warn!(
-                                volume,
-                                path,
-                                rollback_dir = %rollback_dir,
-                                error = ?restore_err,
-                                "failed to restore metadata after delete marker commit error"
-                            );
-                        }
-                        return Err(err);
-                    }
-                    return Ok(());
                 }
 
                 return if fi.version_id.is_some() {

crates/ecstore/src/disk/mod.rs

@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_DISK: &str = "disk";
 const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
+const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
+const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
 
 pub fn part_transaction_path(part_path: &str) -> String {
     match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
 use endpoint::Endpoint;
 use error::DiskError;
 use error::{Error, Result};
+use futures::stream::{self, StreamExt};
 use local::LocalDisk;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(level = "trace", skip_all)]
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        match self {
+            Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
+            Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
+        }
+    }
+
     #[tracing::instrument(level = "trace", skip_all)]
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
         match self {
@@ -1028,36 +1039,47 @@ where
     D: DiskAPI + ?Sized,
 {
     validate_batch_read_version_item_count(req.items.len())?;
+    let parallelism = batch_read_version_server_parallelism();
 
-    let mut responses = Vec::with_capacity(req.items.len());
+    let mut responses = stream::iter(req.items.into_iter().enumerate())
-    for (index, item) in req.items.iter().enumerate() {
+        .map(|(index, item)| async move {
-        let response = match disk
+            match disk
-            .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
+                .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
-            .await
+                .await
-        {
+            {
-            Ok(file_info) => BatchReadVersionResp {
+                Ok(file_info) => BatchReadVersionResp {
-                index,
+                    index,
-                path: item.path.clone(),
+                    path: item.path,
-                version_id: item.version_id.clone(),
+                    version_id: item.version_id,
-                success: true,
+                    success: true,
-                file_info,
+                    file_info,
-                error: String::new(),
+                    error: String::new(),
-            },
+                    error_code: 0,
-            Err(err) => BatchReadVersionResp {
+                },
-                index,
+                Err(err) => BatchReadVersionResp {
-                path: item.path.clone(),
+                    index,
-                version_id: item.version_id.clone(),
+                    path: item.path,
-                success: false,
+                    version_id: item.version_id,
-                file_info: FileInfo::default(),
+                    success: false,
-                error: err.to_string(),
+                    file_info: FileInfo::default(),
-            },
+                    error: err.to_string(),
-        };
+                    error_code: err.to_u32(),
-        responses.push(response);
+                },
-    }
+            }
+        })
+        .buffer_unordered(parallelism)
+        .collect::<Vec<_>>()
+        .await;
+    responses.sort_unstable_by_key(|response| response.index);
 
     Ok(responses)
 }
 
+fn batch_read_version_server_parallelism() -> usize {
+    rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
+        .clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
+}
+
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct CheckPartsResp {
     pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
     pub success: bool,
     pub file_info: FileInfo,
     pub error: String,
+    #[serde(default)]
+    pub error_code: u32,
 }
 
 pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
         assert!(!partial_valid_location.valid());
     }
 
+    #[test]
+    fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
+            assert_eq!(batch_read_version_server_parallelism(), 4);
+        });
+    }
+
+    #[test]
+    fn batch_read_version_server_parallelism_honors_env_with_bounds() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 8);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 1);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
+            assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
+        });
+    }
+
     /// Test FileInfoVersions find_version_index
     #[test]
     fn test_file_info_versions_find_version_index() {

crates/ecstore/src/lib.rs

@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
     cluster::rpc::shutdown_background_monitors();
 }
 
+/// Publish that the process is ready to serve user-object GET traffic.
+///
+/// Experimental metadata coalescing is allowed to run only after this point so
+/// startup and internal metadata reads keep the original per-disk path.
+pub fn mark_get_metadata_read_version_coalescing_service_ready() {
+    runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
+}
+
 #[cfg(test)]
 mod rio_tests {
     #[test]

crates/ecstore/src/object_api/types.rs

@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
 pub struct NamespaceLockFence {
     signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
     #[cfg(test)]
-    forced_lost: Arc<std::sync::atomic::AtomicBool>,
+    forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
 }
 
 impl Debug for NamespaceLockFence {
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
         Self {
             signals: Arc::default(),
             #[cfg(test)]
-            forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
         }
     }
 
     pub(crate) fn is_lock_lost(&self) -> bool {
         #[cfg(test)]
-        if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
+        if self
+            .forced_lost
+            .iter()
+            .any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
+        {
             return true;
         }
         self.signals.iter().any(|signal| signal.is_lost())
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
     }
 
     fn extend(&mut self, other: &Self) {
-        if Arc::ptr_eq(&self.signals, &other.signals) {
+        if !Arc::ptr_eq(&self.signals, &other.signals) {
-            return;
+            Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
         }
-        Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
         #[cfg(test)]
-        if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
+        if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
-            self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+            Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
         }
     }
 
     #[cfg(test)]
     pub(crate) fn lost_for_test() -> Self {
         let fence = Self::new();
-        fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+        fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
         fence
     }
 
     #[cfg(test)]
     pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
         let fence = Self::new();
-        (fence.clone(), Arc::clone(&fence.forced_lost))
+        (fence.clone(), Arc::clone(&fence.forced_lost[0]))
     }
 }
 
@@ -411,6 +414,13 @@ impl ObjectOptions {
         self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
     }
 
+    #[cfg(test)]
+    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+        self.namespace_lock_fence
+            .get_or_insert_with(NamespaceLockFence::new)
+            .extend(fence);
+    }
+
     pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
         self.lifecycle_delete_all_journal
             .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
@@ -689,6 +699,9 @@ impl ObjectInfo {
     }
 
     pub fn get_actual_size(&self) -> std::io::Result<i64> {
+        if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
+            return Err(std::io::Error::other("invalid negative actual size"));
+        }
         if self.actual_size > 0 {
             return Ok(self.actual_size);
         }
@@ -700,10 +713,25 @@ impl ObjectInfo {
                 let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
                 return Ok(size);
             }
-            let mut actual_size = 0;
+            if self.actual_size == -1 && self.parts.is_empty() {
-            self.parts.iter().for_each(|part| {
+                return Ok(-1);
-                actual_size += part.actual_size;
+            }
-            });
+            let mut actual_size = 0_i64;
+            let mut unknown = false;
+            for part in self.parts.iter() {
+                match part.actual_size {
+                    -1 => unknown = true,
+                    size if size >= 0 => {
+                        actual_size = actual_size
+                            .checked_add(size)
+                            .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
+                    }
+                    _ => return Err(std::io::Error::other("invalid negative compressed part size")),
+                }
+            }
+            if unknown {
+                return Ok(-1);
+            }
             if actual_size == 0 && actual_size != self.size {
                 return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
             }
@@ -718,6 +746,18 @@ impl ObjectInfo {
         Ok(self.size)
     }
 
+    /// Returns a non-negative size for client and replication boundaries.
+    ///
+    /// Compressed legacy metadata can retain the internal `-1` unknown-size
+    /// sentinel. Those boundaries cannot emit a negative length, so they use
+    /// the persisted physical size while quota accounting keeps the sentinel
+    /// distinction in [`crate::data_usage::quota_object_size`].
+    pub fn get_actual_size_or_physical(&self) -> i64 {
+        self.get_actual_size()
+            .map(|size| if size >= 0 { size } else { self.size.max(0) })
+            .unwrap_or_else(|_| self.size.max(0))
+    }
+
     pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
         let mut version_id = fi.version_id;
 

crates/ecstore/src/runtime/global.rs

@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
 use rustfs_lock::client::LockClient;
 use std::{
     collections::HashMap,
-    sync::{Arc, OnceLock},
+    sync::{
+        Arc, OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
     time::SystemTime,
 };
 use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;
 
+static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
+
+pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
+}
+
+pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
+}
+
 // Global singletons for backward compatibility with MinIO port.
 // These should be migrated to AppContext over time.
 // See issue #730 for migration plan.

crates/ecstore/src/runtime/instance.rs

@@ -160,6 +160,10 @@ pub struct InstanceContext {
     /// workers (scanner/heal/tier/lifecycle) without touching another instance.
     /// Replaces the process-global cancel-token static.
     background_cancel_token: OnceLock<CancellationToken>,
+    /// Serializes decommission data-movement operations with cancellation and
+    /// a subsequent restart. Readers are held across one object side effect;
+    /// the transition path takes the writer after cancelling the routine.
+    decommission_operation_gate: Arc<RwLock<()>>,
     /// Resolves object-encryption material at the application boundary.
     object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
             local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
             bucket_metadata_sys: std::sync::Mutex::new(None),
             background_cancel_token: OnceLock::new(),
+            decommission_operation_gate: Arc::new(RwLock::new(())),
             object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
         self.lock_manager.clone()
     }
 
+    pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
+        Arc::clone(&self.decommission_operation_gate)
+    }
+
     /// Install the application-owned object-encryption resolver once.
     pub fn set_object_encryption_resolver(
         &self,

crates/ecstore/src/services/metrics_realtime.rs

@@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
         cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
         cycle_max_objects: metrics.cycle_max_objects,
         cycle_max_directories: metrics.cycle_max_directories,
+        cycle_timeout_total: metrics.cycle_timeout_total,
+        cycle_recovery_required_total: metrics.cycle_recovery_required_total,
+        cycle_last_progress_age: metrics.cycle_last_progress_age,
+        leader_lease_without_progress: metrics.leader_lease_without_progress,
         bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
         bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
         scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -611,6 +615,10 @@ mod test {
             current_started: chrono_to_jiff_timestamp(current_started),
             last_cycle_partial_source: "usage".to_string(),
             last_cycle_partial_source_code: 1,
+            cycle_timeout_total: 3,
+            cycle_recovery_required_total: 2,
+            cycle_last_progress_age: 17,
+            leader_lease_without_progress: true,
             partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
                 source: "usage".to_string(),
                 cycles: 2,
@@ -622,6 +630,10 @@ mod test {
         assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
         assert_eq!(scanner.last_cycle_partial_source, "usage");
         assert_eq!(scanner.last_cycle_partial_source_code, 1);
+        assert_eq!(scanner.cycle_timeout_total, 3);
+        assert_eq!(scanner.cycle_recovery_required_total, 2);
+        assert_eq!(scanner.cycle_last_progress_age, 17);
+        assert!(scanner.leader_lease_without_progress);
         let usage = scanner
             .partial_cycles_by_source
             .iter()

crates/ecstore/src/services/rebalance/entry.rs

@@ -334,6 +334,7 @@ impl ECStore {
                             lifecycle_guard: bucket_incarnation_fence
                                 .as_ref()
                                 .and_then(|guard| guard.namespace_lock_guard()),
+                            ..Default::default()
                         },
                         "rebalance",
                     ),

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
     GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
     record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
 };
-use crate::disk::disk_store::DiskStoreRenameDataExt;
+use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
 use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
 use crate::disk::{
-    DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
+    BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
-    PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
+    OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
+    STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
 };
 use crate::erasure::coding::BitrotReader;
 use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
     future::Future,
     pin::Pin,
     sync::{
-        OnceLock,
+        Arc, OnceLock,
         atomic::{AtomicUsize, Ordering},
     },
     task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
     [bucket, object].join("/")
 }
 
+fn read_version_coalescing_enabled() -> bool {
+    let enabled = || {
+        rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
+            .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
+    };
+
+    #[cfg(test)]
+    {
+        enabled()
+    }
+
+    #[cfg(not(test))]
+    {
+        static ENABLED: OnceLock<bool> = OnceLock::new();
+        *ENABLED.get_or_init(enabled)
+    }
+}
+
+fn read_version_coalescing_delay() -> Duration {
+    #[cfg(test)]
+    {
+        let micros = rustfs_utils::get_env_u64(
+            ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+            DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+        );
+        Duration::from_micros(micros)
+    }
+
+    #[cfg(not(test))]
+    {
+        static DELAY: OnceLock<Duration> = OnceLock::new();
+        *DELAY.get_or_init(|| {
+            Duration::from_micros(rustfs_utils::get_env_u64(
+                ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+                DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
+            ))
+        })
+    }
+}
+
+struct CoalescedReadVersionRequest {
+    item: BatchReadVersionItem,
+    tx: oneshot::Sender<disk::error::Result<FileInfo>>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+struct ReadVersionCoalescerKey {
+    disk: usize,
+    incl_free_versions: bool,
+    read_data: bool,
+    healing: bool,
+}
+
+impl ReadVersionCoalescerKey {
+    fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
+        Self {
+            disk: Arc::as_ptr(disk) as usize,
+            incl_free_versions: opts.incl_free_versions,
+            read_data: opts.read_data,
+            healing: opts.healing,
+        }
+    }
+}
+
+#[derive(Default)]
+struct ReadVersionCoalescer {
+    lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
+}
+
+fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
+    static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
+    COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
+}
+
+fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
+    counter!(
+        METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
+        "event" => event,
+        "item_count" => item_count.to_string()
+    )
+    .increment(1);
+}
+
+async fn read_version_via_coalescer(
+    disk: DiskStore,
+    org_bucket: &str,
+    bucket: &str,
+    object: &str,
+    version_id: &str,
+    opts: &ReadOptions,
+    allow_coalescing: bool,
+) -> disk::error::Result<FileInfo> {
+    if !allow_coalescing || !read_version_coalescing_enabled() {
+        return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
+    }
+    if !matches!(disk.as_ref(), Disk::Remote(_)) {
+        record_read_version_coalescer_event("bypass_non_remote", 1);
+        return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
+    }
+
+    let (tx, rx) = oneshot::channel();
+    let item = BatchReadVersionItem {
+        org_volume: org_bucket.to_string(),
+        volume: bucket.to_string(),
+        path: object.to_string(),
+        version_id: version_id.to_string(),
+    };
+    let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
+    let pending = {
+        let mut coalescer = read_version_coalescer().lock().await;
+        let lane = coalescer.lanes.entry(lane_key).or_default();
+        let schedule_delayed_flush = lane.is_empty();
+        lane.push(CoalescedReadVersionRequest { item, tx });
+        if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
+            coalescer.lanes.remove(&lane_key)
+        } else if schedule_delayed_flush {
+            let disk = disk.clone();
+            let task_opts = *opts;
+            tokio::spawn(async move {
+                tokio::time::sleep(read_version_coalescing_delay()).await;
+                flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
+            });
+            None
+        } else {
+            None
+        }
+    };
+
+    if let Some(pending) = pending {
+        flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
+    }
+
+    rx.await
+        .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
+}
+
+async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
+    let pending = {
+        let mut coalescer = read_version_coalescer().lock().await;
+        coalescer.lanes.remove(&lane_key).unwrap_or_default()
+    };
+    flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
+}
+
+async fn flush_read_version_coalescer_pending(
+    lane_key: ReadVersionCoalescerKey,
+    disk: DiskStore,
+    opts: ReadOptions,
+    pending: Vec<CoalescedReadVersionRequest>,
+) {
+    if pending.is_empty() {
+        return;
+    }
+
+    #[cfg(test)]
+    {
+        let mut observed_paths = HashSet::new();
+        for request in &pending {
+            if observed_paths.insert(request.item.path.as_str()) {
+                disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
+            }
+        }
+    }
+
+    let mut senders = Vec::with_capacity(pending.len());
+    let mut items = Vec::with_capacity(pending.len());
+    for request in pending {
+        senders.push(request.tx);
+        items.push(request.item);
+    }
+
+    let expected_items = items.clone();
+    record_read_version_coalescer_event("attempted_batch", items.len());
+    let result =
+        match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
+            .await
+        {
+            Ok(result) => result,
+            Err(_) => Err(DiskError::Timeout),
+        };
+    match result {
+        Ok(responses) => {
+            let results = map_batch_read_version_responses(&expected_items, responses);
+            for (tx, result) in senders.into_iter().zip(results) {
+                let _ = tx.send(result);
+            }
+        }
+        Err(err) => {
+            let message = err.to_string();
+            for tx in senders {
+                let _ = tx.send(Err(DiskError::other(message.clone())));
+            }
+        }
+    }
+}
+
+fn map_batch_read_version_responses(
+    expected_items: &[BatchReadVersionItem],
+    responses: Vec<BatchReadVersionResp>,
+) -> Vec<crate::disk::error::Result<FileInfo>> {
+    let mut results = (0..expected_items.len())
+        .map(|_| Err(DiskError::other("coalesced read_version response missing")))
+        .collect::<Vec<_>>();
+    let mut seen = vec![false; expected_items.len()];
+    for response in responses {
+        let Some(expected) = expected_items.get(response.index) else {
+            continue;
+        };
+        let Some(slot) = results.get_mut(response.index) else {
+            continue;
+        };
+        if seen[response.index] {
+            *slot = Err(DiskError::other("coalesced read_version response duplicate index"));
+            continue;
+        }
+        seen[response.index] = true;
+        if response.path != expected.path || response.version_id != expected.version_id {
+            *slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
+        } else {
+            *slot = if response.success {
+                Ok(response.file_info)
+            } else {
+                Err(batch_read_version_response_error(response.error_code, response.error))
+            };
+        }
+    }
+    results
+}
+
+fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
+    match DiskError::from_u32(error_code) {
+        Some(DiskError::Io(_)) | None => DiskError::other(error),
+        Some(error) => error,
+    }
+}
+
 pub(in crate::set_disk) fn bounded_metadata_fanout_order(
     bucket: &str,
     object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
     order
 }
 use tokio::io::{AsyncRead, ReadBuf};
-use tokio::sync::RwLock;
+use tokio::sync::{Mutex, RwLock, oneshot};
 use tokio::task::JoinSet;
 
 pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
 pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
+const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
+const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
 pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
 /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
 /// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
             false,
             true,
             0,
+            false,
         )
         .await?;
         Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
             true,
             caller_allows_early_stop,
             default_parity_count,
+            false,
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
+        disks: &[Option<DiskStore>],
+        org_bucket: &str,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        read_data: bool,
+        incl_free_versions: bool,
+        caller_allows_early_stop: bool,
+        default_parity_count: usize,
+    ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
+        Self::read_all_fileinfo_inner(
+            disks,
+            org_bucket,
+            bucket,
+            object,
+            version_id,
+            read_data,
+            false,
+            incl_free_versions,
+            true,
+            caller_allows_early_stop,
+            default_parity_count,
+            true,
         )
         .await
     }
@@ -2408,6 +2680,7 @@ impl SetDisks {
         // subset would fail write quorum (backlog#872 regression).
         caller_allows_early_stop: bool,
         default_parity_count: usize,
+        allow_coalescing: bool,
     ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
         let early_stop_enabled =
             caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
                 healing,
                 incl_free_versions,
                 default_parity_count,
+                allow_coalescing,
             )
             .await;
         }
@@ -2446,6 +2720,7 @@ impl SetDisks {
             healing,
             incl_free_versions,
             observe,
+            allow_coalescing,
         )
         .await
     }
@@ -2461,6 +2736,7 @@ impl SetDisks {
         healing: bool,
         incl_free_versions: bool,
         observe: bool,
+        allow_coalescing: bool,
     ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
         let fanout_start = observe.then(Instant::now);
         let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
                     if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
                         tokio::time::sleep(delay).await;
                     }
-                    disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
+                    read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
                         .await
                 } else {
                     Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
         healing: bool,
         incl_free_versions: bool,
         default_parity_count: usize,
+        allow_coalescing: bool,
     ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
         let fanout_start = Instant::now();
         let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
                         if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
                             tokio::time::sleep(delay).await;
                         }
-                        disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
+                        read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
                             .await
                     } else {
                         Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
 
     /// Kind label for the per-disk `read_version` metadata RPC.
     pub const KIND_READ_VERSION: &str = "read_version";
+    pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
 
     /// Registry key: (object, kind, disk_index).
     type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
         drop(dirs);
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn metadata_read_version_coalescer_bypasses_local_disks() {
+        const DISKS: usize = 4;
+        let bucket = "coalesced-read-version-local-bypass-bucket";
+        let object_a = "coalesced-local-object-a";
+        let object_b = "coalesced-local-object-b";
+        let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
+
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
+            ],
+            async {
+                let calls = disk_call_counters::observe(object_a);
+                let disks_a = disks.clone();
+                let disks_b = disks.clone();
+                let read_a = tokio::spawn(async move {
+                    SetDisks::read_all_fileinfo_observed_for_get_object(
+                        &disks_a, "", bucket, object_a, "", false, false, false, 2,
+                    )
+                    .await
+                    .map(|(file_infos, errors, _)| (file_infos, errors))
+                });
+                tokio::task::yield_now().await;
+                let read_b = tokio::spawn(async move {
+                    SetDisks::read_all_fileinfo_observed_for_get_object(
+                        &disks_b, "", bucket, object_b, "", false, false, false, 2,
+                    )
+                    .await
+                    .map(|(file_infos, errors, _)| (file_infos, errors))
+                });
+
+                let (metadata_a, errs_a) = read_a
+                    .await
+                    .expect("first read task should not panic")
+                    .expect("first coalesced read should resolve");
+                let (metadata_b, errs_b) = read_b
+                    .await
+                    .expect("second read task should not panic")
+                    .expect("second coalesced read should resolve");
+
+                assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
+                assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
+                assert!(errs_a.iter().all(Option::is_none));
+                assert!(errs_b.iter().all(Option::is_none));
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_READ_VERSION),
+                    DISKS as u64,
+                    "local disks still execute the ordinary per-disk read_version path"
+                );
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
+                    0,
+                    "GET coalescing targets internode RPC count only and must not batch local disk reads"
+                );
+            },
+        )
+        .await;
+
+        drop(dirs);
+    }
+
+    #[tokio::test]
+    async fn metadata_read_version_coalescer_requires_get_object_intent() {
+        const DISKS: usize = 4;
+        let bucket = "coalesced-read-version-default-bypass-bucket";
+        let object = "default-bypass-object";
+        let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
+        install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
+
+        temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
+            let calls = disk_call_counters::observe(object);
+            let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
+                .await
+                .expect("default metadata read should resolve");
+
+            assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
+            assert!(errs.iter().all(Option::is_none));
+            assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
+            assert_eq!(
+                calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
+                0,
+                "non-GET metadata paths must bypass coalescer even when the env gate is enabled"
+            );
+        })
+        .await;
+
+        drop(dirs);
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_preserves_index_and_errors() {
+        let expected_items = vec![
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-b".to_string(),
+                version_id: "v-b".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-c".to_string(),
+                version_id: "v-c".to_string(),
+            },
+        ];
+        let ok_file_info = FileInfo {
+            name: "object-a".to_string(),
+            ..Default::default()
+        };
+        let responses = vec![
+            BatchReadVersionResp {
+                index: 2,
+                path: "object-c".to_string(),
+                version_id: "v-c".to_string(),
+                success: false,
+                file_info: FileInfo::default(),
+                error: "disk read failed".to_string(),
+                error_code: 0,
+            },
+            BatchReadVersionResp {
+                index: 0,
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+                success: true,
+                file_info: ok_file_info,
+                error: String::new(),
+                error_code: 0,
+            },
+        ];
+
+        let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
+        let first = results
+            .next()
+            .expect("slot 0 should exist")
+            .expect("slot 0 should map the success response by index");
+        assert_eq!(first.name, "object-a");
+
+        let missing = results
+            .next()
+            .expect("slot 1 should exist")
+            .expect_err("slot 1 should stay missing");
+        assert!(
+            missing.to_string().contains("response missing"),
+            "unexpected missing response error: {missing}"
+        );
+
+        let failed = results
+            .next()
+            .expect("slot 2 should exist")
+            .expect_err("slot 2 should map the response error");
+        assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
+        assert!(results.next().is_none());
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
+        let expected_items = vec![
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-a".to_string(),
+                version_id: "v-a".to_string(),
+            },
+            BatchReadVersionItem {
+                org_volume: String::new(),
+                volume: "bucket".to_string(),
+                path: "object-b".to_string(),
+                version_id: "v-b".to_string(),
+            },
+        ];
+        let results = map_batch_read_version_responses(
+            &expected_items,
+            vec![
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: DiskError::FileNotFound.to_string(),
+                    error_code: DiskError::FileNotFound.to_u32(),
+                },
+                BatchReadVersionResp {
+                    index: 1,
+                    path: "object-b".to_string(),
+                    version_id: "v-b".to_string(),
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: DiskError::FileVersionNotFound.to_string(),
+                    error_code: DiskError::FileVersionNotFound.to_u32(),
+                },
+            ],
+        );
+
+        assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
+        assert!(matches!(
+            results.get(1).expect("slot 1 should exist"),
+            Err(DiskError::FileVersionNotFound)
+        ));
+    }
+
+    #[test]
+    fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
+        let expected_items = vec![BatchReadVersionItem {
+            org_volume: String::new(),
+            volume: "bucket".to_string(),
+            path: "object-a".to_string(),
+            version_id: "v-a".to_string(),
+        }];
+        let mismatched = map_batch_read_version_responses(
+            &expected_items,
+            vec![BatchReadVersionResp {
+                index: 0,
+                path: "object-b".to_string(),
+                version_id: "v-a".to_string(),
+                success: true,
+                file_info: FileInfo {
+                    name: "object-b".to_string(),
+                    ..Default::default()
+                },
+                error: String::new(),
+                error_code: 0,
+            }],
+        )
+        .pop()
+        .expect("slot 0 should exist")
+        .expect_err("identity mismatch should fail closed");
+        assert!(
+            mismatched.to_string().contains("identity mismatch"),
+            "unexpected mismatch error: {mismatched}"
+        );
+
+        let duplicate = map_batch_read_version_responses(
+            &expected_items,
+            vec![
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: true,
+                    file_info: FileInfo {
+                        name: "object-a".to_string(),
+                        ..Default::default()
+                    },
+                    error: String::new(),
+                    error_code: 0,
+                },
+                BatchReadVersionResp {
+                    index: 0,
+                    path: "object-a".to_string(),
+                    version_id: "v-a".to_string(),
+                    success: true,
+                    file_info: FileInfo {
+                        name: "object-a".to_string(),
+                        ..Default::default()
+                    },
+                    error: String::new(),
+                    error_code: 0,
+                },
+            ],
+        )
+        .pop()
+        .expect("slot 0 should exist")
+        .expect_err("duplicate response index should fail closed");
+        assert!(
+            duplicate.to_string().contains("duplicate index"),
+            "unexpected duplicate error: {duplicate}"
+        );
+    }
+
     /// Isolation guard: unobserved objects record nothing (so parallel tests do
     /// not inflate one another), and a scope clears its own counts on drop.
     #[tokio::test]

crates/ecstore/src/set_disk/mod.rs

@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
         CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
     },
     namespace::NamespaceLocking as _,
-    object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
     range::HTTPRangeSpec,
 };
 use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters;
 mod ctx;
 mod metadata;
 mod ops;
+#[cfg(test)]
+pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 #[cfg(any(test, feature = "test-util"))]
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
+#[cfg(test)]
+pub(crate) use ops::object::DeleteObjectCommitBarrier;
 #[cfg(feature = "test-util")]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
@@ -3025,6 +3029,16 @@ pub struct SetDisks {
     storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
 }
 
+// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
+// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
+pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    left.iter()
+        .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
+        && right
+            .iter()
+            .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -3600,6 +3614,15 @@ impl SetDisks {
         &self.ctx
     }
 
+    /// Whether both sets' namespace-lock implementations cover the same object key.
+    pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
+        match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
+            (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
+            (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
+            _ => false,
+        }
+    }
+
     /// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
     #[cfg(test)]
     pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
@@ -4584,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
 }
 
 fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
-    opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
+    opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
 }
 
 fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
-    let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
+    let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
     let mut delete_marker = opts.versioned;
 
     if opts.version_id.is_some() {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint};
 use crate::multipart_listing::paginate_multipart_listing;
 use futures::{StreamExt, stream};
 use std::future::Future;
+#[cfg(test)]
+use std::sync::atomic::AtomicBool;
 #[cfg(any(test, feature = "test-util"))]
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
@@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard {
 #[cfg(any(test, feature = "test-util"))]
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum MultipartCommitPause {
+    NewUploadBeforeLockLost,
     PutPartBeforeLockAcquire,
     PutPartBeforeLockLost,
     PutPartAfterRename,
@@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier {
     }
 }
 
+#[cfg(test)]
+struct NewMultipartUploadCommitObservationState {
+    bucket: String,
+    object: String,
+    committed: AtomicBool,
+}
+
+#[cfg(test)]
+pub(crate) struct NewMultipartUploadCommitObservation {
+    state: Arc<NewMultipartUploadCommitObservationState>,
+}
+
+#[cfg(test)]
+static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl NewMultipartUploadCommitObservation {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(NewMultipartUploadCommitObservationState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            committed: AtomicBool::new(false),
+        });
+        let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("new multipart upload commit observation mutex should not poison");
+        assert!(slot.is_none(), "new multipart upload commit observation must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) fn committed(&self) -> bool {
+        self.state.committed.load(Ordering::Acquire)
+    }
+}
+
+#[cfg(test)]
+impl Drop for NewMultipartUploadCommitObservation {
+    fn drop(&mut self) {
+        let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("new multipart upload commit observation mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
+    let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("new multipart upload commit observation mutex should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket && state.object == object)
+        .cloned();
+    if let Some(state) = state {
+        state.committed.store(true, Ordering::Release);
+    }
+}
+
 #[cfg(any(test, feature = "test-util"))]
 async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
     let barrier = {
@@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
         let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
 
+        #[cfg(any(test, feature = "test-util"))]
+        pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
+        if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "new_multipart_upload_commit",
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                required: 1,
+                achieved: 0,
+            });
+        }
+        if opts
+            .namespace_lock_fence
+            .as_ref()
+            .is_some_and(NamespaceLockFence::is_lock_lost)
+        {
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "new_multipart_upload_outer_lock",
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                required: 1,
+                achieved: 0,
+            });
+        }
         ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
         Self::write_unique_file_info(
             &shuffle_disks,
@@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         )
         .await
         .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
+        #[cfg(test)]
+        observe_new_multipart_upload_commit(bucket, object);
 
         // evalDisks
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -45,6 +45,7 @@ use crate::bucket::replication::{
     DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
     replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
 };
+use crate::data_usage::quota_object_size;
 use crate::diagnostics::get::GetObjectFailureReason;
 use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
 use crate::error::is_err_invalid_upload_id;
@@ -1293,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
             (prepared.snapshot, prepared.object_info)
         } else {
             match self
-                .get_object_fileinfo(
+                .get_object_fileinfo_for_get_object_reader(
                     bucket,
                     object,
                     opts,
@@ -2122,14 +2123,27 @@ impl SetDisks {
             let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
 
             let put_object_size = known_put_object_storage_size(data.size());
+            let shard_file_size_raw = erasure.shard_file_size(put_object_size);
             let is_inline_buffer =
-                storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned);
+                storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
 
             let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
-            let shard_file_size = erasure.shard_file_size(put_object_size);
+            let shard_file_size = shard_file_size_raw;
             let shard_size = erasure.shard_size();
             let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
             let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
+            {
+                use std::io::Write;
+                let msg = format!(
+                    "INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
+                    bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
+                    is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
+                );
+                if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
+                    let _ = f.write_all(msg.as_bytes());
+                }
+                let _ = std::io::stderr().write_all(msg.as_bytes());
+            }
             rustfs_io_metrics::record_put_object_path(write_path.metric_label());
             let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
             let (mut writers, errors) = if direct_inline_commit {
@@ -2483,6 +2497,7 @@ impl SetDisks {
                         })
                         .await?,
                     );
+                    notify_put_object_commit_namespace_acquired(bucket, object);
                 }
                 #[cfg(not(any(test, feature = "test-util")))]
                 {
@@ -4630,6 +4645,7 @@ struct PutObjectCommitBarrierState {
     arrived: tokio::sync::Notify,
     release: tokio::sync::Notify,
     namespace_pending: tokio::sync::Notify,
+    namespace_acquired: std::sync::atomic::AtomicBool,
 }
 
 #[cfg(any(test, feature = "test-util"))]
@@ -4651,6 +4667,7 @@ impl PutObjectCommitBarrier {
             arrived: tokio::sync::Notify::new(),
             release: tokio::sync::Notify::new(),
             namespace_pending: tokio::sync::Notify::new(),
+            namespace_acquired: std::sync::atomic::AtomicBool::new(false),
         });
         let mut slot = PUT_OBJECT_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -4685,6 +4702,10 @@ impl PutObjectCommitBarrier {
             .await
             .expect("put object should wait for the namespace lock after leaving the commit barrier");
     }
+
+    pub fn namespace_acquired(&self) -> bool {
+        self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
+    }
 }
 
 #[cfg(any(test, feature = "test-util"))]
@@ -4741,6 +4762,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
     }
 }
 
+#[cfg(any(test, feature = "test-util"))]
+fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
+    let barrier = PUT_OBJECT_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("put object commit barrier mutex should not poison")
+        .iter()
+        .find(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
+        })
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
+    }
+}
+
 #[cfg(test)]
 struct DeleteObjectCommitBarrierState {
     bucket: String,
@@ -4750,7 +4787,7 @@ struct DeleteObjectCommitBarrierState {
 }
 
 #[cfg(test)]
-struct DeleteObjectCommitBarrier {
+pub(crate) struct DeleteObjectCommitBarrier {
     state: Arc<DeleteObjectCommitBarrierState>,
 }
 
@@ -4760,7 +4797,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
 
 #[cfg(test)]
 impl DeleteObjectCommitBarrier {
-    fn install(bucket: &str, object: &str) -> Self {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(DeleteObjectCommitBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
@@ -4776,13 +4813,13 @@ impl DeleteObjectCommitBarrier {
         Self { state }
     }
 
-    async fn wait_until_paused(&self) {
+    pub(crate) async fn wait_until_paused(&self) {
         tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
             .await
             .expect("delete object should reach the deterministic commit barrier");
     }
 
-    fn release(&self) {
+    pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
 }
@@ -5655,7 +5692,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         objects: Vec<ObjectToDelete>,
         opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
+    }
+
+    async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
             .delete_replication_config_snapshot
             .clone()
@@ -5745,7 +5793,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 *item = Some(Error::other(message.clone()));
                             }
                         }
-                        return (del_objects, del_errs);
+                        return (del_objects, del_errs, accounting);
                     }
                 },
             }
@@ -5792,6 +5840,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             let source_missing = gerr
                 .as_ref()
                 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
+            // Resolve accounting from the generation selected under this
+            // object's write lock. A request-layer pre-stat is only an
+            // optimization and cannot identify a concurrent overwrite.
+            let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
+                || dobj.synthetic_version_id
+                || set_disk_delete_creates_delete_marker(&check_opts)
+                || goi.delete_marker
+            {
+                (None, None, false)
+            } else {
+                (
+                    quota_object_size(&goi).ok(),
+                    goi.version_id.filter(|version_id| !version_id.is_nil()),
+                    (dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
+                )
+            };
             // Normalize both sides before comparing. `goi.version_id` is the
             // client-facing identity, where `from_file_info` synthesizes
             // `Some(Uuid::nil())` for a null version on a versioned or
@@ -5877,6 +5941,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             if dobj.version_id.is_none() && (version_suspended || versioned) {
                 vr.mod_time = Some(OffsetDateTime::now_utc());
                 vr.deleted = true;
+                vr.mark_deleted = true;
                 if versioned {
                     vr.version_id = Some(Uuid::new_v4());
                 }
@@ -5920,7 +5985,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     },
                     replication_state: vr.replication_state_internal.clone(),
                     ..Default::default()
-                }
+                };
+                accounting[i] = Some(DeleteAccounting {
+                    size: accounting_size,
+                    version_id: accounting_version_id,
+                    removed_current_object,
+                });
             }
 
             // Only add to vers_map if we hold the lock
@@ -5966,7 +6036,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     });
                 }
             }
-            return (del_objects, del_errs);
+            return (del_objects, del_errs, accounting);
         }
 
         let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6204,7 +6274,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        (del_objects, del_errs)
+        // An accounting identity is actionable only when the delete result is
+        // successful. Never let a failed commit (including a partial quorum
+        // failure) reach the request-layer fast delta path.
+        for (index, err) in del_errs.iter().enumerate() {
+            if err.is_some() {
+                accounting[index] = None;
+            }
+        }
+
+        (del_objects, del_errs, accounting)
     }
 
     #[tracing::instrument(skip(self))]
@@ -6533,6 +6612,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
         obj_info.size = goi.size;
+        // Keep the committed source metadata on the internal delete result so
+        // the request layer can derive canonical accounting for this exact
+        // generation. Delete responses do not expose these fields.
+        obj_info.actual_size = goi.actual_size;
+        obj_info.user_defined = Arc::clone(&goi.user_defined);
+        obj_info.parts = Arc::clone(&goi.parts);
         obj_info.user_tags = Arc::clone(&goi.user_tags);
         self.invalidate_get_object_metadata_cache(bucket, object).await;
         Ok(obj_info)
@@ -7824,6 +7909,113 @@ mod replication_quota_safety_tests {
         assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
     }
 
+    #[tokio::test]
+    async fn delete_returns_canonical_compressed_accounting_size() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "compressed-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        set_disks
+            .put_object(
+                bucket,
+                "object",
+                &mut reader,
+                &ObjectOptions {
+                    user_defined,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                ObjectOptions {
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
+        assert!(deleted[0].found, "the committed object must be reported as found");
+        assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
+        assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
+    }
+
+    #[tokio::test]
+    async fn suspended_delete_marker_does_not_return_body_accounting() {
+        let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "suspended-delete-accounting";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut user_defined = HashMap::new();
+        insert_str(
+            &mut user_defined,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
+                .expect("compressed fixture reader should be valid"),
+        );
+        let suspended_opts = ObjectOptions {
+            version_suspended: true,
+            delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
+                s3s::dto::VersioningConfiguration {
+                    status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
+                    ..Default::default()
+                },
+                None,
+            ))),
+            user_defined,
+            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
+            ..Default::default()
+        };
+        set_disks
+            .put_object(bucket, "object", &mut reader, &suspended_opts)
+            .await
+            .expect("compressed object should be written");
+
+        let (deleted, errors, accounting) = set_disks
+            .delete_objects_with_accounting(
+                bucket,
+                vec![ObjectToDelete {
+                    object_name: "object".to_string(),
+                    ..Default::default()
+                }],
+                suspended_opts,
+            )
+            .await;
+        assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
+        assert!(deleted[0].delete_marker);
+        assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
+    }
+
     #[tokio::test]
     async fn direct_put_cannot_persist_a_tiny_logical_size() {
         let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
@@ -11636,6 +11828,7 @@ mod transition_upload_integrity_tests {
                 crate::data_movement::SourceCleanupBucketFence {
                     expected_incarnation_id: None,
                     lifecycle_guard: Some(&bucket_guard),
+                    ..Default::default()
                 },
                 "test_data_movement",
             )

crates/ecstore/src/set_disk/read.rs

@@ -259,10 +259,33 @@ impl SetDisks {
         read_data: bool,
         caller_allows_early_stop: bool,
     ) -> Result<GetObjectFileInfo> {
-        self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
+        self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
             .await
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
+    #[hotpath::measure(impl_type = "SetDisks")]
+    pub(super) async fn get_object_fileinfo_for_get_object_reader(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        read_data: bool,
+        caller_allows_early_stop: bool,
+    ) -> Result<GetObjectFileInfo> {
+        let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
+            && crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
+        self.get_object_fileinfo_gated_inner(
+            bucket,
+            object,
+            opts,
+            read_data,
+            caller_allows_early_stop,
+            allow_read_version_coalescing,
+        )
+        .await
+    }
+
     /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
     /// quorum fanout. Read-before-write callers (object tagging) must use this:
     /// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
         opts: &ObjectOptions,
         read_data: bool,
         allow_early_stop: bool,
+    ) -> Result<GetObjectFileInfo> {
+        self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
+            .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn get_object_fileinfo_gated_inner(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        read_data: bool,
+        allow_early_stop: bool,
+        allow_read_version_coalescing: bool,
     ) -> Result<GetObjectFileInfo> {
         let vid = opts.version_id.clone().unwrap_or_default();
         let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
         // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
         // core/io_primitives.rs); unsafe requests and callers that opt out
         // (allow_early_stop=false) fall back to full-wait.
-        let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
+        let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
-            &disks,
+            Self::read_all_fileinfo_observed_for_get_object(
-            "",
+                &disks,
-            bucket,
+                "",
-            object,
+                bucket,
-            vid.as_str(),
+                object,
-            read_data,
+                vid.as_str(),
-            false,
+                read_data,
-            opts.incl_free_versions,
+                opts.incl_free_versions,
-            allow_early_stop,
+                allow_early_stop,
-            self.default_parity_count,
+                self.default_parity_count,
-        )
+            )
-        .await?;
+            .await?
+        } else {
+            Self::read_all_fileinfo_observed(
+                &disks,
+                "",
+                bucket,
+                object,
+                vid.as_str(),
+                read_data,
+                false,
+                opts.incl_free_versions,
+                allow_early_stop,
+                self.default_parity_count,
+            )
+            .await?
+        };
         let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
             GET_OBJECT_PATH_INTERNAL_META
         } else {

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -62,8 +62,8 @@ pub(crate) mod object {
     use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
     use crate::storage_api_contracts::range::HTTPRangeSpec;
     pub(crate) use rustfs_storage_api::{
-        DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
+        DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
-        ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
+        ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
     };
 
     pub(crate) trait EcstoreObjectIO:

crates/ecstore/src/store/heal.rs

@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use super::*;
+use crate::core::pools::POOL_META_NAME;
+use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::heal::HealOperations as _;
+use crate::storage_api_contracts::namespace::NamespaceLocking as _;
+use rustfs_lock::NamespaceLockGuard;
 use tracing::trace;
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+#[derive(Debug, Clone, Copy)]
+enum HealFormatPoolSkip {
+    Completed,
+    Retryable,
+}
+
+fn classify_heal_format_pool(
+    pool_idx: usize,
+    pool_cmd_line: &str,
+    pool_meta: &PoolMeta,
+    rebalance_meta: Option<&RebalanceMeta>,
+) -> Option<HealFormatPoolSkip> {
+    let Some(pool) = pool_meta.pools.get(pool_idx) else {
+        return Some(HealFormatPoolSkip::Retryable);
+    };
+
+    if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
+        return Some(HealFormatPoolSkip::Retryable);
+    }
+
+    if let Some(decommission) = pool.decommission.as_ref() {
+        if decommission.complete {
+            return Some(HealFormatPoolSkip::Completed);
+        }
+        if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    if let Some(meta) = rebalance_meta {
+        let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
+            return Some(HealFormatPoolSkip::Retryable);
+        };
+        if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
+            return Some(HealFormatPoolSkip::Retryable);
+        }
+    }
+
+    None
+}
+
+fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
+    match skip {
+        HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
+        HealFormatPoolSkip::Retryable => StorageError::SlowDown,
+    }
+}
+
+fn heal_format_fence_lost_error() -> Error {
+    StorageError::SlowDown
+}
+
 impl ECStore {
+    async fn acquire_heal_format_fence(
+        &self,
+    ) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option<RebalanceMeta>)> {
+        let metadata_pool = self
+            .pools
+            .first()
+            .cloned()
+            .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
+
+        // Metadata fence order is part of the decommission/rebalance protocol:
+        // pool.bin must always be acquired before rebalance.bin.
+        let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
+        let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
+        let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut pool_meta = PoolMeta::default();
+        pool_meta.load_no_lock(metadata_pool.clone()).await?;
+        if pool_meta.pools.len() != self.pools.len()
+            || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
+                pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
+            })
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        let mut rebalance_meta = RebalanceMeta::new();
+        let rebalance_meta = match rebalance_meta
+            .load_with_opts(
+                metadata_pool,
+                ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+        {
+            Ok(()) => Some(rebalance_meta),
+            Err(Error::ConfigNotFound) => None,
+            Err(err) => return Err(err),
+        };
+
+        if rebalance_meta
+            .as_ref()
+            .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
+        {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+            return Err(heal_format_fence_lost_error());
+        }
+
+        Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta))
+    }
+
     fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
         match opts.pool {
             Some(pool_idx) => Ok(vec![
@@ -52,9 +169,26 @@ impl ECStore {
         };
 
         let mut count_no_heal = 0;
+        let mut count_completed = 0;
         let mut first_error = None;
-        for pool in self.pools.iter() {
+        for (pool_idx, pool) in self.pools.iter().enumerate() {
-            let (mut result, err) = pool.heal_format(dry_run).await?;
+            let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
+            if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
+            {
+                if matches!(skip, HealFormatPoolSkip::Completed) {
+                    count_completed += 1;
+                } else {
+                    first_error.get_or_insert(heal_format_pool_skip_error(skip));
+                }
+                continue;
+            }
+
+            let fence_lost = || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost();
+            let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
             if let Some(err) = err {
                 match err {
                     StorageError::NoHealRequired => {
@@ -69,11 +203,18 @@ impl ECStore {
             r.set_count += result.set_count;
             r.before.drives.append(&mut result.before.drives);
             r.after.drives.append(&mut result.after.drives);
+
+            // A lease can be lost after the final write; fail closed before
+            // reporting the pool as successfully healed.
+            if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
+                first_error.get_or_insert(heal_format_fence_lost_error());
+                break;
+            }
         }
         if let Some(err) = first_error {
             return Ok((r, Some(err)));
         }
-        if count_no_heal == self.pools.len() {
+        if count_no_heal + count_completed == self.pools.len() {
             info!(
                 event = EVENT_HEAL_FORMAT_COMPLETED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -297,10 +438,17 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::bucket::metadata_sys;
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
-    use crate::disk::{DiskOption, format::FormatV3, new_disk};
+    use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
-    use crate::layout::endpoints::{Endpoints, PoolEndpoints};
+    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+    use crate::runtime::instance::InstanceContext;
+    use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
+    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
     use crate::store::init_format::{load_format_erasure, save_format_file};
+    use crate::store::init_local_disks_with_instance_ctx;
+    use tokio_util::sync::CancellationToken;
 
     async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
         let format = FormatV3::new(1, 1);
@@ -347,6 +495,209 @@ mod tests {
         }
     }
 
+    fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
+        PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(info),
+            }],
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn heal_format_pool_state_barriers_are_classified() {
+        let active = pool_meta_with_decommission(PoolDecommissionInfo {
+            start_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &active, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        for info in [
+            PoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            },
+            PoolDecommissionInfo {
+                canceled: true,
+                ..Default::default()
+            },
+        ] {
+            assert!(matches!(
+                classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
+                Some(HealFormatPoolSkip::Retryable)
+            ));
+        }
+
+        let completed = pool_meta_with_decommission(PoolDecommissionInfo {
+            complete: true,
+            ..Default::default()
+        });
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &completed, None),
+            Some(HealFormatPoolSkip::Completed)
+        ));
+    }
+
+    #[test]
+    fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
+        let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        let rebalance = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopping = RebalanceMeta {
+            pool_stats: vec![RebalanceStats {
+                info: RebalanceInfo {
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let identity_without_decommission = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: None,
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "", &identity_meta, None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+
+        let stopped = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Stopped,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
+
+        let stopping_after_stop = RebalanceMeta {
+            stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
+            pool_stats: vec![RebalanceStats {
+                participating: true,
+                info: RebalanceInfo {
+                    status: RebalStatus::Started,
+                    stopping: true,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        assert!(matches!(
+            classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
+            Some(HealFormatPoolSkip::Retryable)
+        ));
+    }
+
+    #[test]
+    fn skipped_heal_format_pool_is_never_reported_as_success() {
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
+            StorageError::SlowDown
+        ));
+        assert!(matches!(
+            heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
+            StorageError::NoHealRequired
+        ));
+    }
+
+    async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
+        let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
+        let mut pool_endpoints = Vec::new();
+        for pool_index in 0..2 {
+            let mut endpoints = Vec::new();
+            for disk_index in 0..4 {
+                let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
+                tokio::fs::create_dir_all(&disk_path)
+                    .await
+                    .expect("multi-pool heal test disk should be created");
+                let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
+                    .expect("test endpoint should parse");
+                endpoint.set_pool_index(pool_index);
+                endpoint.set_set_index(0);
+                endpoint.set_disk_index(disk_index);
+                endpoints.push(endpoint);
+            }
+            pool_endpoints.push(PoolEndpoints {
+                legacy: false,
+                set_count: 1,
+                drives_per_set: 4,
+                endpoints: Endpoints::from(endpoints),
+                cmd_line: format!("heal-owner-pool-{pool_index}"),
+                platform: "test".to_string(),
+            });
+        }
+
+        let endpoint_pools = EndpointServerPools::from(pool_endpoints);
+        let instance_ctx = Arc::new(InstanceContext::new());
+        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+            .await
+            .expect("multi-pool local disks should initialize");
+        let shutdown = CancellationToken::new();
+        let store = ECStore::new_with_instance_ctx(
+            "127.0.0.1:0".parse().expect("test address should parse"),
+            endpoint_pools,
+            shutdown.clone(),
+            instance_ctx,
+        )
+        .await
+        .expect("multi-pool test store should initialize");
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        (temp_dir, store, shutdown)
+    }
+
     #[tokio::test]
     async fn heal_object_pool_scope_selects_only_requested_pool() {
         let store = minimal_heal_store().await;
@@ -506,6 +857,229 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn unscoped_heal_object_suspended_owner_semantics() {
+        let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
+        let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
+        let active_object = "active-owner";
+        let suspended_only_object = "suspended-only";
+        let duplicate_object = "duplicate-owner";
+        let marker_object = "marker-owner";
+        let quorum_object = "quorum-owner";
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created in all pools");
+
+        let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
+        store.pools[0]
+            .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
+            .await
+            .expect("active owner object should be written");
+        let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
+        let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
+        missing_active_disk
+            .delete(
+                &bucket,
+                active_object,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("active owner shard should be removed for repair");
+        assert!(
+            missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
+            "the active owner fixture must start with one missing metadata copy"
+        );
+
+        let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
+        store.pools[1]
+            .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
+            .await
+            .expect("suspended owner object should be written");
+        for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
+            let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
+            store.pools[pool_index]
+                .put_object(
+                    &bucket,
+                    duplicate_object,
+                    &mut duplicate_reader,
+                    &ObjectOptions {
+                        mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("duplicate owner object should be written");
+        }
+        let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
+            .clone()
+            .expect("duplicate active owner disk should be online");
+        duplicate_missing_disk
+            .delete(
+                &bucket,
+                duplicate_object,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("duplicate active owner shard should be removed for repair");
+        let history_version = Uuid::new_v4();
+        let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
+        store.pools[0]
+            .put_object(
+                &bucket,
+                marker_object,
+                &mut history_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(history_version.to_string()),
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("versioned marker history should be written");
+        store.pools[0]
+            .delete_object(
+                &bucket,
+                marker_object,
+                ObjectOptions {
+                    versioned: true,
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("delete marker should be written");
+        let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
+        store.pools[0]
+            .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
+            .await
+            .expect("quorum boundary object should be written");
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            let mut next = PoolMeta::new(&store.pools, &pool_meta);
+            next.pools[1].decommission = Some(PoolDecommissionInfo {
+                start_time: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            });
+            *pool_meta = next;
+        }
+
+        let (_, duplicate_owner) = store
+            .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
+            .await
+            .expect("duplicate owner should resolve");
+        assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
+        let (_, active_duplicate_owner) = store
+            .get_latest_object_info_with_idx(
+                &bucket,
+                duplicate_object,
+                &ObjectOptions {
+                    skip_decommissioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("active duplicate owner should resolve");
+        assert_eq!(
+            active_duplicate_owner, 0,
+            "suspended duplicate must be excluded from active owner selection"
+        );
+        let (duplicate_result, duplicate_err) = store
+            .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
+            .await
+            .expect("duplicate owner heal should complete through the production path");
+        assert_eq!(duplicate_result.object, duplicate_object);
+        assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
+        assert!(
+            duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
+            "production heal must repair the active duplicate owner rather than the suspended owner"
+        );
+        let (marker_info, marker_owner) = store
+            .get_latest_object_info_with_idx(
+                &bucket,
+                marker_object,
+                &ObjectOptions {
+                    skip_decommissioned: true,
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("latest delete marker should resolve");
+        assert_eq!(marker_owner, 0);
+        assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
+
+        let (active_result, active_err) = store
+            .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
+            .await
+            .expect("unscoped active-owner heal should complete");
+        assert_eq!(active_result.object, active_object);
+        assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
+        assert!(
+            missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
+            "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
+        );
+        assert!(
+            store.pools[1]
+                .get_object_info(&bucket, active_object, &ObjectOptions::default())
+                .await
+                .is_err(),
+            "the suspended pool must not be written for an active-owner object"
+        );
+
+        let (suspended_result, suspended_err) = store
+            .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
+            .await
+            .expect("unscoped suspended-only heal should return a terminal result");
+        assert!(suspended_result.object.is_empty());
+        assert!(matches!(suspended_err, Some(Error::FileNotFound)));
+        assert!(
+            store.pools[1]
+                .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
+                .await
+                .is_ok(),
+            "suspended-only data must remain untouched when unscoped heal reports absent"
+        );
+
+        let (_, explicit_err) = store
+            .handle_heal_object(
+                &bucket,
+                suspended_only_object,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("explicit suspended-owner heal should return a mapped error");
+        assert!(matches!(explicit_err, Some(Error::SlowDown)));
+
+        let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
+        let surviving_quorum_disk = original_quorum_disks[3].clone();
+        *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
+        let (_, quorum_err) = store
+            .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
+            .await
+            .expect("quorum boundary heal should return a mapped result");
+        *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
+        assert!(
+            matches!(quorum_err, Some(Error::ErasureReadQuorum)),
+            "quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
+        );
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     async fn handle_heal_format_continues_after_a_pool_error() {
         let canonical_format = FormatV3::new(1, 3);
@@ -615,6 +1189,18 @@ mod tests {
             bucket_fence_registry: std::sync::Arc::default(),
         };
 
+        let err = store
+            .handle_heal_format(false)
+            .await
+            .expect_err("missing pool metadata must fail closed before format writes");
+        assert!(matches!(err, StorageError::SlowDown));
+
+        let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        pool_meta
+            .save(store.pools.clone())
+            .await
+            .expect("pool metadata should be persisted before format heal");
+
         let (result, err) = store
             .handle_heal_format(false)
             .await
@@ -628,5 +1214,22 @@ mod tests {
             .await
             .expect("the later pool should be healed despite the first pool error");
         assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
+
+        let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
+        for status in &mut completed_meta.pools {
+            status.decommission = Some(PoolDecommissionInfo {
+                complete: true,
+                ..Default::default()
+            });
+        }
+        completed_meta
+            .save(store.pools.clone())
+            .await
+            .expect("completed pool metadata should be persisted");
+        let (_, err) = store
+            .handle_heal_format(false)
+            .await
+            .expect("completed pools should be reported as a no-op");
+        assert!(matches!(err, Some(StorageError::NoHealRequired)));
     }
 }

crates/ecstore/src/store/mod.rs

@@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args;
 use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
 use crate::cluster::rpc::{RemoteClient, S3PeerSys};
 use crate::config::storageclass;
-use crate::core::pools::PoolMeta;
+use crate::core::pools::{DecommissionCanceler, PoolMeta};
 use crate::disk::endpoint::{Endpoint, EndpointType};
 use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
 use crate::error::{Error, Result};
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-pub(crate) use object::ObjectLockDiagGuard;
+pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
 pub use object::{
     PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
     SnapshotConsistencyError,
@@ -176,7 +176,7 @@ pub struct ECStore {
     // pub local_disks: Vec<DiskStore>,
     pub pool_meta: RwLock<PoolMeta>,
     pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
-    pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
+    pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
     /// Serializes rebalance/decommission start transitions.
     ///
     /// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,

crates/ecstore/src/store/multipart.rs

@@ -400,7 +400,7 @@ impl ECStore {
         object: &str,
         opts: &ObjectOptions,
     ) -> Result<MultipartUploadResult> {
-        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
+        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
             .await
             .map(|(res, _, _)| res)
     }
@@ -410,20 +410,22 @@ impl ECStore {
         bucket: &str,
         object: &str,
         opts: &ObjectOptions,
+        mutation_fence: Option<&ObjectLockDiagGuard>,
     ) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
         check_new_multipart_args(bucket, object)?;
-        let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
-        let opts = &opts;
 
         if self.single_pool() {
+            self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
+                .await;
             return self.pools[0]
-                .new_multipart_upload(bucket, object, opts)
+                .new_multipart_upload(bucket, object, &opts)
                 .await
                 .map(|res| (res, 0, opts.expected_bucket_incarnation_id));
         }
 
         if opts.data_movement && opts.version_id.is_some() {
-            let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
+            let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
             if idx == opts.src_pool_idx {
                 return Err(StorageError::DataMovementOverwriteErr(
                     bucket.to_owned(),
@@ -431,7 +433,9 @@ impl ECStore {
                     opts.version_id.clone().unwrap_or_default(),
                 ));
             }
-            let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+            self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                .await;
+            let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
             return Ok((res, idx, opts.expected_bucket_incarnation_id));
         }
 
@@ -454,7 +458,9 @@ impl ECStore {
             .await?;
 
             if !res.uploads.is_empty() {
-                let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+                self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                    .await;
+                let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
                 return Ok((res, idx, opts.expected_bucket_incarnation_id));
             }
         }
@@ -467,7 +473,9 @@ impl ECStore {
             ));
         }
 
-        let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+        self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+            .await;
+        let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
         Ok((res, idx, opts.expected_bucket_incarnation_id))
     }
 
@@ -704,13 +712,14 @@ impl ECStore {
 
     pub(crate) async fn complete_multipart_upload_for_data_movement(
         self: Arc<Self>,
-        target_pool_idx: usize,
+        target: (usize, Option<&ObjectLockDiagGuard>),
         bucket: &str,
         object: &str,
         upload_id: &str,
         uploaded_parts: Vec<CompletePart>,
         opts: &ObjectOptions,
     ) -> Result<ObjectInfo> {
+        let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
         if !opts.data_movement {
             return Err(Error::other("targeted multipart completion requires data_movement options"));
@@ -739,6 +748,8 @@ impl ECStore {
             snapshot.add_lock_fences(&mut opts);
             opts.object_lock_config_snapshot = Some(snapshot);
         }
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+            .await;
         #[cfg(test)]
         pause_data_movement_multipart_before_selected_completion(bucket).await;
         let pool = self

crates/ecstore/src/store/rebalance.rs

@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
     use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
     use crate::disk::error::DiskError;
     use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
         }
     }
 
+    fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
+        ObjectInfo {
+            version_id: Some(version_id),
+            etag,
+            ..object_info_with_mod_time(unix_ts, delete_marker)
+        }
+    }
+
     #[test]
     fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
         let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
     }
 
     #[test]
-    fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
+    fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
         let candidates = vec![
             LatestObjectInfoCandidate {
                 info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
         assert_eq!(idx, 1);
     }
 
+    #[test]
+    fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 2,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 7,
+                err: None,
+            },
+        ];
+
+        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect("equivalent replicas must resolve deterministically");
+
+        assert_eq!(idx, 7);
+        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("divergent version ids must not silently resolve to the higher pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("divergent etags must not silently resolve to the higher pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 1,
+                err: None,
+            },
+        ];
+
+        let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect_err("a delete marker tied with a live version must not be masked by the pool index");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
+        let err = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(left),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(right),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect_err("equal-time identity divergence must fail closed");
+
+        assert_eq!(err, Error::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        let mut data_dir = base.clone();
+        data_dir.data_dir = Some(Uuid::from_u128(2));
+        assert_equal_time_identity_conflict(base.clone(), data_dir);
+
+        let mut size = base.clone();
+        size.size = 1;
+        assert_equal_time_identity_conflict(base.clone(), size);
+
+        let mut actual_size = base.clone();
+        actual_size.actual_size = 1;
+        assert_equal_time_identity_conflict(base.clone(), actual_size);
+
+        let mut checksum = base.clone();
+        checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
+        assert_equal_time_identity_conflict(base.clone(), checksum);
+
+        let mut parts = base.clone();
+        parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+            etag: "part-etag".to_string(),
+            number: 1,
+            size: 1,
+            ..Default::default()
+        }]);
+        assert_equal_time_identity_conflict(base.clone(), parts);
+
+        let mut transition = base;
+        transition.transitioned_object.tier = "tier-a".to_string();
+        assert_equal_time_identity_conflict(
+            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
+            transition,
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut rustfs_alias = base.clone();
+        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+        let mut minio_alias = base.clone();
+        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "X-MINIO-INTERNAL-COMPRESSION".to_string(),
+            "zstd".to_string(),
+        )]));
+
+        let (_, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(rustfs_alias),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(minio_alias),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("same-value internal aliases should resolve");
+        assert_eq!(idx, 1);
+
+        let mut dual_alias = base.clone();
+        dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
+            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
+            ("x-minio-internal-compression".to_string(), "zstd".to_string()),
+        ]));
+        let mut single_alias = base;
+        single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+
+        let (_, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(dual_alias),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(single_alias),
+                    idx: 1,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("dual-key and single-key internal metadata should resolve");
+        assert_eq!(idx, 1);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut rustfs_alias = base.clone();
+        rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-rustfs-internal-compression".to_string(),
+            "zstd".to_string(),
+        )]));
+        let mut minio_alias = base;
+        minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+            "x-minio-internal-compression".to_string(),
+            "snappy".to_string(),
+        )]));
+
+        assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
+        for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
+            let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+            let mut rustfs_alias = base.clone();
+            rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!(
+                    "X-RUSTFS-INTERNAL-{}{suffix}",
+                    suffix_prefix.to_uppercase(),
+                    suffix = "arn:aws:s3:::Bucket"
+                ),
+                "value".to_string(),
+            )]));
+            let mut minio_alias = base.clone();
+            minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
+                "value".to_string(),
+            )]));
+
+            let (_, idx) = resolve_latest_object_info_candidates(
+                vec![
+                    LatestObjectInfoCandidate {
+                        info: Some(rustfs_alias.clone()),
+                        idx: 0,
+                        err: None,
+                    },
+                    LatestObjectInfoCandidate {
+                        info: Some(minio_alias),
+                        idx: 1,
+                        err: None,
+                    },
+                ],
+                "bucket",
+                "object",
+                &ObjectOptions::default(),
+            )
+            .expect("dynamic internal aliases with the same target should resolve");
+            assert_eq!(idx, 1);
+
+            let mut different_target_case = base;
+            different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
+                format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
+                "value".to_string(),
+            )]));
+
+            assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
+        }
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        let mut first = base.clone();
+        first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
+            ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
+            ("x-minio-internal-compression".to_string(), "snappy".to_string()),
+        ]));
+        let mut second = base;
+        second.user_defined = first.user_defined.clone();
+
+        assert_equal_time_identity_conflict(first, second);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
+        let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        let mut replication = base.clone();
+        replication.replication_status_internal = Some("PENDING".to_string());
+        replication.replication_status = ReplicationStatusType::Pending;
+        assert_equal_time_identity_conflict(base.clone(), replication);
+
+        let mut purge = base.clone();
+        purge.version_purge_status_internal = Some("PENDING".to_string());
+        purge.version_purge_status = VersionPurgeStatusType::Pending;
+        assert_equal_time_identity_conflict(base.clone(), purge);
+
+        let mut decision = base;
+        decision.replication_decision = "replicate".to_string();
+        assert_equal_time_identity_conflict(
+            object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
+            decision,
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
+        let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+        without_mod_time.mod_time = None;
+        let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
+
+        assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
+        let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
+        let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
+        older.data_dir = Some(Uuid::from_u128(2));
+
+        let (info, idx) = resolve_latest_object_info_candidates(
+            vec![
+                LatestObjectInfoCandidate {
+                    info: Some(latest),
+                    idx: 0,
+                    err: None,
+                },
+                LatestObjectInfoCandidate {
+                    info: Some(older),
+                    idx: 9,
+                    err: None,
+                },
+            ],
+            "bucket",
+            "object",
+            &ObjectOptions::default(),
+        )
+        .expect("older identity divergence must not affect the latest candidate");
+
+        assert_eq!(idx, 0);
+        assert_eq!(
+            info.mod_time,
+            Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
+        );
+    }
+
+    #[test]
+    fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
+        let candidates = vec![
+            LatestObjectInfoCandidate {
+                info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
+                idx: 0,
+                err: None,
+            },
+            LatestObjectInfoCandidate {
+                info: None,
+                idx: 1,
+                err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+            },
+        ];
+
+        let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
+            .expect("not-found pools must not block resolution of found candidates");
+
+        assert_eq!(idx, 0);
+        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
+    }
+
     #[test]
     fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
         let err = resolve_latest_object_info_candidates(

crates/ecstore/src/store/rebalance/support.rs

@@ -12,10 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cmp::Ordering;
+use std::collections::HashMap;
 
 use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
 use crate::object_api::{ObjectInfo, ObjectOptions};
+use rustfs_utils::http::metadata_compat::{
+    SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
+    strip_internal_prefix_preserving_case,
+};
 use rustfs_utils::path::decode_dir_object;
 use time::OffsetDateTime;
 
@@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
     object: &str,
 ) -> Result<ObjectInfo> {
     result.map_err(|err| {
-        if err == Error::PreconditionFailed {
+        if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
             err
         } else {
             Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
@@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
 }
 
 fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
-    if err == Error::PreconditionFailed {
+    if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
         err
     } else {
         Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
     ))
 }
 
+fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
+    candidate
+        .info
+        .as_ref()
+        .map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
+}
+
+fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    left.transition_version_state == right.transition_version_state
+        && left.transitioned_object.name == right.transitioned_object.name
+        && left.transitioned_object.version_id == right.transitioned_object.version_id
+        && left.transitioned_object.tier == right.transitioned_object.tier
+        && left.transitioned_object.free_version == right.transitioned_object.free_version
+        && left.transitioned_object.status == right.transitioned_object.status
+}
+
+#[derive(PartialEq, Eq)]
+struct LatestUserDefinedIdentity {
+    internal: HashMap<String, String>,
+    other: HashMap<String, String>,
+}
+
+fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
+    let suffix = strip_internal_prefix_preserving_case(key)?;
+
+    for dynamic_prefix in [
+        SUFFIX_REPLICATION_RESET_ARN_PREFIX,
+        SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
+    ] {
+        let prefix_len = dynamic_prefix.len();
+        if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
+            && prefix.eq_ignore_ascii_case(dynamic_prefix)
+        {
+            return Some(format!("{dynamic_prefix}{remainder}"));
+        }
+    }
+
+    Some(suffix.to_lowercase())
+}
+
+fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
+    let mut identity = LatestUserDefinedIdentity {
+        internal: HashMap::with_capacity(user_defined.len()),
+        other: HashMap::with_capacity(user_defined.len()),
+    };
+
+    for (key, value) in user_defined {
+        if let Some(suffix) = normalize_internal_identity_suffix(key) {
+            if identity
+                .internal
+                .insert(suffix, value.clone())
+                .is_some_and(|previous| previous != *value)
+            {
+                return None;
+            }
+        } else {
+            identity.other.insert(key.clone(), value.clone());
+        }
+    }
+
+    Some(identity)
+}
+
+fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    match (
+        normalize_user_defined_identity(&left.user_defined),
+        normalize_user_defined_identity(&right.user_defined),
+    ) {
+        (Some(left), Some(right)) => left == right,
+        _ => false,
+    }
+}
+
+/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
+/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
+/// differ for the same object version while the selected winner still carries
+/// the chosen pool's layout. `put_object_reader` is also intentionally
+/// excluded because it is a transient request handle that `ObjectInfo::clone`
+/// drops. Every other ObjectInfo field is part of the production-visible
+/// identity and must agree before the pool index can provide a deterministic
+/// tie-break.
+fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
+    left.bucket == right.bucket
+        && left.name == right.name
+        && left.storage_class == right.storage_class
+        && left.mod_time == right.mod_time
+        && left.size == right.size
+        && left.actual_size == right.actual_size
+        && left.is_dir == right.is_dir
+        && same_user_defined_identity(left, right)
+        && left.user_tags == right.user_tags
+        && left.version_id == right.version_id
+        && left.data_dir == right.data_dir
+        && left.delete_marker == right.delete_marker
+        && same_transition_identity(left, right)
+        && left.restore_ongoing == right.restore_ongoing
+        && left.restore_expires == right.restore_expires
+        && left.parts == right.parts
+        && left.is_latest == right.is_latest
+        && left.content_type == right.content_type
+        && left.content_encoding == right.content_encoding
+        && left.expires == right.expires
+        && left.num_versions == right.num_versions
+        && left.successor_mod_time == right.successor_mod_time
+        && left.etag == right.etag
+        && left.inlined == right.inlined
+        && left.metadata_only == right.metadata_only
+        && left.version_only == right.version_only
+        && left.replication_status_internal == right.replication_status_internal
+        && left.replication_status == right.replication_status
+        && left.version_purge_status_internal == right.version_purge_status_internal
+        && left.version_purge_status == right.version_purge_status
+        && left.replication_decision == right.replication_decision
+        && left.checksum == right.checksum
+}
+
 pub(super) fn resolve_latest_object_info_candidates(
-    mut candidates: Vec<LatestObjectInfoCandidate>,
+    candidates: Vec<LatestObjectInfoCandidate>,
     bucket: &str,
     object: &str,
     opts: &ObjectOptions,
 ) -> Result<(ObjectInfo, usize)> {
-    candidates.sort_by(|a, b| {
+    let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
-        let a_mod = if let Some(info) = &a.info {
+
-            info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
+    if let Some(latest_mod_time) = latest_mod_time {
-        } else {
+        let mut latest_candidates = candidates
-            OffsetDateTime::UNIX_EPOCH
+            .into_iter()
+            .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
+            .collect::<Vec<_>>();
+
+        latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
+
+        let Some(winner) = latest_candidates.first() else {
+            return Err(Error::ErasureReadQuorum);
+        };
+        let Some(winner_info) = winner.info.as_ref() else {
+            return Err(Error::ErasureReadQuorum);
         };
 
-        let b_mod = if let Some(info) = &b.info {
+        if latest_candidates.iter().skip(1).any(|candidate| {
-            info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
+            candidate
-        } else {
+                .info
-            OffsetDateTime::UNIX_EPOCH
+                .as_ref()
-        };
+                .is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
-
+        }) {
-        if a_mod == b_mod {
+            return Err(Error::ErasureReadQuorum);
-            return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
         }
 
-        b_mod.cmp(&a_mod)
+        return Ok((winner_info.clone(), winner.idx));
-    });
+    }
 
     for candidate in candidates {
-        if let Some(info) = candidate.info {
-            return Ok((info, candidate.idx));
-        }
-
         if let Some(err) = candidate.err
             && !is_err_object_not_found(&err)
             && !is_err_version_not_found(&err)
@@ -191,6 +316,18 @@ mod tests {
         assert_eq!(err, Error::PreconditionFailed);
     }
 
+    #[test]
+    fn rebalance_delete_result_preserves_prefix_access_denied() {
+        let err = resolve_rebalance_delete_from_all_pools_result(
+            Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
+            "bucket",
+            "object",
+        )
+        .expect_err("prefix access denial should remain structured");
+
+        assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
+    }
+
     #[test]
     fn rebalance_delete_pool_result_preserves_precondition_failed() {
         let err = resolve_rebalance_delete_from_all_pools_results(
@@ -205,4 +342,19 @@ mod tests {
 
         assert_eq!(err, Error::PreconditionFailed);
     }
+
+    #[test]
+    fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![RebalanceDeletePoolResult {
+                pool_idx: 0,
+                result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
+            }],
+            "bucket",
+            "object",
+        )
+        .expect_err("prefix access denial should remain structured");
+
+        assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
+    }
 }

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -231,6 +231,10 @@ impl HealTask {
                             "Heal erasure set format repair skipped because no format heal was required"
                         );
                     } else {
+                        let error = e;
+                        if error.is_recoverable_heal() {
+                            return Err(error);
+                        }
                         error!(
                             target: "rustfs::heal::task",
                             event = EVENT_HEAL_ERASURE_SET_RESULT,
@@ -239,7 +243,7 @@ impl HealTask {
                             task_id = %self.id,
                             set_disk_id,
                             result = "format_failed",
-                            error = %e,
+                            error = %error,
                             "Heal erasure set failed"
                         );
                         {
@@ -247,7 +251,7 @@ impl HealTask {
                             progress.update_progress(4, 4, 0, 0);
                         }
                         return Err(Error::TaskExecutionFailed {
-                            message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
+                            message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
                         });
                     }
                 } else {
@@ -284,6 +288,9 @@ impl HealTask {
             Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
             Err(e) => {
+                if e.is_recoverable_heal() {
+                    return Err(e);
+                }
                 error!(
                     target: "rustfs::heal::task",
                     event = EVENT_HEAL_ERASURE_SET_RESULT,

crates/heal/src/heal/task/tests.rs

@@ -547,6 +547,7 @@ struct MockStorage {
     heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
     heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
     format_no_heal_required: Mutex<bool>,
+    format_error: Mutex<Option<Error>>,
     global_format_calls: Mutex<u32>,
     replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
     replacement_targets_ready: Mutex<bool>,
@@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage {
 
     async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         *self.global_format_calls.lock().unwrap() += 1;
+        if let Some(error) = self.format_error.lock().unwrap().take() {
+            return Err(error);
+        }
         let no_heal_required = *self.format_no_heal_required.lock().unwrap();
         if no_heal_required {
             Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
@@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() {
     );
 }
 
+#[tokio::test]
+async fn erasure_set_format_slowdown_is_propagated() {
+    let storage = Arc::new(MockStorage {
+        format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::ErasureSet {
+            buckets: Vec::new(),
+            set_disk_id: "pool_0_set_0".to_string(),
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage);
+
+    let error = task
+        .execute()
+        .await
+        .expect_err("format SlowDown must remain recoverable for the task manager");
+
+    assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
+}
+
 #[tokio::test]
 async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/io-metrics/src/internode_metrics.rs

@@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
 pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
 pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
 pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
+pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
 
 const OPERATION_LABEL: &str = "operation";
 const BACKEND_LABEL: &str = "backend";

crates/madmin/src/metrics.rs

@@ -689,6 +689,14 @@ pub struct ScannerMetrics {
     pub cycle_max_objects: u64,
     #[serde(rename = "cycle_max_directories", default)]
     pub cycle_max_directories: u64,
+    #[serde(rename = "cycle_timeout_total", default)]
+    pub cycle_timeout_total: u64,
+    #[serde(rename = "cycle_recovery_required_total", default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(rename = "cycle_last_progress_age", default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(rename = "leader_lease_without_progress", default)]
+    pub leader_lease_without_progress: bool,
     #[serde(rename = "bitrot_cycle_enabled", default)]
     pub bitrot_cycle_enabled: bool,
     #[serde(rename = "bitrot_cycle_seconds", default)]
@@ -764,6 +772,8 @@ impl ScannerMetrics {
             self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
             self.cycle_max_objects = other.cycle_max_objects;
             self.cycle_max_directories = other.cycle_max_directories;
+            self.cycle_last_progress_age = other.cycle_last_progress_age;
+            self.leader_lease_without_progress = other.leader_lease_without_progress;
             self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
             self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
         }
@@ -857,6 +867,12 @@ impl ScannerMetrics {
             .saturating_add(other.last_cycle_replication_checks);
         self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
         self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
+        self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
+        self.cycle_recovery_required_total = self
+            .cycle_recovery_required_total
+            .saturating_add(other.cycle_recovery_required_total);
+        self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
+        self.leader_lease_without_progress |= other.leader_lease_without_progress;
         self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
         self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
         self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);

crates/policy/src/policy/policy.rs

@@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> {
 #[derive(Serialize, Deserialize, Clone, Default, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct BucketPolicy {
-    #[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
+    // RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
+    #[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
     pub id: ID,
     #[serde(rename = "Version")]
     pub version: String,
@@ -2786,7 +2787,7 @@ mod test {
         let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
 
         // Verify empty fields are omitted
-        assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
+        assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");
 
         let statement = &parsed["Statement"][0];
         assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
@@ -2809,6 +2810,43 @@ mod test {
         assert_eq!(statement["Principal"]["AWS"], "*");
     }
 
+    #[test]
+    fn test_bucket_policy_deserializes_legacy_id() {
+        let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
+
+        let policy: BucketPolicy =
+            serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
+        assert!(policy.id.is_empty());
+        policy.is_valid().expect("legacy bucket policy should remain valid");
+
+        let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
+            .expect("non-empty legacy ID should deserialize");
+        assert_eq!(policy.id.0, "legacy-policy");
+
+        let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
+        assert_eq!(serialized["Id"], "legacy-policy");
+        assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
+    }
+
+    #[test]
+    fn test_bucket_policy_legacy_id_alias_remains_strict() {
+        let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
+        let error =
+            serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
+        assert!(
+            error.to_string().contains("unknown field `Unexpected`"),
+            "unexpected deserialization error: {error}"
+        );
+
+        let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
+        let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
+            .expect_err("canonical and legacy ID fields should not be accepted together");
+        assert!(
+            error.to_string().contains("duplicate field `Id`"),
+            "unexpected deserialization error: {error}"
+        );
+    }
+
     #[test]
     fn test_existing_object_tag_condition_helpers() {
         let identity_policy = Policy::parse_config(

crates/replication/src/config.rs

@@ -60,7 +60,9 @@ pub const REPLICATION_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[
     "Destination.ReplicationTime",
 ];
 
-pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 1;
+// v2: disableProxy moved from unsupported to writable (per-target read-proxy
+// opt-out is accepted by set-remote-target and the `proxy` update op).
+pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 2;
 
 pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
     "sourcebucket",
@@ -83,9 +85,12 @@ pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
     // madmin default of 60s); the per-target health-check interval is not
     // yet applied — the heartbeat keeps its global env-configured interval.
     "healthCheckDuration",
+    // Per-target read-proxy opt-out, consumed by the proxy-target selector
+    // (contract v2; previously only importable via MinIO bucket-targets.json).
+    "disableProxy",
 ];
 
-pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["disableProxy", "edge", "edgeSyncBeforeExpiry"];
+pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["edge", "edgeSyncBeforeExpiry"];
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct ObjectOpts {

crates/scanner/src/data_usage_define.rs

@@ -125,6 +125,34 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
     }
 }
 
+/// Read only the object revision without materializing its body.
+pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
+    match store
+        .get_object_reader(
+            RUSTFS_META_BUCKET,
+            path,
+            None,
+            HeaderMap::new(),
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(reader) => reader
+            .object_info
+            .etag
+            .filter(|etag| !etag.is_empty())
+            .map(DataUsageCacheRevision::Etag)
+            .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
+        Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
+            Ok(DataUsageCacheRevision::Missing)
+        }
+        Err(err) => Err(err),
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct DataUsageCacheRevisions {
     main: DataUsageCacheRevision,
@@ -146,6 +174,11 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
 pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
 
+/// Durable companion object for a cycle-state object which cannot be decoded.
+/// The primary object is deliberately never replaced or deleted by recovery.
+pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock<String> =
+    LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str()));
+
 pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
 

crates/scanner/src/data_usage_define/persistence.rs

@@ -74,7 +74,7 @@ impl DataUsageCache {
         let loaded = Self::load_cache(store.clone(), name).await?;
         let backup = match loaded.backup_revision {
             Some(revision) => Some(revision),
-            None => match Self::revision_for_path(store, &backup_path).await {
+            None => match read_config_revision(store, &backup_path).await {
                 Ok(revision) => Some(revision),
                 Err(err) => {
                     counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
@@ -336,33 +336,6 @@ impl DataUsageCache {
         }
     }
 
-    async fn revision_for_path<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
-        match store
-            .get_object_reader(
-                RUSTFS_META_BUCKET,
-                path,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    no_lock: true,
-                    ..Default::default()
-                },
-            )
-            .await
-        {
-            Ok(reader) => reader
-                .object_info
-                .etag
-                .filter(|etag| !etag.is_empty())
-                .map(DataUsageCacheRevision::Etag)
-                .ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))),
-            Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
-                Ok(DataUsageCacheRevision::Missing)
-            }
-            Err(err) => Err(err),
-        }
-    }
-
     pub(super) fn cache_save_timeout() -> Duration {
         crate::runtime_config::scanner_cache_save_timeout()
     }

crates/scanner/src/data_usage_define/tests.rs

@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
 use super::*;
 use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
 use crate::{ScannerGetObjectReader, ScannerPutObjReader};
-use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
+use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
 use serde_json::Value;
 use std::io::Cursor;
 use std::pin::Pin;
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
             replication_stats: Some(ReplicationAllStats {
                 targets: HashMap::from([(
                     "arn:test:threshold".to_string(),
-                    ReplicationStats {
+                    ReplicationTargetUsage {
                         after_threshold_count: 1,
                         ..Default::default()
                     },

crates/scanner/src/lib.rs

@@ -75,7 +75,10 @@ pub use remote_scanner::{
 };
 pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
 pub use rustfs_common::last_minute;
-pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
+pub use scanner::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
+    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
+};
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
     record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,

crates/scanner/src/runtime_config.rs

@@ -125,7 +125,10 @@ impl Default for ScannerRuntimeConfig {
             cycle_interval_source: ScannerRuntimeConfigSource::Default,
             bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)),
             bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
-            cycle_budget: ScannerCycleBudgetConfig::default(),
+            cycle_budget: ScannerCycleBudgetConfig {
+                max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
+                ..Default::default()
+            },
             cycle_max_duration_source: ScannerRuntimeConfigSource::Default,
             cycle_max_objects_source: ScannerRuntimeConfigSource::Default,
             cycle_max_directories_source: ScannerRuntimeConfigSource::Default,
@@ -374,7 +377,10 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<()
     }
     validate_optional_config_u64(scanner_kvs, SCANNER_START_DELAY, "")?;
     validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE, "")?;
-    validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)?;
+    if let Some(value) = config_value(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
+        let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
+        cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)?;
+    }
     validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS)?;
     validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES)?;
     if let Some(value) = config_value(heal_kvs, HEAL_BITROT_CYCLE, DEFAULT_HEAL_BITROT_CYCLE_SECS) {
@@ -436,19 +442,46 @@ fn lookup_max_wait(
     Ok((speed.max_sleep(), speed_source))
 }
 
-fn lookup_optional_seconds(
+fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
-    kvs: Option<&KVS>,
+    match rustfs_utils::get_env_parse_outcome::<u64>(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) {
-    key: &'static str,
+        rustfs_utils::EnvParseOutcome::Parsed(secs) => {
-    env_key: &'static str,
+            return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs)
-    default: u64,
+                .map(|duration| (duration, ScannerRuntimeConfigSource::Env));
-) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
+        }
-    if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) {
+        rustfs_utils::EnvParseOutcome::Invalid => {
-        return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env));
+            // Do not include the raw environment value in the typed error:
+            // deployments occasionally put sensitive material in inherited
+            // environment snapshots. The key still identifies the control.
+            return Err(invalid_value(
+                ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
+                "<invalid>",
+                "expected unsigned integer seconds",
+            ));
+        }
+        rustfs_utils::EnvParseOutcome::Absent => {}
     }
-    if let Some(value) = config_value(kvs, key, default) {
+
-        return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config));
+    if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
+        let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
+        return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)
+            .map(|duration| (duration, ScannerRuntimeConfigSource::Config));
     }
-    Ok((None, ScannerRuntimeConfigSource::Default))
+
+    Ok((
+        Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
+        ScannerRuntimeConfigSource::Default,
+    ))
+}
+
+fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
+    if secs == 0 {
+        return Ok(None);
+    }
+    let duration = Duration::from_secs(secs);
+    if std::time::Instant::now().checked_add(duration).is_none() {
+        return Err(invalid_value(key, "<overflow>", "duration exceeds the timer range"));
+    }
+    Ok(Some(duration))
 }
 
 fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
@@ -553,12 +586,7 @@ pub(crate) fn lookup_scanner_runtime_config(
         (speed.cycle_interval(), speed_source)
     };
 
-    let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds(
+    let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?;
-        scanner_kvs,
-        SCANNER_CYCLE_MAX_DURATION,
-        ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
-        DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
-    )?;
     let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget(
         scanner_kvs,
         SCANNER_CYCLE_MAX_OBJECTS,
@@ -863,10 +891,10 @@ mod tests {
     use rustfs_config::server_config::{Config as ServerConfig, KVS};
     use rustfs_config::{
         DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
-        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
+        ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY,
-        HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
+        ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
-        SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
+        SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
-        SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
+        SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
     };
     use std::collections::HashMap;
     use std::time::Duration;
@@ -941,6 +969,50 @@ mod tests {
         });
     }
 
+    #[test]
+    fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() {
+        let config = server_config_with_scanner(&[]);
+        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
+            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
+            assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800)));
+            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default);
+        });
+
+        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]);
+        with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
+            let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
+            assert_eq!(resolved.cycle_budget.max_duration, None);
+            assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config);
+        });
+    }
+
+    #[test]
+    fn cycle_budget_invalid_or_overflow_config_is_rejected() {
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || {
+            let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected");
+            assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS));
+            assert!(error.to_string().contains("<invalid>"));
+            assert!(!error.to_string().contains(": invalid ("));
+        });
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || {
+            assert!(lookup_scanner_runtime_config(None).is_err());
+        });
+        with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || {
+            assert!(lookup_scanner_runtime_config(None).is_err());
+        });
+        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]);
+        assert!(lookup_scanner_runtime_config(Some(&config)).is_err());
+    }
+
+    #[test]
+    fn scanner_runtime_config_validation_rejects_overflow_persisted_duration() {
+        let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "18446744073709551615")]);
+
+        let error = validate_scanner_runtime_config(&config)
+            .expect_err("persisted duration that exceeds the timer range must be rejected");
+        assert!(error.to_string().contains(SCANNER_CYCLE_MAX_DURATION));
+    }
+
     #[test]
     fn scanner_runtime_config_normalizes_persisted_default_speed() {
         let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);

crates/scanner/src/scanner.rs

@@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock};
 
 use crate::data_usage_define::{
     BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
-    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision,
+    DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
 };
 use crate::runtime_config::{
     ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
@@ -52,11 +52,10 @@ use rustfs_config::{
 };
 use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
 use rustfs_data_usage::observed_data_usage_is_newer;
+use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
-#[cfg(test)]
+use tokio::sync::{Notify, mpsc};
-use tokio::sync::Notify;
-use tokio::sync::mpsc;
 use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tokio_util::task::AbortOnDropHandle;
@@ -104,6 +103,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
 /// unavailable peer cannot drive a tight retry loop.
 const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
 const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
+/// A transient backend outage remains self-healing after the short retry
+/// budget is exhausted, but the probe is intentionally sparse until storage
+/// recovers or an operator reset wakes the scanner.
+const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
+/// Permanent recovery states still get a sparse status probe so a reset that
+/// races the wait registration cannot leave the scanner asleep forever.
+const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
 const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
 #[cfg(not(test))]
 const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
@@ -125,6 +131,12 @@ type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
 static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
     LazyLock::new(|| StdMutex::new(None));
 
+static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
+
+pub(super) fn notify_scanner_cycle_recovery_wake() {
+    SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
+}
+
 #[cfg(test)]
 struct ScannerCycleStatePersistTestHookGuard;
 
@@ -576,19 +588,21 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
             tokio::time::sleep(sleep_time).await;
         }
 
+        let mut transient_backoff = ScannerRetryBackoff::default();
+        let mut recovery_retry_count = 0_u32;
         loop {
             if ctx_clone.is_cancelled() {
                 break;
             }
 
-            if let Err(e) = run_data_scanner_with_maintenance_state(
+            let run_result = run_data_scanner_with_maintenance_state(
                 ctx_clone.clone(),
                 storeapi_clone.clone(),
                 startup_features,
                 startup_maintenance_generation,
             )
-            .await
+            .await;
-            {
+            if let Err(e) = &run_result {
                 error!(
                     target: "rustfs::scanner",
                     event = EVENT_SCANNER_CYCLE_STATE,
@@ -599,11 +613,52 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
                     "Scanner runtime iteration failed"
                 );
             }
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.retryable {
+                recovery_retry_count = recovery_retry_count.saturating_add(1);
+                let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
+            } else {
+                recovery_retry_count = 0;
+            }
+
+            let recovery_status = scanner_cycle_recovery_status();
+            if recovery_status.state == "paused" {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
+                }
+                recovery_retry_count = 0;
+                continue;
+            }
+            if !recovery_status.retryable
+                && matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
+            {
+                transient_backoff.record_retryable_cycle(false);
+                tokio::select! {
+                    _ = ctx_clone.cancelled() => break,
+                    _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                    _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
+                }
+                continue;
+            }
+
+            let retry_delay = if recovery_status.retryable || run_result.is_err() {
+                transient_backoff.record_retryable_cycle(true);
+                transient_backoff
+                    .retry_interval(scanner_cycle_interval())
+                    .unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
+            } else {
+                transient_backoff.record_retryable_cycle(false);
+                randomized_cycle_delay()
+            };
             // Backoff before retrying after lock contention or scanner-level failures.
             // Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
             tokio::select! {
                 _ = ctx_clone.cancelled() => break,
-                _ = tokio::time::sleep(randomized_cycle_delay()) => {}
+                _ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
+                _ = tokio::time::sleep(retry_delay) => {}
             }
         }
     });
@@ -983,20 +1038,116 @@ fn data_usage_persist_timeout() -> Duration {
     DataUsageCache::persistence_timeout()
 }
 
+#[cfg(not(test))]
+const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
+#[cfg(test)]
+const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
+
+async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
+    ctx: &CancellationToken,
+    storeapi: Arc<Store>,
+    cycle_info: &mut CurrentCycle,
+    cycle_revision: &mut DataUsageCacheRevision,
+    leader_epoch: &mut u64,
+    lock_lost: LockLost,
+) -> bool
+where
+    Store: ScannerObjectIO,
+    LockLost: Future<Output = ()>,
+{
+    let fence_ctx = ctx.child_token();
+    let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch);
+    tokio::pin!(claim);
+    tokio::pin!(lock_lost);
+    tokio::select! {
+        biased;
+        _ = &mut lock_lost => {
+            fence_ctx.cancel();
+            false
+        }
+        result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
+            result.unwrap_or(false) && !fence_ctx.is_cancelled()
+        }
+    }
+}
+
+struct ScannerCycleDeadlineState<'a> {
+    cycle_info: &'a mut CurrentCycle,
+    cycle_revision: &'a mut DataUsageCacheRevision,
+    leader_epoch: &'a mut u64,
+    cycle_budget: &'a ScannerCycleBudget,
+}
+
+fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
+    !worker_stopped || !cycle_state_persisted || !generation_fenced
+}
+
+async fn handle_scanner_cycle_deadline<Store>(
+    ctx: &CancellationToken,
+    storeapi: Arc<Store>,
+    state: ScannerCycleDeadlineState<'_>,
+    worker_stopped: bool,
+    guard: &mut NamespaceLockGuard,
+) where
+    Store: ScannerObjectIO,
+{
+    let fenced = fence_scanner_epoch_after_cycle_timeout(
+        ctx,
+        storeapi,
+        state.cycle_info,
+        state.cycle_revision,
+        state.leader_epoch,
+        guard.lock_lost_notified(),
+    )
+    .await;
+    let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
+    let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
+    warn!(
+        target: "rustfs::scanner",
+        event = EVENT_SCANNER_CYCLE_STATE,
+        component = LOG_COMPONENT_SCANNER,
+        subsystem = LOG_SUBSYSTEM_RUNTIME,
+        state = "cycle_timeout",
+        worker_stopped,
+        cycle_state_persisted,
+        generation_fenced = fenced,
+        recovery_required,
+        "Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
+    );
+    global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
+    // Stop renewing before releasing the lease. A new leader can then claim the
+    // higher persisted generation instead of inheriting the expired worker.
+    guard.release();
+    global_metrics().set_cycle(None).await;
+}
+
 async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
     cycle_info.current = 0;
     global_metrics().clear_current_scan_mode();
     cycle_metrics_guard.finish(cycle_info.clone()).await;
 }
 
-#[instrument(skip_all)]
+#[cfg(test)]
-#[hotpath::measure]
 async fn run_data_scanner_cycle(
     ctx: &CancellationToken,
     storeapi: &Arc<ECStore>,
     cycle_info: &mut CurrentCycle,
     cycle_revision: &mut DataUsageCacheRevision,
     leader_epoch: u64,
+) -> ScannerCycleOutcome {
+    let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
+    run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
+}
+
+#[instrument(skip_all)]
+#[hotpath::measure]
+async fn run_data_scanner_cycle_with_budget(
+    ctx: &CancellationToken,
+    storeapi: &Arc<ECStore>,
+    cycle_info: &mut CurrentCycle,
+    cycle_revision: &mut DataUsageCacheRevision,
+    leader_epoch: u64,
+    cycle_budget: Arc<ScannerCycleBudget>,
 ) -> ScannerCycleOutcome {
     let _activity_guard = ScannerActivityGuard::new();
     if let Err(err) = refresh_scanner_runtime_config_from_global() {
@@ -1012,7 +1163,11 @@ async fn run_data_scanner_cycle(
     }
     let configured_cycle_interval = scanner_cycle_interval();
     let configured_bitrot_cycle = scanner_bitrot_cycle();
-    let cycle_budget_config = scanner_cycle_budget_config();
+    let cycle_budget_config = ScannerCycleBudgetConfig {
+        max_duration: cycle_budget.max_duration(),
+        max_objects: cycle_budget.max_objects(),
+        max_directories: cycle_budget.max_directories(),
+    };
     let usage_persist_timeout = data_usage_persist_timeout();
     global_metrics().record_scanner_cycle_config(
         configured_cycle_interval,
@@ -1083,7 +1238,6 @@ async fn run_data_scanner_cycle(
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
-    let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
     let scan_result = storeapi
         .clone()
         .nsscanner_with_status(
@@ -1223,7 +1377,7 @@ async fn run_data_scanner_cycle(
                 "Scanner cycle is recovering to a newer durable cache generation"
             );
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            return if persist_required_scanner_cycle_floor(
+            let persisted = persist_required_scanner_cycle_floor(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1232,8 +1386,9 @@ async fn run_data_scanner_cycle(
                 required_cycle,
                 &mut cycle_metrics_guard,
             )
-            .await
+            .await;
-            {
+            return if persisted {
+                cycle_budget.mark_cycle_state_persisted();
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1291,7 +1446,7 @@ async fn run_data_scanner_cycle(
             scan_cycle_partial_reason(budget_reason),
             scan_cycle_partial_source(budget_reason),
         );
-        return if finalize_partial_scan_cycle(
+        let persisted = finalize_partial_scan_cycle(
             ctx,
             storeapi.clone(),
             cycle_info,
@@ -1299,8 +1454,9 @@ async fn run_data_scanner_cycle(
             leader_epoch,
             &mut cycle_metrics_guard,
         )
-        .await
+        .await;
-        {
+        return if persisted {
+            cycle_budget.mark_cycle_state_persisted();
             ScannerCycleOutcome::Partial
         } else {
             ScannerCycleOutcome::Failed
@@ -1375,7 +1531,7 @@ async fn run_data_scanner_cycle(
                 );
             }
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            return if finalize_partial_scan_cycle(
+            let persisted = finalize_partial_scan_cycle(
                 ctx,
                 storeapi.clone(),
                 cycle_info,
@@ -1383,8 +1539,9 @@ async fn run_data_scanner_cycle(
                 leader_epoch,
                 &mut cycle_metrics_guard,
             )
-            .await
+            .await;
-            {
+            return if persisted {
+                cycle_budget.mark_cycle_state_persisted();
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -1425,6 +1582,7 @@ async fn run_data_scanner_cycle(
             )
             .await
             {
+                cycle_budget.mark_cycle_state_persisted();
                 emit_scan_cycle_superseded(cycle_start.elapsed());
                 return ScannerCycleOutcome::Superseded;
             }
@@ -1457,6 +1615,7 @@ async fn run_data_scanner_cycle(
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         return ScannerCycleOutcome::Failed;
     }
+    cycle_budget.mark_cycle_state_persisted();
 
     done_cycle();
     emit_scan_cycle_complete(true, cycle_start.elapsed());
@@ -1521,7 +1680,7 @@ async fn run_data_scanner_with_maintenance_state(
 ) -> Result<(), ScannerError> {
     reset_scanner_cycle_schedule();
     // Acquire leader lock (write lock) to ensure only one scanner runs
-    let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
+    let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
         Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
             Ok(guard) => {
                 record_scanner_leader_lock_state("acquired");
@@ -1606,40 +1765,22 @@ async fn run_data_scanner_with_maintenance_state(
         observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
     }
 
-    let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await {
+    let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
-        Ok((buf, revision)) => (buf.unwrap_or_default(), revision),
+        match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
-        Err(err) => {
+            ScannerCycleStateStartup::Ready {
-            error!(
+                cycle,
-                target: "rustfs::scanner",
+                leader_epoch,
-                event = EVENT_SCANNER_PERSIST_STATE,
+                revision,
-                component = LOG_COMPONENT_SCANNER,
+            } => (cycle, leader_epoch, revision),
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
+            ScannerCycleStateStartup::Blocked => {
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
+                global_metrics().set_cycle(None).await;
-                state = "revision_load_failed",
+                return Ok(());
-                error = %err,
+            }
-                "Scanner cycle state revision load failed"
+            ScannerCycleStateStartup::Transient(err) => {
-            );
+                global_metrics().set_cycle(None).await;
-            global_metrics().set_cycle(None).await;
+                return Err(err);
-            return Ok(());
+            }
-        }
+        };
-    };
-    let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) {
-        Ok(state) => state,
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner",
-                event = EVENT_SCANNER_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_RUNTIME,
-                path = %&*DATA_USAGE_BLOOM_NAME_PATH,
-                state = "cycle_decode_failed",
-                error = %err,
-                "Scanner stopped because persisted cycle state is invalid"
-            );
-            global_metrics().set_cycle(None).await;
-            return Ok(());
-        }
-    };
     let usage_floor = match persisted_usage_floor(storeapi.clone()).await {
         Ok(floor) => floor,
         Err(err) => {
@@ -1704,13 +1845,49 @@ async fn run_data_scanner_with_maintenance_state(
             return Ok(());
         }
         let cycle_ctx = ctx.child_token();
-        let initial_outcome = await_scanner_cycle_with_lock_fence(
+        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
+        let initial_outcome = match await_scanner_cycle_with_budget_fence(
             &cycle_ctx,
-            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
+            &cycle_budget,
+            run_data_scanner_cycle_with_budget(
+                &cycle_ctx,
+                &storeapi,
+                &mut cycle_info,
+                &mut cycle_revision,
+                leader_epoch,
+                cycle_budget.clone(),
+            ),
             guard.lock_lost_notified(),
         )
         .await
-        .unwrap_or(ScannerCycleOutcome::Failed);
+        {
+            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
+            ScannerCycleWaitOutcome::LockLost => {
+                record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Cancelled => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
+                handle_scanner_cycle_deadline(
+                    &ctx,
+                    storeapi.clone(),
+                    ScannerCycleDeadlineState {
+                        cycle_info: &mut cycle_info,
+                        cycle_revision: &mut cycle_revision,
+                        leader_epoch: &mut leader_epoch,
+                        cycle_budget: &cycle_budget,
+                    },
+                    worker_stopped,
+                    &mut guard,
+                )
+                .await;
+                return Ok(());
+            }
+        };
         superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -1916,13 +2093,49 @@ async fn run_data_scanner_with_maintenance_state(
         }
         let dirty_generation_before_cycle = dirty_usage_generation();
         let cycle_ctx = ctx.child_token();
-        let outcome = await_scanner_cycle_with_lock_fence(
+        let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
+        let outcome = match await_scanner_cycle_with_budget_fence(
             &cycle_ctx,
-            run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
+            &cycle_budget,
+            run_data_scanner_cycle_with_budget(
+                &cycle_ctx,
+                &storeapi,
+                &mut cycle_info,
+                &mut cycle_revision,
+                leader_epoch,
+                cycle_budget.clone(),
+            ),
             guard.lock_lost_notified(),
         )
         .await
-        .unwrap_or(ScannerCycleOutcome::Failed);
+        {
+            ScannerCycleWaitOutcome::Completed(outcome) => outcome,
+            ScannerCycleWaitOutcome::LockLost => {
+                record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Cancelled => {
+                global_metrics().set_cycle(None).await;
+                return Ok(());
+            }
+            ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
+                handle_scanner_cycle_deadline(
+                    &ctx,
+                    storeapi.clone(),
+                    ScannerCycleDeadlineState {
+                        cycle_info: &mut cycle_info,
+                        cycle_revision: &mut cycle_revision,
+                        leader_epoch: &mut leader_epoch,
+                        cycle_budget: &cycle_budget,
+                    },
+                    worker_stopped,
+                    &mut guard,
+                )
+                .await;
+                return Ok(());
+            }
+        };
         superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -2219,7 +2432,12 @@ pub(crate) use activity::{
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
-pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence};
+pub use cycle_state::{
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
+};
+pub(crate) use cycle_state::{
+    current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
+};
 pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
 pub use usage_store::store_data_usage_in_backend;
 

crates/scanner/src/scanner/leadership.rs

@@ -196,7 +196,7 @@ pub(super) async fn claim_scanner_leadership(
         if ctx.is_cancelled() {
             return false;
         }
-        let Some(claimed_epoch) = persisted_epoch.checked_add(1) else {
+        let Some(claimed_epoch) = persisted_epoch.checked_add(1).filter(|epoch| *epoch < u64::MAX) else {
             error!(
                 target: "rustfs::scanner",
                 event = EVENT_SCANNER_PERSIST_STATE,

crates/scanner/src/scanner_budget.rs

@@ -14,17 +14,16 @@
 
 use std::sync::{
     Arc,
-    atomic::{AtomicU8, AtomicU64, Ordering},
+    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
 };
-use std::time::Instant;
+use tokio::time::{Duration, Instant};
-
-use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
 
 const BUDGET_REASON_NONE: u8 = 0;
 const BUDGET_REASON_RUNTIME: u8 = 1;
 const BUDGET_REASON_OBJECTS: u8 = 2;
 const BUDGET_REASON_DIRECTORIES: u8 = 3;
+const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub(crate) struct ScannerCycleBudgetConfig {
@@ -63,29 +62,51 @@ pub struct ScannerCycleBudget {
     token: CancellationToken,
     reason: Arc<AtomicU8>,
     started_at: Instant,
+    deadline: Option<Instant>,
     max_duration: Option<Duration>,
     max_objects: Option<u64>,
     max_directories: Option<u64>,
     track_progress: bool,
+    track_unbounded_counts: bool,
     objects_scanned: AtomicU64,
     directories_started: AtomicU64,
     entries_visited: AtomicU64,
+    last_progress_millis: AtomicU64,
+    cycle_state_persisted: AtomicBool,
 }
 
 impl ScannerCycleBudget {
+    #[cfg(test)]
     pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, false)
+        Self::new_inner(parent, config, false, false)
     }
 
     pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
-        Self::new_inner(parent, config, true)
+        Self::new_inner(parent, config, true, true)
     }
 
-    fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
+    pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
+        let track_progress = config.max_duration.is_some();
+        Self::new_inner(parent, config, track_progress, false)
+    }
+
+    fn new_inner(
+        parent: &CancellationToken,
+        config: ScannerCycleBudgetConfig,
+        track_progress: bool,
+        track_unbounded_counts: bool,
+    ) -> Arc<Self> {
         let token = parent.child_token();
         let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
+        let started_at = Instant::now();
+        let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) {
+            Some(deadline) => deadline,
+            // Runtime config rejects this range, but keep programmatic callers
+            // fail-closed instead of panicking or silently disabling the wall clock.
+            None => started_at,
+        });
 
-        if let Some(duration) = config.max_duration {
+        if let Some(deadline) = deadline {
             let parent = parent.clone();
             let token_wait = token.clone();
             let token_cancel = token.clone();
@@ -94,7 +115,7 @@ impl ScannerCycleBudget {
                 tokio::select! {
                     _ = parent.cancelled() => {}
                     _ = token_wait.cancelled() => {}
-                    _ = tokio::time::sleep(duration) => {
+                    _ = tokio::time::sleep_until(deadline) => {
                         Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime);
                     }
                 }
@@ -104,14 +125,18 @@ impl ScannerCycleBudget {
         Arc::new(Self {
             token,
             reason,
-            started_at: Instant::now(),
+            started_at,
+            deadline,
             max_duration: config.max_duration,
             max_objects: config.max_objects,
             max_directories: config.max_directories,
             track_progress,
+            track_unbounded_counts,
             objects_scanned: AtomicU64::new(0),
             directories_started: AtomicU64::new(0),
             entries_visited: AtomicU64::new(0),
+            last_progress_millis: AtomicU64::new(0),
+            cycle_state_persisted: AtomicBool::new(false),
         })
     }
 
@@ -131,6 +156,14 @@ impl ScannerCycleBudget {
         self.max_duration
     }
 
+    pub(crate) fn deadline(&self) -> Option<Instant> {
+        self.deadline
+    }
+
+    pub(crate) fn cancel_for_runtime(&self) {
+        self.cancel_for(ScannerCycleBudgetReason::Runtime);
+    }
+
     pub(crate) fn max_objects(&self) -> Option<u64> {
         self.max_objects
     }
@@ -173,15 +206,43 @@ impl ScannerCycleBudget {
         self.entries_visited.load(Ordering::Relaxed)
     }
 
+    pub(crate) fn mark_cycle_state_persisted(&self) {
+        self.cycle_state_persisted.store(true, Ordering::Release);
+    }
+
+    pub(crate) fn cycle_state_persisted(&self) -> bool {
+        self.cycle_state_persisted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn progress_age(&self) -> Duration {
+        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
+        let last_progress = self.last_progress_millis.load(Ordering::Relaxed);
+        Duration::from_millis(elapsed_millis.saturating_sub(last_progress))
+    }
+
+    fn record_progress_sample(&self, event: u64) {
+        // Clock reads are sampled at batch/count boundaries; the scanner's
+        // per-object path does not add a second progress atomic.
+        if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) {
+            return;
+        }
+        let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
+        self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed);
+    }
+
     pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
         if self.track_progress {
-            saturating_fetch_add(&self.entries_visited, entries_visited);
+            let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
+            self.record_progress_sample(entries);
         }
     }
 
     pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
         if self.track_progress || self.max_objects.is_some() {
             let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
+            if self.track_progress {
+                self.record_progress_sample(objects);
+            }
             if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
                 self.cancel_for(ScannerCycleBudgetReason::Objects);
             }
@@ -189,9 +250,12 @@ impl ScannerCycleBudget {
 
         if self.track_progress || self.max_directories.is_some() {
             let directories = saturating_fetch_add(&self.directories_started, directories_started);
+            if self.track_progress {
+                self.record_progress_sample(directories);
+            }
             if self
                 .max_directories
-                .is_some_and(|max_directories| directories > max_directories)
+                .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories))
             {
                 self.cancel_for(ScannerCycleBudgetReason::Directories);
             }
@@ -207,14 +271,17 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn try_start_directory(&self) -> bool {
-        if !self.track_progress && self.max_directories.is_none() {
+        if self.max_directories.is_none() && !self.track_unbounded_counts {
             return true;
         }
 
         let directories = saturating_fetch_add(&self.directories_started, 1);
+        if self.track_progress {
+            self.record_progress_sample(directories);
+        }
         if self
             .max_directories
-            .is_some_and(|max_directories| directories > max_directories)
+            .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories))
         {
             self.cancel_for(ScannerCycleBudgetReason::Directories);
             return false;
@@ -224,11 +291,14 @@ impl ScannerCycleBudget {
     }
 
     pub(crate) fn record_object_scanned(&self) {
-        if !self.track_progress && self.max_objects.is_none() {
+        if self.max_objects.is_none() && !self.track_unbounded_counts {
             return;
         }
 
         let objects = saturating_fetch_add(&self.objects_scanned, 1);
+        if self.track_progress {
+            self.record_progress_sample(objects);
+        }
         if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
             self.cancel_for(ScannerCycleBudgetReason::Objects);
         }
@@ -259,6 +329,13 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
     }
 }
 
+fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool {
+    // Saturation hides a remote max+1 update when the configured limit is the
+    // largest representable counter. Treat that boundary as exhausted rather
+    // than allowing work to continue indefinitely.
+    directories > max_directories || (directories == u64::MAX && max_directories == u64::MAX)
+}
+
 impl Drop for ScannerCycleBudget {
     fn drop(&mut self) {
         self.token.cancel();
@@ -401,6 +478,35 @@ mod tests {
         assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
     }
 
+    #[test]
+    fn directory_budget_fails_closed_when_progress_saturates() {
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(
+            &parent,
+            ScannerCycleBudgetConfig {
+                max_directories: Some(u64::MAX),
+                ..Default::default()
+            },
+        );
+
+        budget.record_remote_progress(0, u64::MAX);
+
+        assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Directories));
+        assert!(budget.token().is_cancelled());
+
+        let local_budget = ScannerCycleBudget::new(
+            &parent,
+            ScannerCycleBudgetConfig {
+                max_directories: Some(u64::MAX),
+                ..Default::default()
+            },
+        );
+        local_budget.record_remote_progress(0, u64::MAX - 1);
+        assert!(!local_budget.budget_elapsed());
+        assert!(!local_budget.try_start_directory());
+        assert_eq!(local_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
+    }
+
     #[test]
     fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() {
         let parent = CancellationToken::new();
@@ -461,4 +567,29 @@ mod tests {
         assert!(object_limited.requires_serial_progress_accounting());
         assert!(directory_limited.requires_serial_progress_accounting());
     }
+
+    #[tokio::test(start_paused = true)]
+    async fn progress_age_uses_virtual_time_and_sampled_progress() {
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new_with_runtime_progress_tracking(
+            &parent,
+            ScannerCycleBudgetConfig {
+                max_duration: Some(Duration::from_secs(60)),
+                ..Default::default()
+            },
+        );
+
+        tokio::time::advance(Duration::from_secs(5)).await;
+        assert_eq!(budget.progress_age(), Duration::from_secs(5));
+        budget.record_entries_visited(1);
+        assert_eq!(budget.progress_age(), Duration::ZERO);
+
+        tokio::time::advance(Duration::from_secs(2)).await;
+        for _ in 0..126 {
+            budget.record_entries_visited(1);
+        }
+        assert_eq!(budget.progress_age(), Duration::from_secs(2));
+        budget.record_entries_visited(1);
+        assert_eq!(budget.progress_age(), Duration::ZERO);
+    }
 }

crates/scanner/src/scanner_io.rs

@@ -48,6 +48,7 @@ use time::OffsetDateTime;
 use tokio::sync::{Mutex, Notify, Semaphore, mpsc};
 use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
+use tokio_util::task::AbortOnDropHandle;
 use tracing::{debug, error, warn};
 
 use crate::ScannerObjectInfo as ObjectInfo;

crates/scanner/src/scanner_io/io_cache.rs

@@ -314,7 +314,7 @@ impl ScannerIOCache for SetDisks {
         let ctx_clone = ctx.clone();
         let completed_bucket_count = Arc::new(AtomicUsize::new(0));
         let completed_bucket_count_clone = completed_bucket_count.clone();
-        let collect_bucket_results_fut = tokio::spawn(async move {
+        let collect_bucket_results_fut = AbortOnDropHandle::new(tokio::spawn(async move {
             let mut cancelled = false;
 
             loop {
@@ -333,7 +333,7 @@ impl ScannerIOCache for SetDisks {
                     }
                 }
             }
-        });
+        }));
 
         let mut futs = Vec::new();
 
@@ -365,7 +365,7 @@ impl ScannerIOCache for SetDisks {
                 NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
                 NamespaceScannerWorkerMode::Coordinator => None,
             };
-            futs.push(tokio::spawn(async move {
+            futs.push(AbortOnDropHandle::new(tokio::spawn(async move {
                 let remote_session_id = uuid::Uuid::new_v4();
                 let mut remote_session_sequence = 0_u64;
                 loop {
@@ -1038,7 +1038,7 @@ impl ScannerIOCache for SetDisks {
                         );
                     }
                 }
-            }));
+            })));
         }
         drop(bucket_tx);
         drop(bucket_result_tx);

crates/scanner/src/scanner_io/io_cycle.rs

@@ -242,7 +242,7 @@ impl ScannerIOCycle for ECStore {
                         results[results_index_clone] = result;
                     }
                 });
-                wait_futs.push(receiver_fut);
+                wait_futs.push(AbortOnDropHandle::new(receiver_fut));
 
                 let scan_plan = ScannerBucketScanPlan {
                     buckets: set_buckets,
@@ -318,7 +318,7 @@ impl ScannerIOCycle for ECStore {
                         record_set_scan_failure(&mut first_err, e);
                     }
                 });
-                wait_futs.push(scanner_fut);
+                wait_futs.push(AbortOnDropHandle::new(scanner_fut));
             }
         }
 

crates/scanner/src/scanner_io/publish_gate_tests.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use super::*;
-use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
+use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
 
 const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
 
@@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() {
         replication_stats: Some(ReplicationAllStats {
             targets: HashMap::from([(
                 "arn:target".to_string(),
-                ReplicationStats {
+                ReplicationTargetUsage {
                     replicated_size: 2048,
                     replicated_count: 2,
                     ..Default::default()

crates/storage-api/src/lib.rs

@@ -76,6 +76,7 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
 pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
 pub use error::{StorageErrorCode, StorageResult};
 pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
+pub use object::DeleteAccounting;
 pub use object::ObjectLockDeleteOptions;
 pub use object::{DeletedObject, ObjectToDelete};
 pub use object::{ExpirationOptions, TransitionedObject};

crates/storage-api/src/object.rs

@@ -218,6 +218,17 @@ pub struct DeletedObject {
     pub force_delete_generation: Option<i64>,
 }
 
+/// Accounting identity returned by the internal commit-time delete path.
+///
+/// This is carried separately from [`DeletedObject`] so adding quota details
+/// does not change the source shape of the public S3 delete result contract.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct DeleteAccounting {
+    pub size: Option<u64>,
+    pub version_id: Option<Uuid>,
+    pub removed_current_object: bool,
+}
+
 impl DeletedObject {
     pub fn version_purge_status(&self) -> VersionPurgeStatusType {
         self.replication_state
@@ -341,6 +352,19 @@ pub trait ObjectOperations: Send + Sync + fmt::Debug {
         objects: Vec<Self::ObjectToDelete>,
         opts: Self::ObjectOptions,
     ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>);
+    /// Delete objects and optionally return commit-time accounting identities.
+    /// The default preserves the ordinary delete contract for implementations
+    /// that do not expose storage-level accounting details.
+    async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<Self::ObjectToDelete>,
+        opts: Self::ObjectOptions,
+    ) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>, Vec<Option<DeleteAccounting>>) {
+        let object_count = objects.len();
+        let (deleted, errors) = self.delete_objects(bucket, objects, opts).await;
+        (deleted, errors, vec![None; object_count])
+    }
     async fn put_object_metadata(
         &self,
         bucket: &str,

crates/test-utils/src/lib.rs

@@ -245,6 +245,18 @@ impl TestECStoreEnvBuilder {
             .await
             .expect("build test ECStore");
 
+        // The production bootstrap only persists pool.bin from the elected
+        // first cluster node.  Test stores intentionally have no cluster
+        // election, but heal-format still requires that durable fence before
+        // it can write any disk format.  Materialize the validated topology
+        // here so the shared fixture models a ready single-node store.
+        let mut pool_meta = ecstore.pool_meta.read().await.clone();
+        pool_meta.dont_save = false;
+        pool_meta
+            .save(ecstore.pools.clone())
+            .await
+            .expect("persist test pool metadata");
+
         if self.init_bucket_metadata {
             let buckets_list = ecstore
                 .list_bucket(&BucketOptions {

crates/utils/src/envs.rs

@@ -268,7 +268,7 @@ where
         .parse::<T>()
         .map_err(|_| {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
             });
         })
         .ok()
@@ -570,7 +570,7 @@ where
         Ok(parsed) => EnvParseOutcome::Parsed(parsed),
         Err(_) => {
             log_once(&format!("env_invalid_value:{used_key}"), || {
-                format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
+                format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
             });
             EnvParseOutcome::Invalid
         }

docs/architecture/compat-cleanup-register.md

@@ -12,6 +12,7 @@ for later deletion.
 
 ## Open Items
 
+- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
 - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
 - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
 - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.

docs/operations/scanner-runtime-controls.md

@@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a
 | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. |
 | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. |
 | `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. |
-| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. |
+| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. |
 | `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. |
 | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. |
 | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
@@ -70,6 +70,21 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
 `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
 needs a precise override.
 
+When the cycle duration control is unset, RustFS uses a finite 1800-second
+(30-minute) default, matching the scanner benchmark guidance. An explicit `0`
+preserves the compatibility behavior of an unbounded cycle; object and
+directory budgets likewise remain unbounded when explicitly set to `0`. Invalid
+or overflowing duration environment values are configuration errors rather than
+silent fallback values.
+
+When a finite deadline expires, RustFS cancels cooperative scanner work and
+waits only for the existing bounded shutdown window. A non-yielding I/O future
+is dropped after that window. RustFS then attempts a higher leadership epoch so
+late cycle, usage, cache, and remote writes from the old generation fail closed.
+If the worker cannot stop cooperatively, the cycle state was not confirmed
+durable, or that epoch fence cannot be durably persisted, the scanner reports
+`recovery-required`; it does not claim an uncooperative cursor was saved.
+
 An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
 cadence: dirty-usage notifications do not bypass that configured interval.
 The default adaptive policy continues to use dirty-usage notifications to wake
@@ -144,6 +159,10 @@ metrics.maintenance_control.primary_control
 metrics.source_work
 metrics.replication_repair
 metrics.scan_checkpoint
+metrics.cycle_timeout_total
+metrics.cycle_last_progress_age
+metrics.leader_lease_without_progress
+metrics.cycle_recovery_required_total
 ```
 
 ## Reading Pacing Pressure