fix(api): drop imports left unused by the base merge

# Conflicts:
#	rustfs/src/app/object/put.rs

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
+sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
+# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+# Reserve the runner so other server fixtures do not starve its disk operations.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+threads-required = "num-test-threads"
+
 # Fixed, small contract for every executable server change. These existing
 # scenarios exercise write/read, anonymous denial and retained bytes on restart.
 [profile.core-contract]

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,6 +296,32 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
+    {
+      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
+      "name": "RustFS Validation Sweep",
+      "roles": [
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "30 19 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 60,
+      "result_owner": "rustfs-validation-sweep.yml",
+      "disposition": "canonical validation cases executed in rotating batches",
+      "failure_owner": "workflow",
+      "origin": "baseline",
+      "alert_contract": "scheduled-validation"
+    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -743,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -823,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -497,6 +497,63 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     # Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1397,6 +1454,7 @@ jobs:
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1424,6 +1482,7 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
+            target/x86_64-unknown-linux-gnu/release/rustfs
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-validation-sweep.yml

@@ -0,0 +1,457 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Validation Sweep
+
+# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
+# on the shared functional fleet. Everything with a lane executor is swept in
+# rotating nightly batches so the whole set is covered roughly weekly; a
+# single case or one module can be dispatched manually. That is the
+# implemented catalog (66 today) plus the eight planned cases whose dedicated
+# executors already exist and have live PASS evidence - fresh nightly
+# evidence feeds their acceptance review. heal.* cases stay on the
+# interactive validation flow: they require reviewed native/tool collectors
+# built by separate validation-tools runs before fleet execution.
+
+on:
+  schedule:
+    # After the nightly functional chain settles (chain starts ~16:48 UTC and
+    # runs for about two hours).
+    - cron: "30 19 * * *"
+  workflow_dispatch:
+    inputs:
+      case:
+        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
+        type: string
+        required: false
+        default: ''
+      module:
+        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
+        type: string
+        required: false
+        default: ''
+      limit:
+        description: 'Maximum cases per sweep (0 = run everything selected)'
+        type: number
+        required: false
+        default: 12
+      build_run_id:
+        description: 'Successful nightly package run; empty resolves the latest'
+        type: string
+        required: false
+        default: ''
+      build_run_attempt:
+        description: 'Attempt of that package run'
+        type: string
+        required: false
+        default: '1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-sweep
+  cancel-in-progress: false
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  sweep:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: smoke-testing
+    timeout-minutes: 150
+    steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Resolve nightly candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
+        run: |
+          set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
+          python3 scripts/resolve_functional_candidate.py
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
+          root.mkdir(parents=True, exist_ok=True)
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          manifest = chain["candidate"]["manifest"]
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
+              output.write("package_url=" + manifest["package_url"] + "\n")
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.candidate.outputs.testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Select sweep cases
+        id: select
+        env:
+          SELECTED_CASE: ${{ inputs.case }}
+          SELECTED_MODULE: ${{ inputs.module }}
+          CASE_LIMIT: ${{ inputs.limit }}
+          SWEEP_EVENT: ${{ github.event_name }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import datetime
+          import json
+          import os
+          from pathlib import Path
+          sweep_root = Path(os.environ["SWEEP_ROOT"])
+
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          def sweepable(case):
+              status = case.get("source_status")
+              if status == "implemented":
+                  return True
+              # The remaining planned non-heal cases route to the lane
+              # executors and already have live PASS evidence; sweeping them
+              # nightly produces fresh evidence for their acceptance review.
+              return status == "planned" and not case["id"].startswith("heal.")
+
+          selected = [(case["id"], module["id"]) for module in catalog["modules"]
+                      for case in module["cases"]
+                      if sweepable(case)]
+          if not selected:
+              raise SystemExit("catalog has no sweepable cases")
+
+          one_case = os.environ.get("SELECTED_CASE", "").strip()
+          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
+          known = {case_id for case_id, _ in selected}
+          modules = {module_id for _, module_id in selected}
+          if one_case:
+              if one_case not in known:
+                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
+              selected = [(one_case, dict(selected)[one_case])]
+          elif module_filter:
+              if module_filter not in modules:
+                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
+              selected = [pair for pair in selected if pair[1] == module_filter]
+          elif os.environ.get("SWEEP_EVENT") == "schedule":
+              # Stateless rotation: every night take one window of the catalog
+              # so a full sweep needs len(selected)/limit nights and no state
+              # has to be carried between runs.
+              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
+              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
+              start = (day * limit) % len(selected)
+              selected = (selected[start:] + selected[:start])[:limit]
+          limit = int(os.environ.get("CASE_LIMIT") or 0)
+          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
+              if limit > 0:
+                  selected = selected[:limit]
+
+          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"count={len(selected)}\n")
+          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
+          PY
+
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
+      - name: Run validation cases
+        id: run_cases
+        timeout-minutes: 120
+        env:
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -uo pipefail
+          mkdir -p "${SWEEP_ROOT}/results"
+          : > "${SWEEP_ROOT}/results.txt"
+          while read -r CASE_ID; do
+            [ -n "${CASE_ID}" ] || continue
+            echo "::group::case ${CASE_ID}"
+            rm -rf "${SWEEP_ROOT}/run"
+            if ./auto-testing/scripts/run_validation_case.sh \
+                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
+                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
+                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
+              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: PASS"
+            else
+              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
+            fi
+            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
+              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
+            fi
+            echo "::endgroup::"
+            # Fleet hygiene between cases: the case runners set up their own
+            # topology, this purge guarantees the next case starts clean even
+            # when the previous one died mid-run.
+            for node in ${RUSTFS_NODES}; do
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+                ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                  ${SUDO} dpkg -P rustfs
+                fi
+                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                ${SUDO} rm -f /etc/default/rustfs
+              ' || echo "inter-case cleanup failed on ${node} (continuing)"
+            done
+          done < "${SWEEP_ROOT}/sweep-cases.txt"
+          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
+
+      - name: Generate sweep report
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
+          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["SWEEP_ROOT"])
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
+          rows = []
+          for line in (root / "results.txt").read_text().splitlines():
+            if not line:
+                continue
+            case_id, status = line.split("\t")
+            rows.append((case_id, module_of.get(case_id, "?"), status))
+          passed = sum(1 for _, _, s in rows if s == "PASS")
+          failed = len(rows) - passed
+          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
+                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
+                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
+
+          report = [
+              "# RustFS validation sweep report", "",
+              f"- Run: {run_url}",
+              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
+              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
+              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
+              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
+              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
+              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
+          ]
+          # Group the executed cases by catalog module (the major category:
+          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
+          # data services, identity, s3 compatibility). Per-module subtotals
+          # deliberately use "- Subtotal:" so the dashboard overview regexes
+          # (which anchor on "- Total:") keep reading the grand totals below.
+          by_module = {}
+          module_order = []
+          for case_id, module, status in rows:
+              if module not in by_module:
+                  by_module[module] = []
+                  module_order.append(module)
+              by_module[module].append((case_id, status))
+          report += ["## Case Summary", ""]
+          for module in module_order:
+              cases = by_module[module]
+              mod_pass = sum(1 for _, st in cases if st == "PASS")
+              mod_fail = len(cases) - mod_pass
+              report += [f"### {module} ({len(cases)})", "",
+                         "| Case | Status |", "| --- | --- |"]
+              report += [f"| {case_id} | {status} |" for case_id, status in cases]
+              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
+          report += [
+              "## Totals", "",
+              f"- Total: {len(rows)}",
+              f"- PASS: {passed}",
+              f"- FAIL: {failed}",
+              "- UNSUPPORTED: 0", "",
+              "Failing cases keep this workflow red only as a report; the sweep artifact",
+              "retains every case log and per-case report for diagnosis.", "",
+          ]
+          (root / "report.md").write_text("\n".join(report))
+          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
+          PY
+          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 2
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
+          SUITE: validation
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Retain sweep evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/validation-sweep/report.md
+            ${{ runner.temp }}/validation-sweep/results.txt
+            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
+            ${{ runner.temp }}/validation-sweep/candidate.json
+            ${{ runner.temp }}/validation-sweep/results/
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        id: environment_cleanup
+        timeout-minutes: 5
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        run: |
+          set -uo pipefail
+          cleanup_status=0
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || cleanup_status=$?
+          done
+          exit "${cleanup_status}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
+
+      - name: Enforce sweep result
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
+            echo "validation sweep has failing cases:"
+            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
+            exit 1
+          fi
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # A scheduled sweep failure usually means a canonical validation case
+    # regressed or the fleet environment broke; surface it as a tracking
+    # issue instead of leaving it in the Actions tab.
+    needs: [sweep]
+    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
+csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +236,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +255,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -337,7 +339,7 @@ strum = { version = "0.28.0" }
 sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.2", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +429,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
-                    .copy_object()
-                    .bucket(&bucket)
-                    .key(&copy_key)
-                    .copy_source(format!("{bucket}/{key}"))
-                    .send()
-                    .await?;
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                after_lock_contention(&format!("{context} COPY retained"), || {
+                    copier
+                        .copy_object()
+                        .bucket(&bucket)
+                        .key(&copy_key)
+                        .copy_source(format!("{bucket}/{key}"))
+                        .send()
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
-                let missing = reader
-                    .head_object()
-                    .bucket(&bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .expect_err("deleted source key must not remain visible");
+                after_lock_contention(&format!("{context} DELETE source"), || {
+                    writer.delete_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
-        .delete_object()
+    let wait_for_source_writeback = || async {
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
-        .version_id(&purged_version_id)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/storage_api.rs

@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
     pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
     pub(crate) use ecstore_rpc::PeerRestClient;
 }
+
+/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
+#[cfg(test)]
+pub(crate) mod bucket_tag_fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
+    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,12 +360,26 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, action, bucket, department);
+        Ok(())
+    }
+
+    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
+        let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
+        assert_eq!(payload["input"]["resource"]["object"], object);
+        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
+        Ok(())
+    }
+
+    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
+        assert_eq!(input["identity"]["account"], "opabuckettags");
+        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
-        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
+/// Stop and reap the directory's owning server before opening the fixture store.
+async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
+    use crate::storage_api::bucket_tag_fixture::{
+        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
+    };
+
+    let server = env
+        .process
+        .as_mut()
+        .ok_or("missing server process before offline fixture write")?;
+    server.kill()?;
+    server.wait()?;
+    env.process = None;
+    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
+    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
+    let context = std::sync::Arc::new(InstanceContext::new());
+    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
+    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
+    let _cancel = context
+        .background_cancel_token()
+        .expect("store binds its shutdown token")
+        .drop_guard();
+    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
+    let incarnation = metadata.bucket_incarnation_id;
+    let policy = metadata.policy_config_json.clone();
+    assert!(!incarnation.is_nil());
+    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
+    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
+    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
+    metadata.tagging_config_xml = xml.to_vec();
+    metadata.save_with_store(store.clone()).await?;
+    let persisted = load_bucket_metadata(store, bucket).await?;
+    assert_eq!(persisted.bucket_incarnation_id, incarnation);
+    assert_eq!(persisted.policy_config_json, policy);
+    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
     };
 
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_without_cleanup_with_env(&[
-        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+    let opa_url = opa.url.clone();
+    let server_env = [
+        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ])
-    .await?;
+    ];
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    client
+    let head = client
         .head_object()
         .bucket(source)
         .key("report")
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    assert_eq!(head.tag_count(), Some(1));
+    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    let error = client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await
-        .expect_err("source tags must not authorize a different destination");
-    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
-        .await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        let error = client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await
+            .expect_err("source tags must not authorize a different destination");
+        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        if directive == TaggingDirective::Copy {
+            opa.expect_versioned_tag_permission(source, "report", version).await?;
+        }
+        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+            .await?;
+        let missing = admin
+            .head_object()
+            .bucket(destination)
+            .key("copied")
+            .send()
+            .await
+            .expect_err("denied COPY and REPLACE must leave no destination object");
+        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
     set_department(&admin, destination, "finance").await?;
-    client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await?;
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await?;
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        // The access hook checks preliminary tags, then the copy use case checks
+        // the exact source reader's tags before publishing the destination.
+        for _ in 0..2 {
+            if directive == TaggingDirective::Copy {
+                opa.expect_versioned_tag_permission(source, "report", version).await?;
+            }
+            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+        }
+        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
+        assert_eq!(tags.tag_set().len(), 1);
+        assert_eq!(tags.tag_set()[0].key(), "department");
+        assert_eq!(
+            tags.tag_set()[0].value(),
+            if directive == TaggingDirective::Copy {
+                "engineering"
+            } else {
+                "legal"
+            }
+        );
+    }
 
     let upload = client
         .create_multipart_upload()
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
+    let missing = admin
+        .head_object()
+        .bucket(destination)
+        .key("denied-source")
+        .send()
+        .await
+        .expect_err("a denied source must leave no destination object");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    admin
+    let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
+        .await
+        .expect_err("new duplicate tags must be rejected before publication");
+    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
+    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
+
+    set_department(&admin, source, "finance").await?;
+    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
+    let read = client
+        .get_object()
+        .bucket(source)
+        .key("report")
+        .version_id(version)
+        .send()
         .await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+
     let error = client
         .get_object()
         .bucket(destination)

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/api/mod.rs

@@ -192,6 +192,7 @@ pub mod bucket {
             table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -331,8 +332,8 @@ pub mod bucket {
     pub mod utils {
         pub use crate::bucket::utils::{
             check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
-            serialize,
+            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
+            is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
 
@@ -451,9 +452,15 @@ pub mod disk {
     pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
     pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
+    #[cfg(feature = "test-util")]
+    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
+    pub use crate::disk::multipart_index::{
+        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
+        MultipartIndexRpcRequest,
+    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -466,7 +473,9 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
+        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
+        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
+        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
@@ -617,7 +626,7 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, wrap_reader,
+        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 
@@ -654,7 +663,9 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
+        pub use crate::bucket::quota::reservation::{
+            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
+        };
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
+    UploadResponseDeadline, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
+/// Replication request budgets, snapshotted when a target client is built.
+/// Streamed GET bodies are not subject to the upload-response budget.
+#[derive(Debug, Clone, Copy)]
+pub struct TargetRequestTimeouts {
+    request: Duration,
+    complete: Duration,
+    minimum_transfer_rate: u64,
+}
+
+impl Default for TargetRequestTimeouts {
+    fn default() -> Self {
+        Self {
+            request: Duration::from_secs(60),
+            complete: Duration::from_secs(15 * 60),
+            minimum_transfer_rate: 1024 * 1024,
+        }
+    }
+}
+
+impl TargetRequestTimeouts {
+    fn from_env() -> Self {
+        let defaults = Self::default();
+        let seconds = |name: &str, fallback: Duration| {
+            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
+            Duration::from_secs(if (1..=86400).contains(&value) {
+                value
+            } else {
+                fallback.as_secs()
+            })
+        };
+        let minimum_transfer_rate =
+            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
+        Self {
+            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
+            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
+            minimum_transfer_rate: if minimum_transfer_rate == 0 {
+                defaults.minimum_transfer_rate
+            } else {
+                minimum_transfer_rate
+            },
+        }
+    }
+
+    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
+        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
+        // The SDK can make three attempts. Leave transmission headroom for
+        // all three; the separate response deadline still bounds a trickling
+        // target after each upload finishes.
+        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
+            RemoteS3RetryPolicy::Disabled => 1,
+            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
+        };
+        let seconds = size
+            .div_ceil(self.minimum_transfer_rate)
+            .checked_mul(attempts)
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        let timeout = self
+            .request
+            .checked_add(Duration::from_secs(seconds))
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        if Instant::now().checked_add(timeout).is_none() {
+            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
+        }
+        Ok(timeout)
+    }
+
+    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
+        Ok(aws_sdk_s3::config::Builder::new()
+            .timeout_config(
+                aws_smithy_types::timeout::TimeoutConfig::builder()
+                    .operation_timeout(self.upload_timeout(size)?)
+                    .build(),
+            )
+            .interceptor(UploadResponseDeadline(self.request)))
+    }
+}
+
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
+    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2360,12 +2441,11 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
+        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            operation = operation.config_override(
-                aws_sdk_s3::config::Builder::new()
-                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
-            );
+            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
         }
+        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2501,6 +2581,7 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2538,6 +2619,13 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(
+                aws_sdk_s3::config::Builder::new().timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(self.timeouts.complete)
+                        .build(),
+                ),
+            )
             .send()
             .await
         {
@@ -2966,11 +3054,241 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
+    #[derive(Debug, Clone)]
+    struct TimedResponseConnector {
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: bytes::Bytes,
+        frames: Arc<std::sync::atomic::AtomicUsize>,
+        dropped: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
+    impl Drop for ResponseDrop {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    impl SmithyHttpConnector for TimedResponseConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let settings = self.clone();
+            HttpConnectorFuture::new(async move {
+                http_body_util::BodyExt::collect(request.into_body())
+                    .await
+                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
+                tokio::time::sleep(settings.upload_delay).await;
+                let guard = ResponseDrop(Arc::clone(&settings.dropped));
+                let stream = futures::stream::unfold(
+                    (settings.prefix_bytes, Some(settings.response), true, guard),
+                    move |(remaining, mut body, first, guard)| {
+                        let frames = Arc::clone(&settings.frames);
+                        async move {
+                            if remaining == 0 && body.is_none() {
+                                return None;
+                            }
+                            if !first {
+                                tokio::time::sleep(settings.response_delay).await;
+                            }
+                            let bytes = if remaining > 0 {
+                                bytes::Bytes::from_static(b" ")
+                            } else {
+                                body.take()?
+                            };
+                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                            Some((
+                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
+                                (remaining.saturating_sub(1), body, false, guard),
+                            ))
+                        }
+                    },
+                )
+                .fuse();
+                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
+                let mut response =
+                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
+                response.headers_mut().insert("etag", "\"test-etag\"");
+                Ok(response)
+            })
+        }
+    }
+
+    fn timed_response_target(
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: &'static [u8],
+    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
+        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let connector = SharedHttpConnector::new(TimedResponseConnector {
+            upload_delay,
+            response_delay,
+            prefix_bytes,
+            response: bytes::Bytes::from_static(response),
+            frames: Arc::clone(&frames),
+            dropped: Arc::clone(&dropped),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let (mut target, _) = header_recording_target_client(Vec::new());
+        target.timeouts = TargetRequestTimeouts {
+            request: Duration::from_secs(2),
+            complete: Duration::from_secs(10),
+            minimum_transfer_rate: 1024 * 1024,
+        };
+        target.client = Arc::new(S3Client::from_conf(
+            target
+                .client
+                .config()
+                .to_builder()
+                .http_client(http_client)
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+                .timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(target.timeouts.request)
+                        .build(),
+                )
+                .build(),
+        ));
+        (target, frames, dropped)
+    }
+
+    #[test]
+    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
+        let policy = TargetRequestTimeouts::default();
+        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
+        assert_eq!(
+            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
+            Duration::from_secs(15420)
+        );
+        assert!(policy.upload_timeout(-1).is_err());
+        let slow = TargetRequestTimeouts {
+            minimum_transfer_rate: 1,
+            ..policy
+        };
+        assert!(slow.upload_timeout(i64::MAX).is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
+        for operation in ["put", "forwarded-checksum", "part"] {
+            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
+            let body = vec![b'x'; 2 * 1024 * 1024];
+            let size = i64::try_from(body.len()).expect("size");
+            let started = tokio::time::Instant::now();
+            let error = if operation == "part" {
+                target
+                    .put_object_part(
+                        "bucket",
+                        "key",
+                        "upload",
+                        1,
+                        size,
+                        ByteStream::from(body),
+                        &PutObjectPartOptions::default(),
+                    )
+                    .await
+                    .expect_err("response deadline")
+            } else {
+                let mut opts = PutObjectOptions::default();
+                if operation == "forwarded-checksum" {
+                    opts.user_metadata.insert(
+                        "x-amz-checksum-sha256".to_owned(),
+                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
+                    );
+                }
+                target
+                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
+                    .await
+                    .expect_err("response deadline")
+            };
+            assert!(
+                started.elapsed() >= Duration::from_secs(6),
+                "upload gets its full independent window: {error}"
+            );
+            assert!(
+                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
+                "response must end before the larger total deadline"
+            );
+            assert!(
+                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
+                "trickle delivered progress before failure"
+            );
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
+        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
+        let body = vec![b'x'; 2 * 1024 * 1024];
+        let size = i64::try_from(body.len()).expect("size");
+        target
+            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
+            .await
+            .expect("slow upload, prompt response");
+        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
+        let response = target
+            .get_object("bucket", "key", None, None, None, HeaderMap::new())
+            .await
+            .expect("GET headers");
+        let started = tokio::time::Instant::now();
+        let body = response
+            .body
+            .collect()
+            .await
+            .expect("streamed GET remains outside upload deadline")
+            .into_bytes();
+        assert!(started.elapsed() > target.timeouts.request);
+        assert_eq!(&body[..], b"     payload");
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
+        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
+        for prefix in [5, 100] {
+            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
+            let started = tokio::time::Instant::now();
+            let result = target
+                .complete_multipart_upload(
+                    "bucket",
+                    "key",
+                    "upload",
+                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
+                    &PutObjectOptions::default(),
+                )
+                .await;
+            if prefix == 5 {
+                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
+            } else {
+                assert!(result.is_err());
+                assert!(started.elapsed() <= Duration::from_secs(11));
+            }
+            assert!(started.elapsed() > target.timeouts.request);
+            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_control_operation_deadline_covers_response_trickle() {
+        let (target, frames, dropped) =
+            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
+        let started = tokio::time::Instant::now();
+        assert!(target.get_bucket_versioning("bucket").await.is_err());
+        assert!(started.elapsed() <= Duration::from_secs(3));
+        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3279,6 +3597,7 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3591,6 +3910,7 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
+    if let Some(expires) = oi.expires {
+        meta.insert(
+            "expires".to_string(),
+            expires
+                .format(&time::format_description::well_known::Rfc3339)
+                .map_err(std::io::Error::other)?,
+        );
+    }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
         metadata_keys::RESTORE.to_string(),
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
-        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -7144,6 +7151,25 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
+        let unread = get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+            None,
+        )
+        .await
+        .expect("open an unread transitioned response");
+        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
+        drop(unread);
+        assert_eq!(
+            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
+            0,
+            "canceling an unread response must release the tier lease"
+        );
     }
 
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+/// Shared tag-entry validation for object requests and persisted bucket tags.
+pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
+    for tag in tag_set {
+        let key = tag
+            .key
+            .as_deref()
+            .filter(|key| !key.is_empty())
+            .ok_or("Tag key cannot be empty")?;
+        if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+            return Err("The TagKey you have provided is invalid");
+        }
+        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
+        if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+            return Err("The TagValue you have provided is invalid");
+        }
+        if !keys.insert(key) {
+            return Err("Cannot provide multiple Tags with the same key");
+        }
+    }
+    Ok(())
+}
+
+pub const MAX_BUCKET_TAGS: usize = 50;
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -990,6 +1022,20 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
+                if !data.is_empty() {
+                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
+                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
+                    })?;
+                    if tagging.tag_set.len() > MAX_BUCKET_TAGS {
+                        return Err(Error::InvalidArgument(
+                            self.name.clone(),
+                            String::new(),
+                            "Cannot have more than 50 tags per bucket".to_owned(),
+                        ));
+                    }
+                    validate_tag_set_entries(&tagging.tag_set)
+                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
+                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1784,6 +1830,31 @@ mod test {
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
+        let mut metadata = BucketMetadata::new("validated-tags");
+        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
+            .expect("empty value is valid");
+        let updated = metadata.tagging_config_updated_at;
+        for invalid in [
+            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
+            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
+        ] {
+            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
+            assert_eq!(metadata.tagging_config_xml, valid);
+            assert_eq!(metadata.tagging_config_updated_at, updated);
+        }
+        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
+            .expect("deletion repairs legacy corruption");
+        assert!(metadata.tagging_config_xml.is_empty());
+    }
+
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
+    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -690,7 +691,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
+        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -3427,7 +3428,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -4555,8 +4556,13 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(config_file, raw.to_vec())
-            .expect("raw config should be stored");
+        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
+            // Model legacy persisted corruption without passing the write-time validator.
+            bm.tagging_config_xml = raw.to_vec();
+        } else {
+            bm.update_config(config_file, raw.to_vec())
+                .expect("raw config should be stored");
+        }
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -5209,6 +5215,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "catalog-cleanup-incarnation";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("read bucket incarnation");
+        let called = std::sync::atomic::AtomicBool::new(false);
+        let result = store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                Uuid::new_v4(),
+                || {
+                    called.store(true, std::sync::atomic::Ordering::SeqCst);
+                    Ok(())
+                },
+            )
+            .await;
+        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
+        assert!(
+            !called.load(std::sync::atomic::Ordering::SeqCst),
+            "incarnation must be checked before the external fence"
+        );
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read unchanged bucket metadata");
+        assert!(
+            metadata.quota_config_json.is_empty(),
+            "a stale cleanup must not modify the replacement bucket"
+        );
+        store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                incarnation,
+                || Ok(()),
+            )
+            .await
+            .expect("the current incarnation permits the validated update");
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read updated bucket metadata");
+        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5682,7 +5743,8 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
+            let tagging =
+                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5755,7 +5817,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
+            if protocol == 1 {
+                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
+                assert!(
+                    matches!(
+                        store
+                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
+                            .await,
+                        Err(StorageError::SlowDown)
+                    ),
+                    "a missing authoritative ledger must not admit a write from scanner reporting"
+                );
+                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
+                    .await
+                    .expect("bootstrap authoritative usage through the production worker");
+            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/quota/reservation.rs

@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::disk::{DiskAPI, error::DiskError};
 use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
-use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
+use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::namespace::NamespaceLocking;
 use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
 use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::sync::Arc;
 use std::time::Duration;
 use time::OffsetDateTime;
@@ -36,6 +36,8 @@ use tracing::warn;
 use uuid::Uuid;
 
 const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
+const RECONCILE_RETRY_SECONDS: i64 = 30;
+const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
 const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
 const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
 const ORPHAN_PROBE_CONCURRENCY: usize = 32;
@@ -96,6 +98,10 @@ struct QuotaLedger {
     version: u8,
     bucket_incarnation: Uuid,
     quota_revision_unix_nanos: i128,
+    /// Changes on every durable write, including zero-growth reservations.
+    /// Missing on an older writer's output, which invalidates a concurrent scan.
+    #[serde(default)]
+    writer_revision: Uuid,
     accounted_usage: u64,
     reservations: BTreeMap<Uuid, PersistedReservation>,
     #[serde(default)]
@@ -110,6 +116,7 @@ impl QuotaLedger {
             version: QUOTA_LEDGER_FORMAT_VERSION,
             bucket_incarnation,
             quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
+            writer_revision: Uuid::nil(),
             accounted_usage,
             reservations: BTreeMap::new(),
             reconcile_required: false,
@@ -485,22 +492,23 @@ impl QuotaContext {
             let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
             let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
             fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
-            let mut ledger = load_current_ledger_locked(
-                Arc::clone(&store),
-                &ledger_data.bucket,
-                &ledger_data.ledger_object,
-                bucket_incarnation,
-                quota_revision,
-            )
-            .await?;
-
+            let mut ledger =
+                load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
+                    .await?;
+            let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
+            let Some(mut ledger) = ledger.take() else {
+                // Bootstrap has no authoritative usage yet. Release every
+                // request-owned fence by returning; the worker owns no request.
+                job.schedule();
+                return Err(StorageError::SlowDown);
+            };
             let growth = reservation.growth();
-            let mut current_usage = ledger.admitted_usage()?;
-            let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
-            if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
-                reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
-                current_usage = ledger.admitted_usage()?;
-                expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
+            let current_usage = ledger.admitted_usage()?;
+            let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
+            if ledger.reconcile_required
+                || (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
+            {
+                job.schedule();
             }
             if growth > 0 && expected_usage > quota_limit {
                 return Err(StorageError::QuotaExceeded {
@@ -518,7 +526,7 @@ impl QuotaContext {
                 });
             }
             ledger.reserve(operation_id, reservation)?;
-            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
+            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
 
             Ok(QuotaReservation {
                 ledger: Some(ReservationLedgerData::Single(ledger_data)),
@@ -1082,7 +1090,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
         fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
         let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
         ledger.mark_commit_started(data.operation_id, &data.reservation)?;
-        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
@@ -1113,7 +1121,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
         } else {
             ledger.abort(operation_id, &reservation)?;
         }
-        save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
@@ -1121,22 +1129,15 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
 
 async fn load_current_ledger_locked(
     store: Arc<ECStore>,
-    bucket: &str,
     ledger_object: &str,
     bucket_incarnation: Uuid,
     quota_revision: OffsetDateTime,
-) -> Result<QuotaLedger> {
-    match load_ledger_locked(Arc::clone(&store), ledger_object).await {
-        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
-        Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
-            let usage = exact_bucket_usage(&store, bucket).await?;
-            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
-        }
+) -> Result<Option<QuotaLedger>> {
+    match load_ledger_locked(store, ledger_object).await {
+        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
+        Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
         Ok(_) => Err(StorageError::PartMissingOrCorrupt),
-        Err(StorageError::ConfigNotFound) => {
-            let usage = exact_bucket_usage(&store, bucket).await?;
-            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
-        }
+        Err(StorageError::ConfigNotFound) => Ok(None),
         Err(err) => Err(err),
     }
 }
@@ -1245,15 +1246,11 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
     for (object, target) in orphan_commit_targets {
         fence_namespace_mutations(&store, bucket, &object, target).await?;
     }
-    let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
-    if ledger.reservations.is_empty() && ledger.reconcile_required {
-        reconcile_exact(&store, bucket, &mut ledger).await?;
-        removed = true;
-    }
+    let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
     if !removed && !cursor_changed {
         return Ok(());
     }
-    save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
+    save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
 }
 
 fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
@@ -1273,13 +1270,217 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
     Ok(removed)
 }
 
-async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
-    if !ledger.reservations.is_empty() {
-        return Err(StorageError::PartMissingOrCorrupt);
+#[derive(Debug, Serialize, Deserialize)]
+struct ReconcileControl {
+    bucket_incarnation: Uuid,
+    quota_revision_unix_nanos: i128,
+    next_attempt_at: i64,
+}
+
+#[derive(Clone)]
+struct ReconcileJob {
+    store: Arc<ECStore>,
+    bucket: String,
+    bucket_incarnation: Uuid,
+    quota_revision: OffsetDateTime,
+}
+
+type ReconcileKey = (Uuid, String, Uuid, i128);
+
+fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
+    static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
+    ACTIVE.get_or_init(Default::default)
+}
+
+struct ActiveReconciliation(ReconcileKey);
+
+impl Drop for ActiveReconciliation {
+    fn drop(&mut self) {
+        active_reconciliations()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .remove(&self.0);
+    }
+}
+
+impl ReconcileJob {
+    fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
+        Self {
+            store,
+            bucket,
+            bucket_incarnation,
+            quota_revision,
+        }
+    }
+
+    fn schedule(self) {
+        let key = (
+            self.store.id,
+            self.bucket.clone(),
+            self.bucket_incarnation,
+            self.quota_revision.unix_timestamp_nanos(),
+        );
+        if !active_reconciliations()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .insert(key.clone())
+        {
+            return;
+        }
+        let owner = ActiveReconciliation(key);
+        // The owner carries no object, metadata, lifecycle or reservation guard.
+        // Cancellation of the rejected request cannot cancel its reconciliation.
+        tokio::spawn(async move {
+            let _owner = owner;
+            if let Err(error) = self.run().await {
+                warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
+                    error = %error, "Quota reconciliation deferred");
+            }
+        });
+    }
+
+    fn control_object(&self) -> String {
+        format!(
+            "{}.reconcile/{}/{}",
+            ledger_object(&self.bucket),
+            self.bucket_incarnation,
+            self.quota_revision.unix_timestamp_nanos()
+        )
+    }
+
+    async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
+        if guard.is_lock_lost() {
+            return Err(StorageError::SlowDown);
+        }
+        let mut opts = ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        };
+        let _ = opts.set_quota_admission(0, u64::MAX);
+        opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
+        opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+        save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
+    }
+
+    async fn run(&self) -> Result<()> {
+        let control_object = self.control_object();
+        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
+        let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
+            return Ok(());
+        };
+        let guard = Arc::new(guard);
+        let now = now_unix();
+        match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
+            Ok(bytes) => {
+                let control: ReconcileControl = serde_json::from_slice(&bytes)?;
+                if control.bucket_incarnation != self.bucket_incarnation
+                    || control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
+                {
+                    return Err(StorageError::PartMissingOrCorrupt);
+                }
+                if control.next_attempt_at > now {
+                    return Ok(());
+                }
+            }
+            Err(StorageError::ConfigNotFound) => {}
+            Err(error) => return Err(error),
+        }
+        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
+        let mut control = ReconcileControl {
+            bucket_incarnation: self.bucket_incarnation,
+            quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
+            next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
+        };
+        // Persist the throttle before work, including bootstrap and failed scans.
+        self.save_control(&control_object, &control, &guard).await?;
+        let result = self.scan_and_publish(&guard).await;
+        control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
+        self.save_control(&control_object, &control, &guard).await?;
+        result
+    }
+
+    /// Take a short authoritative configuration snapshot without recursively
+    /// acquiring a reader behind a queued metadata/lifecycle writer.
+    async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
+        let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
+        let guard = snapshot
+            .metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
+            .ok_or(StorageError::PreconditionFailed)?;
+        let opts = ObjectOptions {
+            expected_bucket_incarnation_id: Some(self.bucket_incarnation),
+            object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
+            ..Default::default()
+        };
+        let (quota, incarnation, revision) =
+            metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
+                .await?;
+        if incarnation != self.bucket_incarnation
+            || revision != self.quota_revision
+            || !quota.as_ref().is_some_and(|quota| {
+                quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
+            })
+        {
+            return Err(StorageError::PreconditionFailed);
+        }
+        Ok(snapshot)
+    }
+
+    async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
+        let ledger_object = ledger_object(&self.bucket);
+        let expected_revision = {
+            // Lock order: reconcile owner, lifecycle, metadata, ledger.
+            // Request paths never wait for the reconcile owner.
+            let _configuration = self.configuration_snapshot().await?;
+            let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
+            let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+            match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
+                Ok(mut ledger) => {
+                    if !ledger.reservations.is_empty() {
+                        return Ok(());
+                    }
+                    // Seed a non-nil token even for a legacy ledger. An old
+                    // node drops this field on write and cannot pass the CAS.
+                    fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
+                    save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
+                    Some(ledger.writer_revision)
+                }
+                Err(StorageError::ConfigNotFound) => None,
+                Err(error) => return Err(error),
+            }
+        };
+        #[cfg(test)]
+        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
+        // Only the independent reconcile owner is held during this whole-bucket
+        // walk. Concurrent reserve/settle/reap writes invalidate its result.
+        let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
+        #[cfg(test)]
+        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
+        let configuration = self.configuration_snapshot().await?;
+        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+        let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
+            Ok(ledger) => {
+                expected_revision == Some(ledger.writer_revision)
+                    && !ledger.writer_revision.is_nil()
+                    && ledger.reservations.is_empty()
+            }
+            Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
+            Err(error) => return Err(error),
+        };
+        if !unchanged {
+            return Ok(());
+        }
+        if owner.is_lock_lost()
+            || !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
+        {
+            return Err(StorageError::SlowDown);
+        }
+        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
+        let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
+        save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
     }
-    ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
-    ledger.reconcile_required = false;
-    Ok(())
 }
 
 async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
@@ -1935,7 +2136,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
 async fn save_ledger_locked(
     store: Arc<ECStore>,
     ledger_object: &str,
-    ledger: &QuotaLedger,
+    ledger: &mut QuotaLedger,
     ledger_guard: &Arc<NamespaceLockGuard>,
 ) -> Result<()> {
     if ledger_guard.is_lock_lost() {
@@ -1959,9 +2160,71 @@ async fn save_ledger_locked(
     let _ = opts.set_quota_admission(0, u64::MAX);
     opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
     opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+    ledger.writer_revision = Uuid::new_v4();
     save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
 }
 
+/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
+/// The production reaper, ownership fence, scan and compare-and-swap still run.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
+    let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
+    let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
+    let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let active = active_reconciliations()
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .contains(&key);
+            if !active {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .map_err(|_| StorageError::SlowDown)?;
+    reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
+    let object = job.control_object();
+    {
+        let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+        match read_config_no_lock(Arc::clone(&store), &object).await {
+            Ok(bytes) => {
+                let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
+                control.next_attempt_at = 0;
+                fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
+                job.save_control(&object, &control, &guard).await?;
+            }
+            Err(StorageError::ConfigNotFound) => {}
+            Err(error) => return Err(error),
+        }
+    }
+    job.run().await
+}
+
+/// Bootstrap a test bucket through the real background reconciliation path.
+/// Fixtures call this after enabling V1 quota, before issuing writes whose
+/// contract is independent of the initial retryable admission response.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
+    let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
+    if !quota
+        .as_ref()
+        .is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
+    {
+        return Ok(());
+    }
+    ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
+        .run()
+        .await?;
+    match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
+        Some(_) => Ok(()),
+        None => Err(StorageError::SlowDown),
+    }
+}
+
 #[cfg(any(test, feature = "test-util"))]
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub fn fail_next_quota_ledger_save_for_test() {
@@ -2003,6 +2266,434 @@ mod tests {
     use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
     use serial_test::serial;
 
+    #[derive(Clone, Copy, PartialEq, Eq)]
+    pub(super) enum ReconcilePauseStage {
+        BeforeScan,
+        BeforePublish,
+    }
+
+    struct ReconcilePause {
+        bucket: String,
+        stage: ReconcilePauseStage,
+        armed: std::sync::atomic::AtomicBool,
+        scans: std::sync::atomic::AtomicUsize,
+        fail_scan: std::sync::atomic::AtomicBool,
+        reached: tokio::sync::Notify,
+        release: tokio::sync::Notify,
+    }
+
+    static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
+
+    struct ReconcilePauseGuard(Arc<ReconcilePause>);
+
+    impl ReconcilePauseGuard {
+        fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
+            let pause = Arc::new(ReconcilePause {
+                bucket: bucket.to_owned(),
+                stage,
+                armed: true.into(),
+                scans: 0.into(),
+                fail_scan: false.into(),
+                reached: tokio::sync::Notify::new(),
+                release: tokio::sync::Notify::new(),
+            });
+            *RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
+            Self(pause)
+        }
+
+        async fn reached(&self) {
+            tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
+                .await
+                .expect("reconciliation reached barrier");
+        }
+    }
+
+    impl Drop for ReconcilePauseGuard {
+        fn drop(&mut self) {
+            self.0.release.notify_one();
+            let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
+            if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
+                *hook = None;
+            }
+        }
+    }
+
+    pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
+        use std::sync::atomic::Ordering;
+        let hook = RECONCILE_PAUSE
+            .lock()
+            .expect("reconcile hooks")
+            .as_ref()
+            .filter(|hook| hook.bucket == bucket)
+            .cloned();
+        if let Some(hook) = hook {
+            if stage == ReconcilePauseStage::BeforeScan {
+                hook.scans.fetch_add(1, Ordering::SeqCst);
+            }
+            if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
+                hook.reached.notify_one();
+                hook.release.notified().await;
+            }
+            if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
+                return Err(StorageError::other("injected quota usage failure"));
+            }
+        }
+        Ok(())
+    }
+
+    async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
+        use crate::object_api::PutObjReader;
+        use crate::storage_api_contracts::object::ObjectIO;
+        let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
+        metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+        let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket");
+        let mut seed = PutObjReader::from_vec(vec![1; 10]);
+        store
+            .put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
+            .await
+            .expect("existing usage");
+        store
+            .update_bucket_metadata_config(
+                &bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
+            )
+            .await
+            .expect("enable quota");
+        let (_, incarnation, revision) = store
+            .bucket_quota_and_incarnation_from_disk(&bucket)
+            .await
+            .expect("quota identity");
+        (directories, ReconcileJob::new(store, bucket, incarnation, revision))
+    }
+
+    async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
+        begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
+            .await?
+            .reserve(old, new)
+            .await
+    }
+
+    async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
+        load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
+            .await
+            .expect("ledger")
+    }
+
+    async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
+        let key = (
+            job.store.id,
+            job.bucket.clone(),
+            job.bucket_incarnation,
+            job.quota_revision.unix_timestamp_nanos(),
+        );
+        tokio::time::timeout(Duration::from_secs(10), async {
+            while active_reconciliations().lock().expect("active workers").contains(&key) {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("owned reconciliation finished");
+    }
+
+    async fn expire_reconciliation_backoff(job: &ReconcileJob) {
+        drain_reconciliation_scheduler(job).await;
+        let object = job.control_object();
+        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
+        let mut control: ReconcileControl =
+            serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
+                .expect("control JSON");
+        control.next_attempt_at = 0;
+        fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
+            .await
+            .expect("control fence");
+        job.save_control(&object, &control, &guard).await.expect("advance test clock");
+    }
+
+    async fn wait_for_reconcile_owner(job: &ReconcileJob) {
+        let object = job.control_object();
+        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
+        let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
+            .await
+            .expect("worker finished")
+            .expect("owner guard");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
+        use std::sync::atomic::Ordering;
+        let (_directories, job) = reconciliation_fixture(10).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        assert!(
+            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
+            "unknown usage cannot admit a reservation"
+        );
+        pause.reached().await;
+        assert!(
+            matches!(
+                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+                Err(StorageError::ConfigNotFound)
+            ),
+            "bootstrap must not publish zero usage"
+        );
+        assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
+        // Bypass the local scheduler, exercising the cross-node ownership boundary.
+        tokio::time::timeout(Duration::from_secs(2), job.run())
+            .await
+            .expect("contender must not wait for a scan")
+            .expect("another owner is active");
+        let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
+            .await
+            .expect("scan does not retain lifecycle readers")
+            .expect("lifecycle writer");
+        let metadata = tokio::time::timeout(
+            Duration::from_secs(2),
+            metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
+        )
+        .await
+        .expect("scan does not retain metadata readers")
+        .expect("metadata writer");
+        drop(metadata);
+        drop(lifecycle);
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&job).await;
+        let ledger = read_test_ledger(&job).await;
+        assert_eq!(ledger.accounted_usage, 10);
+        assert!(ledger.reservations.is_empty());
+        assert!(!ledger.writer_revision.is_nil());
+        for _ in 0..3 {
+            assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
+            job.run().await.expect("persisted cooldown");
+        }
+        drain_reconciliation_scheduler(&job).await;
+        assert_eq!(
+            pause.0.scans.load(Ordering::SeqCst),
+            1,
+            "denied requests cannot repeatedly scan a full bucket"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
+        let (_directories, job) = reconciliation_fixture(100).await;
+        job.run().await.expect("bootstrap");
+        for mutation in ["abort", "commit", "reap", "legacy"] {
+            expire_reconciliation_backoff(&job).await;
+            let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+            let worker = tokio::spawn({
+                let job = job.clone();
+                async move { job.run().await }
+            });
+            pause.reached().await;
+            let initial = read_test_ledger(&job).await.writer_revision;
+            if mutation == "legacy" {
+                let object = ledger_object(&job.bucket);
+                let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
+                let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
+                let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
+                document.as_object_mut().expect("ledger map").remove("writer_revision");
+                fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
+                    .await
+                    .expect("ledger fence");
+                let mut opts = ObjectOptions {
+                    max_parity: true,
+                    no_lock: true,
+                    ..Default::default()
+                };
+                let _ = opts.set_quota_admission(0, u64::MAX);
+                opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
+                opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+                save_config_with_opts(
+                    Arc::clone(&job.store),
+                    &object,
+                    serde_json::to_vec(&document).expect("old node JSON"),
+                    &opts,
+                )
+                .await
+                .expect("old node drops new field");
+            } else {
+                let mut reservation = quota_reservation(&job, 10, 10)
+                    .await
+                    .expect("zero-growth admission while scanning");
+                let reserved = read_test_ledger(&job).await.writer_revision;
+                assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
+                reservation.mark_commit_started().await.expect("commit marker");
+                assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
+                match mutation {
+                    "abort" => reservation.abort().await,
+                    "commit" => reservation.commit().await,
+                    "reap" => {
+                        reservation.defer_after_fence();
+                        reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
+                            .await
+                            .expect("reap orphan");
+                    }
+                    _ => unreachable!(),
+                }
+            }
+            let written = read_test_ledger(&job).await;
+            assert!(written.reservations.is_empty());
+            assert_ne!(written.writer_revision, initial);
+            pause.0.release.notify_one();
+            worker.await.expect("worker join").expect("stale scan discarded");
+            let current = read_test_ledger(&job).await;
+            assert_eq!(
+                current.writer_revision, written.writer_revision,
+                "{mutation}: stale usage must not replace a concurrent writer"
+            );
+            assert_eq!(current.accounted_usage, written.accounted_usage);
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
+        use std::sync::atomic::Ordering;
+        let (_directories, job) = reconciliation_fixture(10).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        pause.0.fail_scan.store(true, Ordering::SeqCst);
+        pause.0.release.notify_one();
+        assert!(job.run().await.is_err());
+        assert!(matches!(
+            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+            Err(StorageError::ConfigNotFound)
+        ));
+        job.run().await.expect("failure cooldown");
+        assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
+        pause.0.fail_scan.store(false, Ordering::SeqCst);
+        expire_reconciliation_backoff(&job).await;
+        job.run().await.expect("recover after failure");
+        assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
+        job.store
+            .delete_object(&job.bucket, "seed", ObjectOptions::default())
+            .await
+            .expect("delete stored usage");
+        assert!(
+            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
+            "stale usage remains conservative during cooldown"
+        );
+        expire_reconciliation_backoff(&job).await;
+        job.run().await.expect("refresh deleted usage");
+        assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
+        quota_reservation(&job, 0, 1)
+            .await
+            .expect("delete must not leave bucket permanently full")
+            .abort()
+            .await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
+        let (_directories, job) = reconciliation_fixture(100).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+        let caller = tokio::spawn({
+            let job = job.clone();
+            async move {
+                job.schedule();
+                std::future::pending::<()>().await
+            }
+        });
+        pause.reached().await;
+        caller.abort();
+        assert!(caller.await.expect_err("cancel caller").is_cancelled());
+        job.store
+            .update_bucket_metadata_config(
+                &job.bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
+            )
+            .await
+            .expect("quota changes while scanning");
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&job).await;
+        assert!(
+            matches!(
+                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+                Err(StorageError::ConfigNotFound)
+            ),
+            "old config cannot publish bootstrap usage"
+        );
+        drop(pause);
+        let (_, incarnation, revision) = job
+            .store
+            .bucket_quota_and_incarnation_from_disk(&job.bucket)
+            .await
+            .expect("current configuration");
+        let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        let caller = tokio::spawn({
+            let job = current.clone();
+            async move {
+                job.schedule();
+                std::future::pending::<()>().await
+            }
+        });
+        pause.reached().await;
+        caller.abort();
+        assert!(caller.await.expect_err("cancel new caller").is_cancelled());
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&current).await;
+        assert!(
+            read_test_ledger(&current).await.matches(incarnation, revision),
+            "owned scan publishes after its waiter is canceled"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
+        use crate::storage_api_contracts::bucket::DeleteBucketOptions;
+        let (_directories, job) = reconciliation_fixture(100).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+        let worker = tokio::spawn({
+            let job = job.clone();
+            async move { job.run().await }
+        });
+        pause.reached().await;
+        job.store
+            .delete_object(&job.bucket, "seed", ObjectOptions::default())
+            .await
+            .expect("empty old bucket");
+        job.store
+            .delete_bucket(&job.bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("retire bucket while scan is paused");
+        job.store
+            .make_bucket(&job.bucket, &MakeBucketOptions::default())
+            .await
+            .expect("replacement bucket");
+        job.store
+            .update_bucket_metadata_config(
+                &job.bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
+            )
+            .await
+            .expect("replacement quota");
+        pause.0.release.notify_one();
+        assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
+        assert!(matches!(
+            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+            Err(StorageError::ConfigNotFound)
+        ));
+        drop(pause);
+        initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
+            .await
+            .expect("replacement can bootstrap independently");
+        let replacement = read_test_ledger(&job).await;
+        assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
+        assert_eq!(replacement.accounted_usage, 0);
+    }
+
     fn ledger(accounted_usage: u64) -> QuotaLedger {
         QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
     }
@@ -2090,8 +2781,7 @@ mod tests {
             .expect("persist crash-left reservation fixture");
 
         assert!(matches!(
-            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
-                .await,
+            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
             Err(StorageError::PartMissingOrCorrupt)
         ));
         assert!(matches!(
@@ -2109,10 +2799,10 @@ mod tests {
             Err(StorageError::ConfigNotFound)
         ));
         let replacement =
-            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
+            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
                 .await
                 .expect("new quota admission must not inherit a fresh stale reservation");
-        assert!(replacement.reservations.is_empty());
+        assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
     }
 
     #[test]

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
+use aws_smithy_runtime_api::client::interceptors::context::{
+    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
+};
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
+    /// Entire SDK call, including non-streaming response bodies and retries.
+    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
+/// Only installed on non-streaming upload operations. The budget begins at
+/// response headers, independently of the potentially long request upload.
+#[derive(Debug)]
+pub(crate) struct UploadResponseDeadline(pub Duration);
+
+impl Intercept for UploadResponseDeadline {
+    fn name(&self) -> &'static str {
+        "RustfsUploadResponseDeadline"
+    }
+
+    fn modify_before_deserialization(
+        &self,
+        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
+        _runtime_components: &RuntimeComponents,
+        _cfg: &mut ConfigBag,
+    ) -> Result<(), BoxError> {
+        let response = context.response_mut();
+        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
+        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
+            inner: Some(inner),
+            deadline: Box::pin(tokio::time::sleep(self.0)),
+        });
+        Ok(())
+    }
+}
+
+struct UploadResponseBody {
+    inner: Option<SdkBody>,
+    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl http_body::Body for UploadResponseBody {
+    type Data = bytes::Bytes;
+    type Error = BoxError;
+
+    fn poll_frame(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        use std::future::Future as _;
+        use std::task::Poll;
+        if self.inner.is_none() {
+            return Poll::Ready(None);
+        }
+        // The absolute response budget never renews on data, empty frames,
+        // or trailers. A retry gets its own bounded response window.
+        if self.deadline.as_mut().poll(cx).is_ready() {
+            self.inner.take();
+            return Poll::Ready(Some(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "replication response body deadline exceeded",
+            )
+            .into())));
+        }
+        let Some(inner) = self.inner.as_mut() else {
+            return Poll::Ready(None);
+        };
+        let result = std::pin::Pin::new(inner).poll_frame(cx);
+        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
+            self.inner.take();
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
+    }
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
+    }
+}
+
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
+        if let Some(operation_timeout) = spec.operation_timeout {
+            timeouts = timeouts.operation_timeout(operation_timeout);
+        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -674,6 +753,7 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1027,11 +1107,16 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
+        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
+        assert_eq!(
+            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
+            Some(Duration::from_secs(12))
+        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
-    replication_statuses_map, version_purge_statuses_map,
+    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
+    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A version that is already queued or being uploaded is not driven a
+        // A source generation already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
+            replay_mrf_journal(&storage, None).await;
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name and version. `queue_replica_task` admits a version only once
+/// object name, version and source generation. `queue_replica_task` admits a generation only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
+    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ReplicationInFlightKey {
+    bucket: String,
+    object: String,
+    version_id: Option<uuid::Uuid>,
+    generation: ReplicationGenerationSnapshot,
+}
+
+impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
+    fn from(info: &ReplicateObjectInfo) -> Self {
+        Self {
+            bucket: info.bucket.clone(),
+            object: info.name.clone(),
+            version_id: info.version_id,
+            generation: info.replication_generation.clone(),
+        }
+    }
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; `false` when the same version is already in flight.
+    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
+        self.lock().insert(ReplicationInFlightKey::from(ri))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
+        self.lock().remove(&ReplicationInFlightKey::from(ri));
     }
 
     #[cfg(test)]
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicateObjectInfo,
+    key: ReplicationInFlightKey,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicateObjectInfo {
-                bucket: ri.bucket.clone(),
-                name: ri.name.clone(),
-                version_id: ri.version_id,
-                ..Default::default()
-            },
+            key: ReplicationInFlightKey::from(ri),
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.finish(&self.key);
+        self.in_flight.lock().remove(&self.key);
     }
 }
 
@@ -3421,6 +3525,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3594,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3675,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3806,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -3929,6 +4068,48 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, mut rx) = mpsc::channel(4);
+        pool.workers.write().await.push(tx);
+        let initial = ReplicateObjectInfo {
+            bucket: "pending-tags-bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            size: 4096,
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let metadata = ReplicateObjectInfo {
+            op_type: ReplicationType::Metadata,
+            user_tags: String::new(),
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..initial.clone()
+        };
+        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        assert_eq!(pool.in_flight.len(), 2);
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
+        assert_eq!(pool.in_flight.len(), 1);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        for expected in [&initial, &metadata] {
+            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
+                panic!("expected object replication");
+            };
+            assert_eq!(queued.replication_generation, expected.replication_generation);
+        }
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
+        assert_eq!(pool.in_flight.len(), 0);
+    }
+
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4171,6 +4352,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4404,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5584,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6550,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                arn = %dobj.target_arn,
-                object = %dobj.delete_object.object_name,
-                operation = "apply_replication_delete_state",
-                error = %e,
-                "Replication target operation failed"
-            );
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
+                error!(
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %bucket,
+                    arn = %dobj.target_arn,
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -6604,6 +6609,7 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
+            timeouts: Default::default(),
         })
     }
 

crates/ecstore/src/bucket/utils.rs

@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
+    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
+    }
+
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
+/// New public keys must remain representable in an unencoded XML listing.
+/// Existing keys still use the read/delete validators so they can be removed.
+pub fn has_xml_invalid_control_characters(object: &str) -> bool {
+    object
+        .chars()
+        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
+}
+
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
+    pub hidden_entry_budget: usize,
+    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
+            hidden_entry_budget: self.hidden_entry_budget,
+            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
+        hidden_entry_budget: opts.hidden_entry_budget,
+        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
+    /// Inclusive sorted boundary proved by every reader before a bounded stop.
+    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -722,6 +730,8 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
+        let mut scan_cut = None;
+        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -733,7 +743,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -793,7 +803,12 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
+                                if r.limit_reached() == Some(true) {
+                                    limit_reached = true;
+                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
+                                        DiskError::other("bounded listing reader ended without a scan boundary")
+                                    })?);
+                                }
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -867,7 +882,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     }
                 };
 
@@ -926,7 +941,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -976,6 +991,14 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
+            // The next head from a bounded reader is unknown. The previous
+            // globally consumed group is a safe cut; consuming another reader's
+            // later head would turn this absence of evidence into a skipped key.
+            // Check quorum failures above before accepting the bounded result.
+            if limit_reached {
+                break;
+            }
+
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -983,7 +1006,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -994,6 +1017,7 @@ async fn list_path_raw_inner(
                 break;
             }
 
+            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
                     if !r.skip_current() {
@@ -1005,7 +1029,7 @@ async fn list_path_raw_inner(
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1029,12 +1053,12 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
+        Ok(ListPathRawOutcome { limit_reached, scan_cut })
     });
 
     let merge_started = std::time::Instant::now();
@@ -1199,6 +1223,24 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
+    #[test]
+    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
+        let request = ListPathRawOptions {
+            bucket: "versioned".to_owned(),
+            hidden_entry_budget: 100,
+            pending_prefix: Some("pending/".to_owned()),
+            forward_to: Some("pending/0099".to_owned()),
+            ..Default::default()
+        };
+        // Primary and fallback readers both construct their walk from the
+        // per-job clone, rather than directly from the request options.
+        let job = request.clone();
+        let walk = walk_dir_options(&job);
+        assert_eq!(walk.hidden_entry_budget, 100);
+        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
+        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1316,6 +1358,7 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
+        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -1590,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)
+    check_and_record_tonic_nonce(nonce, signed_at, path)?;
+    // A body-bound request also carries a usable v2 signature. Consume both
+    // nonces so stripping the replay-scope headers cannot authorize it again.
+    if content_sha256 != UNSIGNED_PAYLOAD {
+        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
+    }
+    Ok(())
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2581,6 +2585,43 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
+    #[test]
+    fn replay_scope_consumes_the_body_bound_v2_nonce() {
+        ensure_test_rpc_secret();
+        let audience = "node-a:9000";
+        let path = "/node_service.NodeService/RecoverDeleteTransaction";
+        let digest = "a".repeat(64);
+        for v3_first in [true, false] {
+            let mut headers =
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
+                    .expect("body-bound v2 headers should build");
+            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
+            headers.extend(
+                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
+                    .expect("replay-scope headers should build"),
+            );
+            let mut downgraded = headers.clone();
+            for header in [
+                RPC_REPLAY_SCOPE_VERSION_HEADER,
+                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+                RPC_REPLAY_SCOPE_NONCE_HEADER,
+                RPC_BOOT_EPOCH_HEADER,
+            ] {
+                downgraded.remove(header);
+            }
+            let (first, replay) = if v3_first {
+                (&headers, &downgraded)
+            } else {
+                (&downgraded, &headers)
+            };
+            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
+                .expect("the first delivery must be accepted");
+            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
+                .expect_err("a mutation cannot replay through the other signature generation");
+            assert_eq!(error.to_string(), "RPC request replay detected");
+        }
+    }
+
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2014,9 +2014,21 @@ impl PeerRestClient {
             }
             result
         })
-        .await
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
-        self.finalize_result(result).await
+        .await;
+        self.settle_bucket_metadata_reload(bucket, result).await
+    }
+
+    async fn settle_bucket_metadata_reload(
+        &self,
+        bucket: &str,
+        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
+    ) -> Result<()> {
+        match result {
+            Ok(result) => self.finalize_result(result).await,
+            // A local operation deadline does not establish that the peer is
+            // unreachable. In particular, it must not gate IAM revocations.
+            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
+        }
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4753,6 +4765,31 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
+    #[tokio::test]
+    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
+        let client = test_peer_client();
+        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
+        let error = client
+            .settle_bucket_metadata_reload("slow-policy", elapsed)
+            .await
+            .expect_err("local budget expires");
+        assert!(error.to_string().contains("timed out"));
+        assert!(
+            !client.offline.load(Ordering::Acquire),
+            "local work must not suppress subsequent IAM RPCs"
+        );
+        client
+            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
+            .await
+            .expect("peer reply");
+        let error = client
+            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
+            .await
+            .expect_err("real transport failure");
+        assert!(error.to_string().contains("connection refused"));
+        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
+    }
+
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1082,7 +1082,10 @@ impl RemoteDisk {
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
+        let settle_timeout = if matches!(
+            &operation,
+            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
+        ) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1637,6 +1640,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1712,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1783,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2015,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2030,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2711,6 +2719,54 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
+        let expected_disk = self
+            .id
+            .lock()
+            .await
+            .filter(|id| !id.is_nil())
+            .ok_or(DiskError::DiskNotFound)?;
+        let query = MultipartIndexRpcRequest::new(bucket, marker);
+        let endpoint = self.endpoint.to_string();
+        let canonical = query.canonical_request(&endpoint)?;
+        let opts = serde_json::to_string(&query)?;
+        self.execute_with_timeout(
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(DiskInfoRequest {
+                    disk: endpoint.clone(),
+                    opts: opts.clone(),
+                });
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
+                    return Err(DiskError::FileCorrupt);
+                }
+                // An old server ignores the additive query and returns ordinary DiskInfo.
+                // It cannot satisfy the dedicated reply schema or the fresh challenge.
+                let reply: MultipartIndexRpcReply =
+                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
+                reply.validate(&query, expected_disk)?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                    &query.canonical_response(&endpoint, &reply.page)?,
+                    &reply.proof,
+                )
+                .map_err(Error::other)?;
+                Ok(reply.page)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4294,6 +4350,35 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        // Older peers reject this variant. Do not retry through the unbounded protocol.
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::DiscoverBounded {
+                    object: object.to_owned(),
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4329,6 +4414,58 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        match self
+            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::InspectBounded {
+                    object: object.to_owned(),
+                    transaction,
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4849,6 +4986,113 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("test peer must bind");
+        let directory = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("local disk"),
+        );
+        disk.make_volume("bucket").await.expect("bucket");
+        let prefix = directory.path().join("bucket/prefix");
+        std::fs::create_dir(&prefix).expect("prefix");
+        for index in 0..64 {
+            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+        }
+        let fixture = Arc::new(TestBoundedDiscovery {
+            disk: Arc::clone(&disk),
+            response_gate: TestFormatReadGate::default(),
+        });
+        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
+        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
+            .await
+            .expect("disk completed bounded scan");
+        task.abort();
+        assert!(task.await.expect_err("caller cancelled").is_cancelled());
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        fixture.response_gate.release.cancel();
+        peer.stop().await;
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        assert!(prefix.join("child-63").is_dir());
+    }
+
+    #[tokio::test]
+    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        remote
+            .set_disk_id(Some(Uuid::new_v4()))
+            .await
+            .expect("expected disk identity");
+        assert!(matches!(
+            remote.list_multipart_index_roots("bucket", None).await,
+            Err(DiskError::MethodNotAllowed)
+        ));
+        assert_eq!(
+            peer.peer.disk_info_calls.load(Ordering::Acquire),
+            1,
+            "exercise the actual legacy JSON handler"
+        );
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        assert_eq!(
+            remote
+                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
+                .await
+                .expect_err("old peer cannot prove a bounded inventory"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        let transaction = Uuid::new_v4();
+        assert_eq!(
+            remote
+                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
+                .await
+                .expect_err("old peer cannot inspect with a budget"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        assert_eq!(
+            remote
+                .settle_delete_transaction_bounded(
+                    "bucket",
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "prefix".to_owned(),
+                        transaction,
+                        expected_fingerprint: [0; 32],
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: 65536,
+                    },
+                    None
+                )
+                .await
+                .expect_err("old peer cannot settle with a budget"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods"),
+            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5741,9 +5985,16 @@ mod tests {
         format_data: Bytes,
         read_all_data: Bytes,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
+    #[derive(Debug)]
+    struct TestBoundedDiscovery {
+        disk: Arc<crate::disk::local::LocalDisk>,
+        response_gate: TestFormatReadGate,
+    }
+
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -5760,6 +6011,7 @@ mod tests {
                 format_data,
                 read_all_data,
                 next_format_read_gate: Arc::default(),
+                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5859,6 +6111,56 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
+                "/node_service.NodeService/RecoverDeleteTransaction"
+                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
+                {
+                    #[derive(Clone)]
+                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
+                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
+                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
+                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
+                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
+                            let fixture = Arc::clone(&self.0);
+                            Box::pin(async move {
+                                let request = request.into_inner();
+                                let operation: DeleteRecoveryRpcOperation =
+                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
+                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
+                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
+                                    return Err(Status::invalid_argument("unexpected fallback"));
+                                };
+                                let page = fixture
+                                    .disk
+                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
+                                    .await
+                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
+                                // Lose the response only after the real disk operation has completed.
+                                fixture.response_gate.entered.notify_one();
+                                fixture.response_gate.release.cancelled().await;
+                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
+                                    .map_err(|_| Status::internal("encode bounded reply"))?;
+                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
+                                    success: true,
+                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                                    result: result.into(),
+                                    error: None,
+                                }))
+                            })
+                        }
+                    }
+                    let fixture = self
+                        .peer
+                        .bounded_discovery
+                        .lock()
+                        .expect("bounded fixture")
+                        .clone()
+                        .expect("configured fixture");
+                    Box::pin(async move {
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
+                    })
+                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,6 +445,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -672,6 +676,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
     data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
 }
 
-/// The storage error of a version copy that kept timing out on a namespace
-/// lock through its in-place retries, kept typed so the entry failure policy
-/// defers it.
+/// A copy whose object namespace lock remained unavailable through its
+/// in-place retries, retained as a typed deferral.
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
-struct DecommissionCopyLockTimeout(#[source] Error);
+struct DecommissionCopyLockUnavailable(#[source] Error);
 
-fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
-    let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
-    matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
+fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
+    let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
+    match source {
+        Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "read",
+            bucket: lock_bucket,
+            object: lock_object,
+            ..
+        } => {
+            data_movement::data_movement_error_is_stage(err, "prepare_put_object")
+                && lock_bucket == bucket
+                && lock_object == object
+        }
+        _ => false,
+    }
 }
 
-/// A version copy that timed out acquiring a namespace lock leaves the source
+/// A version copy that could not acquire its namespace lock leaves the source
 /// in place; the next attempt copies the version again or finds an equivalent
 /// committed target. The lock is held by another mutation of the key or, when
 /// this node restarted mid-copy, by its previous incarnation until that lock's
 /// lease expires on the peers. Either clears by waiting, so the entry is
 /// deferred instead of being counted as a permanent per-object failure.
-fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
+fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement::data_movement_context_error(
-        format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
-        DecommissionCopyLockTimeout(err),
+        format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
+        DecommissionCopyLockUnavailable(err),
     )
 }
 
-fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
-    data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
+fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
+    data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
 }
 
 /// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
         return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
     }
     error
-        .take_if(|err| is_decommission_copy_lock_timeout_error(err))
-        .map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
+        .take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
+        .map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
 }
 
 fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
 /// entry is done and must not be counted as a failure or escalated. A quorum
 /// failure of the source cleanup is deferred too: the copied source is still
 /// in place and the next attempt cleans it up again. So is a version copy that
-/// lost drive quorum or timed out on a namespace lock through its in-place
+/// lost drive quorum or could not acquire an object namespace lock through its in-place
 /// retries: the source still holds the version and the next attempt copies it
 /// again. Every other failure stays terminal.
 fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
     if is_decommission_source_cleanup_retryable_error(&err)
         || is_decommission_copy_quorum_deferral(&err)
-        || is_decommission_copy_lock_timeout_deferral(&err)
+        || is_decommission_copy_lock_unavailable_deferral(&err)
     {
         return Ok(DecommissionEntryOutcome::Deferred(err));
     }
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -14534,6 +14606,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14544,10 +14633,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -28353,7 +28442,7 @@ mod tests {
             "one version must commit before the next version times out"
         );
         assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
             "{outcome:?}"
         );
         assert_eq!(
@@ -28387,6 +28476,88 @@ mod tests {
         assert_eq!(actual, b"written after versioning was enabled");
     }
 
+    /// A target lookup that loses its read-lock quorum must retain the source
+    /// and resume an entry after an earlier version has already committed.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
+        use tokio::io::AsyncReadExt as _;
+
+        let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
+        let object = fixture.object;
+        let attempted = Arc::new(AtomicUsize::new(0));
+        let fault = DecommissionTestFaultGuard::install_with_error(
+            Arc::new({
+                let attempted = Arc::clone(&attempted);
+                move |stage, _, faulted, _, success| {
+                    stage == "before_target_commit"
+                        && faulted == object
+                        && success
+                        && attempted.fetch_add(1, Ordering::SeqCst) > 0
+                }
+            }),
+            Some(Arc::new(|_, bucket, object| {
+                data_movement::data_movement_stage_error_for_test(
+                    "decommission_object",
+                    "prepare_put_object",
+                    bucket,
+                    object,
+                    Error::NamespaceLockQuorumUnavailable {
+                        mode: "read",
+                        bucket: bucket.to_owned(),
+                        object: object.to_owned(),
+                        required: 1,
+                        achieved: 0,
+                    },
+                )
+            })),
+        );
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+            "{outcome:?}"
+        );
+        fixture.assert_source_retained().await;
+        assert_eq!(
+            fixture
+                .target_versions()
+                .await
+                .expect("first version committed")
+                .versions
+                .len(),
+            1
+        );
+        assert_eq!(
+            fixture.store.pool_meta.read().await.pools[0]
+                .decommission
+                .as_ref()
+                .expect("active decommission")
+                .items_decommission_failed,
+            0
+        );
+
+        drop(fault);
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
+        fixture.assert_moved().await;
+        let mut reader = fixture.store.pools[1]
+            .get_object_reader(
+                fixture.bucket,
+                fixture.object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the newer version must remain readable");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
+        assert_eq!(actual, b"written after versioning was enabled");
+    }
+
     /// A namespace lock still held on the target key, as by a copy whose node
     /// was killed mid-write, makes target selection fail with a typed
     /// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28429,7 +28600,7 @@ mod tests {
 
             let (outcome, _) = fixture.run_entry().await;
             assert!(
-                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
                 "{outcome:?}"
             );
             assert_eq!(
@@ -28838,17 +29009,17 @@ mod tests {
             timeout(),
         );
         for err in [timeout(), prepare] {
-            assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
+            assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
             assert!(!is_decommission_copy_quorum_error(&err), "{err}");
             let mut error = Some(err);
             let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
                 .expect("a lock timeout copy must be deferred");
             assert!(error.is_none());
-            assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
+            assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
             assert!(!is_decommission_copy_quorum_deferral(&deferred));
-            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
+            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
                 .expect("the deferred error must keep its lock-timeout origin");
-            assert!(is_decommission_copy_lock_timeout_error(&failure.0));
+            assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
             assert!(matches!(
                 resolve_decommission_entry_failure(deferred),
                 Ok(DecommissionEntryOutcome::Deferred(_))
@@ -28873,6 +29044,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
+        let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
+            mode,
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            required: 1,
+            achieved: 0,
+        };
+        let wrap = |stage, err| {
+            data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
+        };
+        let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
+        let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
+            .expect("matching read-lock quorum must defer");
+        assert!(error.is_none());
+        assert!(matches!(
+            resolve_decommission_entry_failure(deferred),
+            Ok(DecommissionEntryOutcome::Deferred(_))
+        ));
+        for err in [
+            quorum("read", "bucket-a", "object-a"),
+            wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
+            wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
+            wrap(
+                "prepare_put_object",
+                quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
+            ),
+            wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
+            wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
+        ] {
+            let mut error = Some(err);
+            assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
+            assert!(error.is_some(), "unrelated lock failures must stay terminal");
+        }
+    }
+
     #[test]
     fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
         let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29351,6 +29559,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();

crates/ecstore/src/core/sets.rs

@@ -317,8 +317,9 @@ impl Sets {
         &self.ctx
     }
 
-    #[cfg(test)]
-    pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
+    /// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn stop_endpoint_monitor_for_test(&self) {
         if let Some(exit_signal) = &self.exit_signal {
             let _ = exit_signal.send(());
             // Wait for an in-flight reconnect pass as well as its receiver to exit.

crates/ecstore/src/data_movement/mod.rs

@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
+    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -626,6 +627,7 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
+        stage: None,
         source: Box::new(err),
     })
 }
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+    Error::other(DataMovementStageError {
+        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
+        stage: Some(stage.to_owned()),
+        source: Box::new(err),
+    })
+}
+
+pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
+    let Error::Io(io_err) = err else {
+        return false;
+    };
+    io_err
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
+        .is_some_and(|source| source.stage.as_deref() == Some(stage))
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/disk_store.rs

@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use uuid::Uuid;
 
+/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
+/// Completion may mean legacy fallback after a failed scan; callers must assert
+/// the expected index result separately. Remote disks have no local worker.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
+    match disk.as_ref() {
+        crate::disk::Disk::Local(local) => {
+            local.disk.wait_for_multipart_index_bootstrap().await;
+            Ok(())
+        }
+        crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
+    }
+}
+
 /// Disk health status constants
 const DISK_HEALTH_OK: u32 = 0;
 const DISK_HEALTH_FAULTY: u32 = 1;
@@ -853,6 +867,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +896,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1591,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1946,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2398,12 +2447,17 @@ impl LocalDiskWrapper {
         .await
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: uuid::Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2411,7 +2465,7 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                        .try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
                         .await
                 })
             },
@@ -2427,6 +2481,7 @@ impl LocalDiskWrapper {
         transaction: uuid::Uuid,
         journal: &crate::disk::DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2434,7 +2489,14 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
+                        .try_finish_and_finalize_prepared_delete_local_with_budget(
+                            volume,
+                            object,
+                            transaction,
+                            journal,
+                            namespace_owner,
+                            work_budget,
+                        )
                         .await
                 })
             },
@@ -2634,6 +2696,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -2682,6 +2751,18 @@ impl LocalDiskWrapper {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.list_multipart_index_roots(bucket, marker).await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         // The local implementation shares this binary's capability. Availability
         // and disk identity remain mutation checks and write-quorum inputs.
@@ -2735,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
         )
         .await
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        self.track_disk_health_with_op(
+            "discover_delete_transactions",
+            || async {
+                self.disk
+                    .discover_delete_transactions_bounded(volume, object, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: uuid::Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        self.track_disk_health_with_op(
+            "inspect_delete_transaction",
+            || async {
+                self.disk
+                    .inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        self.track_disk_health_mutation(
+            "settle_delete_transaction",
+            DiskMetricMutation::Delete,
+            || async {
+                self.disk
+                    .settle_delete_transaction_bounded(volume, request, namespace_owner)
+                    .await
+            },
+            crate::disk::get_delete_recovery_timeout()?,
+        )
+        .await
+    }
+
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -3703,7 +3840,11 @@ mod tests {
         assert!(futures::poll!(&mut operation).is_pending());
         time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
-        assert!(logs.contents().is_empty(), "normal disk latency should not warn");
+        assert!(
+            !logs.contents().contains("disk_operation_slow"),
+            "normal disk latency should not warn: {}",
+            logs.contents()
+        );
         time::advance(Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
         let warning = logs.contents();
@@ -6040,6 +6181,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/error.rs

@@ -172,6 +172,8 @@ pub enum DiskError {
 
     #[error("guarded batch deletion is not supported by this disk")]
     GuardedDeleteUnsupported,
+    #[error("foreground delete recovery work limit reached")]
+    DeleteRecoveryWorkLimit,
 
     #[error("erasure write quorum")]
     ErasureWriteQuorum,
@@ -770,6 +772,7 @@ impl Clone for DiskError {
             DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
             DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
             DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
             DiskError::ShortWrite => DiskError::ShortWrite,
@@ -829,6 +832,7 @@ impl DiskError {
             DiskError::RemoteClientUnavailable(_) => 0x2B,
             DiskError::ConditionalDeleteConflict => 0x2C,
             DiskError::GuardedDeleteUnsupported => 0x2D,
+            DiskError::DeleteRecoveryWorkLimit => 0x2E,
         }
     }
 
@@ -879,6 +883,7 @@ impl DiskError {
             0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
             0x2C => Some(DiskError::ConditionalDeleteConflict),
             0x2D => Some(DiskError::GuardedDeleteUnsupported),
+            0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
             _ => None,
         }
     }
@@ -1662,6 +1667,7 @@ mod tests {
         for (error, code) in [
             (DiskError::ConditionalDeleteConflict, 0x2C),
             (DiskError::GuardedDeleteUnsupported, 0x2D),
+            (DiskError::DeleteRecoveryWorkLimit, 0x2E),
         ] {
             assert_eq!(error.to_u32(), code);
             assert_eq!(DiskError::from_u32(code), Some(error.clone()));

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/local/commit.rs

@@ -184,6 +184,8 @@ pub(super) fn create_local_inline_rollback_backup(
     remove_file_if_exists(&backup_path)?;
     if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
         let write_res = (|| -> std::io::Result<()> {
+            #[cfg(test)]
+            run_owned_file_before_open(&backup_path);
             let mut file = std::fs::File::create(&backup_path)?;
             std::io::Write::write_all(&mut file, old_metadata)?;
             if sync && !old_metadata.is_empty() {
@@ -367,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
 
 #[cfg(target_os = "linux")]
 enum FusedCommitAttempt {
-    Committed(RenameDataResp),
+    Committed(Box<RenameDataResp>),
     CommitErr(DiskError),
-    Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
+    RestoreDataSource(os::RenameCommitGuard, DiskError),
+    Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
+    #[cfg(test)]
+    ResumePanic(Box<dyn std::any::Any + Send + 'static>),
 }
 
 async fn prepare_rename_commit_and_destination_metadata(
@@ -481,6 +486,9 @@ impl LocalDisk {
             state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
         })?;
         crate::hp_guard!("LocalDisk::rename_data");
+        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
+            self.multipart_index.record_info(dst_path, &fi).await?;
+        }
         #[cfg(feature = "e2e-test-hooks")]
         if fi.is_healing() {
             wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
@@ -551,7 +559,9 @@ impl LocalDisk {
             None
         };
         let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
-            Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
+            Some(Arc::new(
+                pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
+            ))
         } else {
             None
         };
@@ -752,11 +762,6 @@ impl LocalDisk {
             && !fi_healing
             && !os::dst_dir_fsync_group_commit_enabled()
             && !os::file_fdatasync_group_commit_enabled()
-            && !super::has_rename_data_test_failpoints(dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
-            && !super::has_delete_version_test_failpoints()
         {
             #[cfg(test)]
             if !no_inline {
@@ -800,7 +805,7 @@ impl LocalDisk {
                 // requires a rollback directory; appending a new version keeps
                 // an inline rollback backup for post-rename directory sync errors.
                 if is_cancelled() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let mut xlmeta = FileMeta::new();
                 let mut dst_meta_unparsable = false;
@@ -823,7 +828,19 @@ impl LocalDisk {
                     }
                 });
                 if rollback_data_dir.is_some() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
+                }
+                #[cfg(test)]
+                let probe_active = os::file_sync_probe::is_active();
+                #[cfg(not(test))]
+                let probe_active = false;
+                if no_inline
+                    && sync_data_shards
+                    && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                    && (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
+                    && os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
+                {
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
                     has_dst_buf.is_some(),
@@ -842,20 +859,31 @@ impl LocalDisk {
                     skip_parent = dst_parent_owned.clone();
                 }
 
+                #[cfg(test)]
+                run_owned_file_before_open(&src_file_path_owned);
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
-                prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
+                prepared_metadata_source.write_all(&new_dst_buf, false)?;
+                if !no_inline {
+                    run_inline_preparation_before_backup(&dst_path_str);
+                }
 
                 let mut staged_rollback_path = None;
-                if sync_commit_metadata && let Some(ref old_metadata) = has_dst_buf {
-                    let backup_path =
-                        create_local_inline_rollback_backup(&dst_file_path_owned, &src_file_path_owned, old_metadata, true)?;
+                if let Some(ref old_metadata) = has_dst_buf
+                    && (sync_commit_metadata || cfg!(test))
+                {
+                    let backup_path = create_local_inline_rollback_backup(
+                        &dst_file_path_owned,
+                        &src_file_path_owned,
+                        old_metadata,
+                        sync_commit_metadata,
+                    )?;
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 
                 if no_inline {
                     if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
-                        let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
+                        let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
                             // The payload was synced, but its part.1 directory entry
                             // must also be persisted before moving the data directory.
                             os::fsync_dir_std(src_data_path)
@@ -871,37 +899,124 @@ impl LocalDisk {
                             return Err(err);
                         }
                     }
+                    if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Err(err);
+                    }
+                } else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(err);
+                }
+
+                if is_cancelled() {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
+                }
+
+                let guard = match super::remove_dst_base_before_commit_std(
+                    &dst_path_str,
+                    guard,
+                    &src_parent_owned,
+                    &dst_parent_owned,
+                    &dst_volume_dir_owned,
+                    &publication_root_owned,
+                ) {
+                    Ok(guard) => guard,
+                    Err(err) => {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(err));
+                    }
+                };
+                if should_remove_staged_meta_before_commit(&dst_path_str) {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    std::fs::remove_file(&src_file_path_owned)?;
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
+                }
+
+                if no_inline {
                     if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
                         && let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
                     {
                         if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
                             let _ = remove_file_if_exists(backup_path);
                         }
-                        return Ok(FusedCommitAttempt::CommitErr(err));
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     #[cfg(test)]
-                    if has_data_dir_path_owned.is_some() {
-                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                    if has_data_dir_path_owned.is_some()
+                        && let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                            run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        }))
+                    {
+                        return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                    }
+                    if should_fail_before_old_metadata_backup(&dst_path_str) {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
                 }
 
-                if let Err(err) = os::rename_prepared_source_on_thread(
-                    prepared_metadata_source,
-                    &src_file_path_owned,
-                    &dst_file_path_owned,
-                    &skip_parent,
-                    &publication_root_owned,
-                    &guard,
-                ) {
+                let rename_res = if should_fail_commit_rename(&dst_path_str) {
+                    drop(prepared_metadata_source);
+                    Err(DiskError::other("test fail during metadata commit rename"))
+                } else {
+                    os::rename_prepared_source_on_thread(
+                        prepared_metadata_source,
+                        &src_file_path_owned,
+                        &dst_file_path_owned,
+                        &skip_parent,
+                        &publication_root_owned,
+                        &guard,
+                    )
+                };
+                if let Err(err) = rename_res {
                     if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
                         let _ = remove_file_if_exists(backup_path);
                     }
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                    if no_inline {
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(err));
                 }
 
+                if should_fail_after_metadata_commit(&dst_path_str) {
+                    if no_inline {
+                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                    }
+                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
+                        .err()
+                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
+                }
+
+                if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                }
+
                 if sync_commit_metadata
                     && let Err(err) = fsync_inline_commit_std(
                         &dst_file_path_owned,
@@ -911,8 +1026,11 @@ impl LocalDisk {
                         &guard,
                     )
                 {
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                    if no_inline {
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
                 }
@@ -925,7 +1043,7 @@ impl LocalDisk {
                     let _ = std::fs::remove_dir(parent);
                 }
 
-                Ok(FusedCommitAttempt::Committed(RenameDataResp {
+                Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
                     multipart_source: None,
                     old_data_dir: None,
                     rollback_data_dir: None,
@@ -934,7 +1052,7 @@ impl LocalDisk {
                     old_current_size,
                     old_current_source_checked,
                     old_current_source,
-                }))
+                })))
             };
 
             let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
@@ -954,7 +1072,23 @@ impl LocalDisk {
                     if let Some(receipt_mutation) = receipt_mutation.as_mut() {
                         receipt_mutation.complete();
                     }
-                    return Ok(resp);
+                    return Ok(*resp);
+                }
+                Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Err(err);
                 }
                 Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
                     for part_path in &invalidate_part_paths {
@@ -962,11 +1096,269 @@ impl LocalDisk {
                     }
                     return Err(err);
                 }
+                #[cfg(test)]
+                Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
+                    std::panic::resume_unwind(panic_payload);
+                }
                 Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
-                    fi = restored_fi;
+                    fi = *restored_fi;
                     (guard, dst_buf)
                 }
             }
+        } else if let Some(publication) = multipart_publication.as_ref()
+            && no_inline
+            && !preserve_upload_metadata
+            && !publication.legacy
+            && !fi_healing
+            && !os::dst_dir_fsync_group_commit_enabled()
+        {
+            enum MultipartFusedCommitOutcome {
+                Committed(Box<RenameDataResp>),
+                AbortBeforeDataRename(DiskError),
+                RestoreDataSource(os::RenameCommitGuard, DiskError),
+                RestoreTargetAfterError(DiskError, bool),
+            }
+            let publication_owned = Arc::clone(publication);
+            let src_parent_owned = src_file_parent.to_path_buf();
+            let dst_parent_owned = dst_file_parent.to_path_buf();
+            let dst_file_path_owned = dst_file_path.clone();
+            let src_file_path_owned = src_file_path.clone();
+            let dst_volume_dir_owned = dst_volume_dir.clone();
+            let publication_root_owned = self.publication_root.clone();
+            let has_data_dir_path_owned = has_data_dir_path.clone();
+            let sync_data_shards = durability.syncs_data_shards();
+            let sync_commit_metadata = durability.syncs_commit_metadata();
+            let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
+            let dst_volume_str = dst_volume.to_string();
+            let dst_path_str = dst_path.to_string();
+            let fi_moved = fi;
+            #[cfg(test)]
+            let root_owned = self.root.clone();
+
+            let outcome = os::run_blocking_namespace_operation_with_sync(
+                mutation_lease.clone(),
+                true,
+                move || -> std::io::Result<MultipartFusedCommitOutcome> {
+                    let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
+                        false,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_file_path_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(res) => res,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                    };
+                    let mut xlmeta = FileMeta::new();
+                    let mut dst_meta_unparsable = false;
+                    if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
+                        if FileMeta::is_xl2_v1_format(dst_buf)
+                            && let Ok(nmeta) = FileMeta::load(dst_buf)
+                        {
+                            xlmeta = nmeta;
+                        } else {
+                            dst_meta_unparsable = true;
+                        }
+                    }
+                    let skip_parent_owned = if has_dst_buf_owned.is_some() {
+                        dst_parent_owned.clone()
+                    } else {
+                        dst_volume_dir_owned.clone()
+                    };
+                    let version_id = fi_moved.version_id.unwrap_or_default();
+                    let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
+                        has_dst_buf_owned.is_some(),
+                        dst_meta_unparsable,
+                        &xlmeta,
+                        &dst_volume_str,
+                        &dst_path_str,
+                        version_id,
+                    );
+                    let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
+                    if let Some(old_data_dir) = has_old_data_dir.as_ref() {
+                        let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
+                    }
+                    if let Err(err) = xlmeta.add_version(fi_moved) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
+                    }
+                    let version_signature = rename_data_versions_signature(&xlmeta);
+                    let new_dst_buf_owned = match xlmeta.marshal_msg() {
+                        Ok(buf) => buf,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
+                    };
+
+                    let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
+                        &new_dst_buf_owned,
+                        has_dst_buf_owned.as_deref(),
+                        &dst_file_path_owned,
+                        &directory_guard_owned,
+                    ) {
+                        Ok(source) => source,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    #[cfg(test)]
+                    let probe_active = os::file_sync_probe::is_active();
+                    #[cfg(not(test))]
+                    let probe_active = false;
+                    // When presynced, shard payloads were synced during UploadPart and
+                    // prepare_and_install_head_on_thread just fsynced src_data_path after
+                    // writing RETIREMENT_ANCHOR.
+                    if sync_data_shards
+                        && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                        && (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
+                        && let Err(err) = os::sync_dir_files_std(src_data_path)
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                    }
+                    let directory_guard_owned = match super::remove_dst_base_before_commit_std(
+                        &dst_path_str,
+                        directory_guard_owned,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(guard) => guard,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    if should_remove_staged_meta_before_commit(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                        }
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
+                    }
+                    if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        let Some(src_parent) = src_data_path.parent() else {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
+                        };
+                        let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
+                            Ok(guard) => guard,
+                            Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                        };
+                        if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                        }
+                        #[cfg(test)]
+                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                        if should_fail_before_old_metadata_backup(&dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
+                                directory_guard_owned,
+                                DiskError::Unexpected,
+                            ));
+                        }
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                    }
+                    let rename_res = if should_fail_commit_rename(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        Err(DiskError::other("test fail during metadata commit rename"))
+                    } else {
+                        os::rename_prepared_source_on_thread(
+                            prepared_metadata_source,
+                            &src_file_path_owned,
+                            &dst_file_path_owned,
+                            &skip_parent_owned,
+                            &publication_root_owned,
+                            &directory_guard_owned,
+                        )
+                    };
+                    if let Err(err) = rename_res {
+                        return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                    }
+                    if should_fail_after_metadata_commit(&dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                    }
+                    if sync_commit_metadata {
+                        if let Some(parent) = dst_file_path_owned.parent()
+                            && let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
+                        }
+                        if has_dst_buf_owned.is_none() {
+                            let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
+                            while let Some(dir) = ancestor {
+                                if !dir.starts_with(&dst_volume_dir_owned) {
+                                    break;
+                                }
+                                if let Err(err) = os::fsync_dir_limited_std(dir) {
+                                    return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
+                                        to_file_error(err).into(),
+                                        true,
+                                    ));
+                                }
+                                if dir == dst_volume_dir_owned.as_path() {
+                                    break;
+                                }
+                                ancestor = dir.parent();
+                            }
+                        }
+                    }
+                    match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
+                        Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
+                            multipart_source: Some(Box::new(result)),
+                            old_data_dir: has_old_data_dir,
+                            rollback_data_dir: None,
+                            cleanup_data_dir: has_old_data_dir,
+                            sign: version_signature,
+                            old_current_size,
+                            old_current_source_checked,
+                            old_current_source,
+                        }))),
+                        Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
+                    }
+                },
+            )
+            .await
+            .map_err(to_file_error)
+            .map_err(DiskError::from)?;
+
+            match outcome {
+                MultipartFusedCommitOutcome::Committed(resp) => {
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Ok(*resp);
+                }
+                MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
+                MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    return Err(err);
+                }
+                MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
+                    if !invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
+                    if invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    return Err(err);
+                }
+            }
         } else {
             prepare_rename_commit_and_destination_metadata(
                 !no_inline && !preserve_upload_metadata,
@@ -1090,13 +1482,29 @@ impl LocalDisk {
             // after the commit rename, and a crash before the rename means the
             // PUT was never acknowledged. A metadata commit: relaxed tiers
             // leave it to the page cache.
+            #[cfg(all(test, target_os = "linux"))]
+            let probe_active = os::file_sync_probe::is_active();
+            #[cfg(all(not(test), target_os = "linux"))]
+            let probe_active = false;
             let shard_sync = async {
                 if durability.syncs_data_shards()
                     && let Some((src_data_path, _)) = has_data_dir_path.as_ref()
-                    && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
-                    && err.kind() != ErrorKind::NotFound
                 {
-                    return Err::<(), DiskError>(to_file_error(err).into());
+                    #[cfg(target_os = "linux")]
+                    if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
+                        if multipart_publication.is_none()
+                            && let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
+                            && err.kind() != ErrorKind::NotFound
+                        {
+                            return Err::<(), DiskError>(to_file_error(err).into());
+                        }
+                        return Ok(());
+                    }
+                    if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Err::<(), DiskError>(to_file_error(err).into());
+                    }
                 }
                 Ok(())
             };
@@ -1488,13 +1896,22 @@ impl LocalDisk {
             }
 
             let multipart_source = match multipart_publication.as_ref() {
-                Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
-                    Ok(result) => Some(Box::new(result)),
-                    Err(error) => {
-                        pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
-                        return Err(error);
+                Some(publication) => {
+                    let target_already_synced = durability.syncs_commit_metadata();
+                    let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
+                    match pin_multipart_future(|| {
+                        publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
+                    })
+                    .await
+                    {
+                        Ok(result) => Some(Box::new(result)),
+                        Err(error) => {
+                            pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
+                                .await?;
+                            return Err(error);
+                        }
                     }
-                },
+                }
                 None => None,
             };
             Ok(RenameDataResp {

crates/ecstore/src/disk/local/multipart_source_abort.rs

@@ -421,6 +421,7 @@ impl LocalDisk {
                     return Err(DiskError::ConditionalDeleteConflict);
                 }
                 let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
+                super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
                 let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
                 let source = store.source.clone();
                 let base = store.base.clone();
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
     root: os::PublicationRoot,
 }
 impl OrdinaryMultipartDeleteCheck {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
     pub(super) fn validate(&self) -> Result<()> {
         let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
             Ok(pinned) => pinned,

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
 }
 
 pub(super) struct SourceStore {
+    multipart_index: Arc<super::multipart_index::IndexOwner>,
     pub disk_id: Uuid,
     pub physical_slot: usize,
     pub disk_ids: Vec<Uuid>,
@@ -186,6 +187,7 @@ impl LocalDisk {
         let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
         Ok(SourceStore {
+            multipart_index: self.multipart_index.clone(),
             disk_id,
             physical_slot,
             disk_ids,
@@ -238,9 +240,177 @@ pub(super) fn read_optional_in_pinned(
         Err(error) => Err(error.into()),
     }
 }
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct FileStamp {
+    len: u64,
+    modified: Option<std::time::SystemTime>,
+    #[cfg(unix)]
+    ino: u64,
+}
+
+impl FileStamp {
+    fn from_path(path: &Path) -> Option<Self> {
+        let meta = std::fs::symlink_metadata(path).ok()?;
+        Some(Self {
+            len: meta.len(),
+            modified: meta.modified().ok(),
+            #[cfg(unix)]
+            ino: {
+                use std::os::unix::fs::MetadataExt;
+                meta.ino()
+            },
+        })
+    }
+}
+
+#[derive(Clone)]
+struct ActiveMultipartSourceEntry {
+    raw: Bytes,
+    fi: FileInfo,
+    head_stamp: FileStamp,
+    unsettled_parts: std::collections::HashSet<usize>,
+    parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
+}
+
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
+
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
+    let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
+        source,
+        ActiveMultipartSourceEntry {
+            raw,
+            fi,
+            head_stamp,
+            unsettled_parts: std::collections::HashSet::new(),
+            parts: std::collections::HashMap::new(),
+        },
+    );
+}
+
+fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
+    FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
+        && !source.join(STATE).exists()
+        && !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
+        && !source.join(".mpu-completions").exists()
+}
+
+pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
+    let (raw, fi, head_stamp) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    Some((raw, fi))
+}
+
+pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    true
+}
+
+pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
+    let Some((head_stamp, clean)) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
+    else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    clean
+}
+
+pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.unsettled_parts.insert(part_number);
+        entry.parts.remove(&part_number);
+    });
+}
+
+pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
+    let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| entry.fi.data_dir)
+        .flatten()
+        .filter(|dir| !dir.is_nil())
+    else {
+        return;
+    };
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
+    let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.parts.insert(part.number, (part_stamp, part));
+    });
+}
+
+pub(super) fn finish_active_multipart_part_transaction(
+    source: &Path,
+    part_number: Option<usize>,
+    committed: bool,
+    retired_existing_txn: bool,
+) {
+    let Some(part_number) = part_number.filter(|_| committed) else {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return;
+    };
+    if !retired_existing_txn {
+        return;
+    }
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
+}
+
+pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| {
+            if !entry.unsettled_parts.is_empty() {
+                return None;
+            }
+            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
+            let (expected_stamp, part) = entry.parts.get(&part_number)?;
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
+        })
+        .flatten()?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
+    if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
+        return None;
+    }
+    Some(part)
+}
+
+pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+}
+
+pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
+}
+
 /// Canonical source mutations are admitted under SourceRW at the syscall,
 /// not on the earlier async caller that may have timed out.
 pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
+    if !initial && has_active_multipart_source_head(source) {
+        return Ok(());
+    }
     let pinned = match os::pin_multipart_source_directory(source, base, root) {
         Ok(pinned) => pinned,
         Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
@@ -440,6 +610,7 @@ impl SourceStore {
         terminal: Option<&Path>,
         sync_source_before: bool,
     ) -> std::io::Result<()> {
+        invalidate_active_multipart_source_head(source);
         #[cfg(unix)]
         let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
         #[cfg(not(unix))]
@@ -470,7 +641,7 @@ impl SourceStore {
         let terminal = terminal_record_path(&source, state);
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
             Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
         })
         .await
@@ -539,7 +710,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
         })
         .await
         .map_err(DiskError::from)
@@ -551,6 +722,7 @@ impl SourceStore {
         owner: Uuid,
         target: bool,
         bytes: &[u8],
+        sync_source_root: bool,
     ) -> std::io::Result<()> {
         let path = if target {
             target_backup_path(source, owner)
@@ -582,9 +754,13 @@ impl SourceStore {
         if !existed {
             os::replace_multipart_source_metadata(&path, bytes, base, root)?;
         }
-        os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
+        os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
+        self.multipart_index
+            .record_head(&format!("{}/xl.meta", self.path), bytes)
+            .await?;
+        invalidate_active_multipart_source_head(&self.source);
         let path = self.source.join(STORAGE_FORMAT_FILE);
         let base = self.base.clone();
         let root = self.publication_root.clone();
@@ -1097,6 +1273,82 @@ pub(super) fn load_source(
     })
 }
 
+fn load_uninitialized_source(
+    raw: Bytes,
+    path: &str,
+    disk_id: Uuid,
+    physical_slot: usize,
+    disk_ids: &[Uuid],
+) -> Result<LoadedSource> {
+    let sha = raw_sha(&raw);
+    let length = raw.len() as u64;
+    let current = get_file_info(
+        &raw,
+        RUSTFS_META_MULTIPART_BUCKET,
+        path,
+        "",
+        FileInfoOpts {
+            data: false,
+            include_free_versions: false,
+            include_part_checksums: true,
+        },
+    )
+    .map_err(Error::other)?;
+    native_source_protocol(&current)?;
+    if source_status(&current)?.is_some()
+        || publication_identity(&current)?.is_some()
+        || SourceConditionV1::take(&mut current.clone())?.is_some()
+    {
+        return Err(DiskError::FileCorrupt);
+    }
+    let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-bucket")
+        .ok_or(DiskError::FileCorrupt)?;
+    let object = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-object")
+        .ok_or(DiskError::FileCorrupt)?;
+    let condition = capture_source_condition(bucket, object, path, &current, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
+    if condition.generation.epoch != 0 {
+        return Err(DiskError::FileCorrupt);
+    }
+    let slot = condition
+        .lineage
+        .geometry
+        .distribution
+        .iter()
+        .position(|index| *index == current.erasure.index)
+        .ok_or(DiskError::FileCorrupt)?;
+    if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
+        return Err(DiskError::ConditionalDeleteConflict);
+    }
+    let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
+    let snapshot = Some(MultipartSourceSnapshot {
+        disk_id,
+        physical_slot: slot,
+        lineage: condition.lineage,
+        generation: condition.generation,
+        revision: 0,
+        phase: MultipartSourcePhase::Active,
+        state_sha256: [0; 32],
+        head_sha256: Some(sha),
+        inventory_sha256: inventory,
+        capability_sha256: None,
+        publication: None,
+    });
+    Ok(LoadedSource {
+        root_missing: false,
+        state: None,
+        head: Some(SourceHead {
+            sha256: sha,
+            length,
+            raw: Some(raw.clone()),
+        }),
+        original: Some(raw),
+        info: Some(current),
+        snapshot,
+        head_matches_state: true,
+        head_matches_activation: false,
+    })
+}
+
 pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
 
 #[derive(Serialize, Deserialize)]
@@ -1421,6 +1673,8 @@ struct ValidatedPreparedPublication {
     length: u64,
 }
 
+type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
+
 pub(super) struct NativeMultipartPublication {
     pub store: Arc<SourceStore>,
     pub publication: PublicationIdentity,
@@ -1428,22 +1682,49 @@ pub(super) struct NativeMultipartPublication {
     state: parking_lot::Mutex<SourceState>,
     capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
     validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
+    deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
     pub legacy: bool,
 }
 
 impl LocalDisk {
+    #[cfg(test)]
     pub(super) async fn prepare_native_multipart_publication(
         &self,
         path: &str,
         incoming: &mut FileInfo,
         staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
+            .await
+    }
+
+    pub(super) async fn prepare_native_multipart_publication_deferred(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
+            .await
+    }
+
+    async fn prepare_native_multipart_publication_inner(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+        defer_persist: bool,
     ) -> Result<NativeMultipartPublication> {
         // Real Complete is parts-backed, including a zero-byte last part.
         if incoming.data.is_some() || incoming.inline_data() {
             return Err(DiskError::UnsupportedDisk);
         }
         let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
-        let loaded = store.load().await?;
+        let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
+            load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
+        } else {
+            store.load().await?
+        };
         let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
         let original = loaded
             .head
@@ -1545,20 +1826,33 @@ impl LocalDisk {
         };
         let anchor_bytes = encode(&anchor_record)?;
         state.validate(store.disk_id, &store.path)?;
-        let state_raw = encode(&state)?;
-        let source_dir = store.source.clone();
-        let base_dir = store.base.clone();
-        let pub_root = store.publication_root.clone();
-        let terminal = terminal_record_path(&source_dir, &state);
         let original_bytes = original.to_vec();
-        os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
-            SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes)?;
-            SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
-            let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
-            os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)
-        })
-        .await
-        .map_err(DiskError::from)?;
+        let deferred_initial = if defer_persist {
+            Some((original_bytes, directory, anchor_bytes))
+        } else {
+            let state_raw = encode(&state)?;
+            let source_dir = store.source.clone();
+            let base_dir = store.base.clone();
+            let pub_root = store.publication_root.clone();
+            let terminal = terminal_record_path(&source_dir, &state);
+            #[cfg(target_os = "linux")]
+            let can_mark_presynced = has_active_multipart_source_head(&source_dir)
+                && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+            os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
+                SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
+                SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+                Ok(())
+            })
+            .await
+            .map_err(DiskError::from)?;
+            None
+        };
         let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
         let context = NativeMultipartPublication {
             store,
@@ -1567,6 +1861,7 @@ impl LocalDisk {
             state: parking_lot::Mutex::new(state),
             capability: parking_lot::Mutex::new(Some(capability)),
             validated_prepared: parking_lot::Mutex::new(None),
+            deferred_initial: parking_lot::Mutex::new(deferred_initial),
             legacy,
         };
         Ok(context)
@@ -1621,7 +1916,7 @@ impl NativeMultipartPublication {
                 return Err(IoError::other(DiskError::ConditionalDeleteConflict));
             }
             if let Some(previous) = old_bytes.as_deref() {
-                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
             }
             SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
         })
@@ -1631,6 +1926,133 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
+    pub(super) fn prepare_and_install_head_on_thread(
+        &self,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+    ) -> Result<os::PreparedRenameSource> {
+        let mut state = self.state.lock().clone();
+        if !self.store.target_matches(&self.publication, new_metadata)? {
+            return Err(DiskError::ConditionalDeleteConflict);
+        }
+        let deferred_initial = self.deferred_initial.lock().take();
+        let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
+        let validated = ValidatedPreparedPublication {
+            publication: self.publication.clone(),
+            sha256: raw_sha(new_metadata),
+            length: new_metadata.len() as u64,
+        };
+        if let Some(previous) = old_metadata {
+            state.previous_target_sha256 = Some(raw_sha(previous));
+            state.previous_target_length = Some(previous.len() as u64);
+        }
+        state.prepared_sha256 = Some(validated.sha256);
+        state.prepared_length = Some(validated.length);
+        state.advance(MultipartSourcePhase::Prepared)?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = &self.store.disk_ids;
+        let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+        let terminal = terminal_record_path(source, &state);
+        let prepared_path = &self.prepared_path;
+        let canonical = self.canonical_head();
+        let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
+            if let Some((ref original_bytes, _, _)) = deferred_initial {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            } else {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            }
+            #[cfg(test)]
+            super::run_owned_file_before_open(prepared_path);
+            let mut prepared_metadata_source =
+                os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
+            if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            let setup_res: std::io::Result<()> = (|| {
+                if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                    SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                    let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
+                    os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
+                    #[cfg(target_os = "linux")]
+                    if can_mark_presynced {
+                        super::mark_presynced_tmp_data_dir(directory);
+                    }
+                } else {
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                }
+                Ok(())
+            })();
+            if let Err(err) = setup_res {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            if let Err(err) = prepared_metadata_source.sync_data() {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
+        })()
+        .map_err(DiskError::from)?;
+        *self.state.lock() = state;
+        *self.validated_prepared.lock() = Some(validated);
+        Ok(prepared)
+    }
+
+    #[cfg(not(target_os = "linux"))]
     pub async fn prepare_and_install_head(
         &self,
         new_metadata: &[u8],
@@ -1643,7 +2065,11 @@ impl NativeMultipartPublication {
         if !self.store.target_matches(&self.publication, new_metadata)? {
             return Err(DiskError::ConditionalDeleteConflict);
         }
+        let deferred_initial = self.deferred_initial.lock().take();
         let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
         let validated = ValidatedPreparedPublication {
             publication: self.publication.clone(),
             sha256: raw_sha(new_metadata),
@@ -1666,6 +2092,9 @@ impl NativeMultipartPublication {
         let physical_slot = self.store.physical_slot;
         let disk_ids = self.store.disk_ids.clone();
         let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(&source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
         let old_bytes = old_metadata.map(|b| b.to_vec());
         let new_bytes = new_metadata.to_vec();
         let terminal = terminal_record_path(&source, &state);
@@ -1675,18 +2104,51 @@ impl NativeMultipartPublication {
         let owned_directory_guard = directory_guard.clone();
         let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
             let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
-            let current_state = current
-                .state
-                .as_ref()
-                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-            let current_raw = encode(current_state).map_err(IoError::other)?;
-            if current_raw != expected_state_raw || !current.head_matches_state {
-                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+            } else {
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
             }
-            if let Some(previous) = old_bytes.as_deref() {
-                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
-            }
-            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
             #[cfg(test)]
             super::run_owned_file_before_open(&prepared_path);
             let mut prepared_metadata_source =
@@ -1700,6 +2162,28 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(prepared)
     }
+
+    #[cfg(target_os = "linux")]
+    pub async fn prepare_and_install_head(
+        self: &Arc<Self>,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+        mutation_lease: Arc<os::NamespaceMutationLease>,
+    ) -> Result<os::PreparedRenameSource> {
+        let this = Arc::clone(self);
+        let new_bytes = new_metadata.to_vec();
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let dst_file_path = dst_file_path.to_path_buf();
+        let owned_directory_guard = directory_guard.clone();
+        os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
+            this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
     #[cfg(test)]
     pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
         {
@@ -1771,7 +2255,11 @@ impl NativeMultipartPublication {
         // return an error after consuming the last claimable source heads.
         self.store.restore_exact_target_with_lease(&state, lease).await
     }
-    pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
+    pub(super) fn finish_on_thread(
+        &self,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
         let mut state = self.state.lock().clone();
         let expected_state_raw = encode(&state)?;
         let (expected_sha256, expected_length) = {
@@ -1790,16 +2278,16 @@ impl NativeMultipartPublication {
         state.validate(self.store.disk_id, &self.store.path)?;
         let next_state_raw = encode(&state)?;
         let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
-        let source = self.store.source.clone();
-        let base = self.store.base.clone();
-        let root = self.store.publication_root.clone();
-        let path = self.store.path.clone();
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
         let disk_id = self.store.disk_id;
         let physical_slot = self.store.physical_slot;
-        let disk_ids = self.store.disk_ids.clone();
-        let terminal = terminal_record_path(&source, &state);
-        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
-            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
+        let disk_ids = &self.store.disk_ids;
+        let terminal = terminal_record_path(source, &state);
+        (|| -> std::io::Result<()> {
+            let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
             let current_state = current
                 .state
                 .as_ref()
@@ -1808,28 +2296,64 @@ impl NativeMultipartPublication {
             if current.head.is_some() || current_raw != expected_state_raw {
                 return Err(IoError::other(DiskError::ConditionalDeleteConflict));
             }
-            os::read_and_sync_multipart_target_metadata(
-                &target_dir,
-                std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
-                &base,
-                &root,
-                |target| {
-                    if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
-                        return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            if target_already_synced {
+                let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
                     }
-                    Ok(())
-                },
-            )
-            .map_err(|error| {
-                if error.kind() == std::io::ErrorKind::NotFound {
-                    IoError::other(DiskError::ConditionalDeleteConflict)
-                } else {
-                    error
+                })?;
+                let target = os::read_multipart_source_metadata_in_pinned(
+                    &pinned,
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+                if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
                 }
-            })?;
-            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), true)
-        })
-        .await
+            } else {
+                os::read_and_sync_multipart_target_metadata(
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                    |target| {
+                        if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
+                            return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                        }
+                        Ok(())
+                    },
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+            }
+            SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
+            if let Some((io_path, cleaned)) = staging_cleanup {
+                match std::fs::remove_dir(&io_path) {
+                    Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
+                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                        cleaned.store(true, std::sync::atomic::Ordering::Release);
+                    }
+                    Err(_) => {}
+                }
+            }
+            Ok(())
+        })()
         .map_err(DiskError::from)?;
         *self.state.lock() = state;
         let proof = self
@@ -1842,6 +2366,21 @@ impl NativeMultipartPublication {
             undo_capability: capability,
         })
     }
+
+    pub async fn finish(
+        self: &Arc<Self>,
+        target_lease: Arc<os::NamespaceMutationLease>,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
+        let this = Arc::clone(self);
+        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
+            this.finish_on_thread(target_already_synced, staging_cleanup)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
 }
 
 impl SourceStore {

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
     initial: bool,
 }
 impl MultipartSourceAdmission {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
+    pub(super) fn is_initial(&self) -> bool {
+        self.initial
+    }
     pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
         let admission = self.clone();
         Arc::new(move || admission.validate())
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
     logical_path: PathBuf,
     io_path: PathBuf,
     owner: Option<Arc<dyn Send + Sync>>,
+    cleaned: Arc<std::sync::atomic::AtomicBool>,
+    fresh_source_head: Option<bytes::Bytes>,
 }
 
 impl MultipartCommitStaging {
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
     pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
         self.io_path.join(directory.to_string())
     }
+    pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
+        (self.io_path.clone(), Arc::clone(&self.cleaned))
+    }
+    pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
+        self.fresh_source_head.clone()
+    }
 }
 
 impl Drop for MultipartCommitStaging {
     fn drop(&mut self) {
+        if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
+            return;
+        }
         let logical_path = self.logical_path.clone();
         let io_path = self.io_path.clone();
         let owner = self.owner.clone();
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
 }
 
 impl LocalDisk {
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        let root = upload_scope_root(upload_or_part_path, part_path);
+        let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
+            return false;
+        };
+        super::multipart_source_state::has_clean_active_multipart_source(&source)
+    }
+
     pub(super) fn multipart_source_admission(
         &self,
         volume: &str,
@@ -356,12 +381,14 @@ impl LocalDisk {
         let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
         // The attempt root is exactly the namespace used by the tmp sweeper.
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
-        let staging = MultipartCommitStaging {
+        let mut staging = MultipartCommitStaging {
             path: attempt,
             lease,
             logical_path,
             io_path,
             owner,
+            cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            fresh_source_head: None,
         };
         let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
         let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
@@ -371,31 +398,37 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }
-            let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
-                // A retired source has no authority for this delayed completion.
-                // Preserve confinement and other I/O failures as their real errors.
-                if error.kind() == std::io::ErrorKind::NotFound {
-                    DiskError::FileNotFound.into()
-                } else {
-                    error
-                }
-            })?;
-            let source = get_file_info(
-                &raw,
-                RUSTFS_META_MULTIPART_BUCKET,
-                &source_path,
-                "",
-                FileInfoOpts {
-                    data: false,
-                    include_free_versions: false,
-                    include_part_checksums: true,
-                },
-            )
-            .map_err(std::io::Error::other)?;
+            let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
+            let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
+                (raw, source, true)
+            } else {
+                let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
+                    // A retired source has no authority for this delayed completion.
+                    // Preserve confinement and other I/O failures as their real errors.
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        DiskError::FileNotFound.into()
+                    } else {
+                        error
+                    }
+                })?;
+                let source = get_file_info(
+                    &raw,
+                    RUSTFS_META_MULTIPART_BUCKET,
+                    &source_path,
+                    "",
+                    FileInfoOpts {
+                        data: false,
+                        include_free_versions: false,
+                        include_part_checksums: true,
+                    },
+                )
+                .map_err(std::io::Error::other)?;
+                (bytes::Bytes::from(raw), source, false)
+            };
             let source_dir = source
                 .data_dir
                 .filter(|dir| !dir.is_nil())
@@ -412,23 +445,31 @@ impl LocalDisk {
             let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
             if incoming.data.is_none() && !incoming.is_remote() {
                 let source_data_dir = source_object.join(source_dir.to_string());
-                let pinned_source_data = (!incoming.parts.is_empty())
-                    .then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
-                    .transpose()?;
+                let mut pinned_source_data = None;
                 let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
                 for part in &incoming.parts {
                     if cancelled() {
                         return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
                     }
                     let meta_name = format!("part.{}.meta", part.number);
-                    let bytes = os::read_multipart_source_metadata_in_pinned(
-                        pinned_source_data.as_ref().expect("pinned source data directory"),
-                        &source_data_dir,
-                        std::ffi::OsStr::new(&meta_name),
-                        &base,
-                        &publication_root,
-                    )?;
-                    let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
+                    let actual = if let Some(cached_part) =
+                        super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
+                    {
+                        cached_part
+                    } else {
+                        if pinned_source_data.is_none() {
+                            pinned_source_data =
+                                Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
+                        }
+                        let bytes = os::read_multipart_source_metadata_in_pinned(
+                            pinned_source_data.as_ref().expect("pinned source data directory"),
+                            &source_data_dir,
+                            std::ffi::OsStr::new(&meta_name),
+                            &base,
+                            &publication_root,
+                        )?;
+                        rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
+                    };
                     let normalized = i64::try_from(actual.size)
                         .map_err(std::io::Error::other)?
                         .max(actual.actual_size);
@@ -464,10 +505,15 @@ impl LocalDisk {
                     cancelled,
                 )?;
             }
-            Ok(())
+            let has_state = !from_active_cache
+                && (source_object.join(".mpu-source-state").exists()
+                    || source_object.join(".mpu-recovery").exists()
+                    || source_object.join(".mpu-completions").exists());
+            Ok((!has_state).then_some(raw))
         })
         .await
         .map_err(DiskError::from)?;
+        staging.fresh_source_head = fresh_head;
         Ok(staging)
     }
 }
@@ -1219,4 +1265,211 @@ mod tests {
             .expect("non-part empty hash cleanup must remain supported");
         assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
     }
+
+    #[tokio::test]
+    async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
+        use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
+        let (_root, disk, completed, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
+            .await
+            .expect("fresh multipart upload metadata");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        let read_fi = disk
+            .read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+            .await
+            .expect("cached read_version");
+        assert_eq!(read_fi.data_dir, Some(data_dir));
+
+        let destination = format!("{upload}/{data_dir}/part.1");
+        let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta.clone(),
+        )
+        .await
+        .expect("prepare part transaction");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "in-flight part transaction must mark active source as having unsettled part txn"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta,
+        )
+        .await
+        .expect("rename part");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("settle part transaction");
+        assert!(
+            disk.has_clean_active_multipart_source(&upload, false),
+            "settled commit must restore clean active multipart source status"
+        );
+        let read_parts = disk
+            .read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
+            .await
+            .expect("cached read_parts");
+        assert_eq!(read_parts.len(), 1);
+        assert_eq!(read_parts[0].number, 1);
+
+        // Prepare part 2, then replay Commit for already-settled part 1: part 2's
+        // pending transaction must keep the upload marked as having unsettled part txns.
+        let destination2 = format!("{upload}/{data_dir}/part.2");
+        let mut part2_info = completed.parts[0].clone();
+        part2_info.number = 2;
+        let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part 2");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta.clone(),
+        )
+        .await
+        .expect("prepare part 2 transaction");
+        assert!(!disk.has_clean_active_multipart_source(&upload, false));
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("replayed part 1 commit settlement");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "replayed settlement for part 1 must not clear part 2's pending transaction"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta,
+        )
+        .await
+        .expect("rename part 2");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
+            .await
+            .expect("settle part 2 transaction");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        // Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
+        disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
+            .await
+            .expect("delete xl.meta via delete_paths");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "per-file head deletion must invalidate the canonical upload cache key"
+        );
+        assert!(matches!(
+            disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+                .await,
+            Err(DiskError::FileNotFound)
+        ));
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
+        use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
+
+        let (_root, disk, _, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let disk = Arc::new(disk);
+        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
+        let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        let source_dir = disk
+            .io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
+            .expect("source dir");
+        let base_dir = disk
+            .io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
+            .expect("multipart bucket path");
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel();
+        let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
+            let _ = entered_tx.send(());
+            let _ = release_rx.recv();
+        });
+
+        let init_wrapper = wrapper.clone();
+        let init_upload = upload.clone();
+        let init_task = tokio::spawn(async move {
+            init_wrapper
+                .write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
+            .await
+            .expect("initial write_metadata reaches parent dir fsync")
+            .expect("entered channel");
+
+        let retire_wrapper = wrapper.clone();
+        let retire_path = format!("{upload}/xl.meta");
+        let mut retire_fut = Box::pin(async move {
+            retire_wrapper
+                .delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
+                .await
+        });
+        assert!(
+            futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
+            "concurrent retirement must block on SourceRW while initial head publication is in flight"
+        );
+
+        let _ = release_tx.send(());
+        init_task.await.expect("init join").expect("initial write_metadata succeeds");
+        retire_fut
+            .await
+            .expect("concurrent retirement completes after SourceRW release");
+
+        assert!(
+            !super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
+            "retired source must not remain active in cache after interleaved initial publication"
+        );
+        assert!(
+            matches!(
+                super::super::multipart_source_state::validate_source_mutation(
+                    &source_dir,
+                    &base_dir,
+                    &disk.publication_root,
+                    false
+                ),
+                Err(DiskError::FileNotFound)
+            ),
+            "validate_source_mutation must reject non-initial mutation on retired source"
+        );
+    }
 }

crates/ecstore/src/disk/mod.rs

@@ -27,6 +27,7 @@ mod io_admission;
 #[cfg(test)]
 pub(crate) mod lifecycle_test_io;
 pub mod local;
+pub mod multipart_index;
 pub mod multipart_source;
 pub mod os;
 
@@ -401,16 +402,24 @@ impl Disk {
         }
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        match self {
+            Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
+            Self::Remote(_) => false,
+        }
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
         match self {
             Self::Local(disk) => {
-                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
                     .await
             }
             Self::Remote(_) => Ok(None),
@@ -424,11 +433,19 @@ impl Disk {
         transaction: Uuid,
         journal: &DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         match self {
             Self::Local(disk) => {
-                disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
-                    .await
+                disk.try_finish_and_finalize_prepared_delete_local(
+                    volume,
+                    object,
+                    transaction,
+                    journal,
+                    namespace_owner,
+                    work_budget,
+                )
+                .await
             }
             Self::Remote(_) => Ok(false),
         }
@@ -505,6 +522,17 @@ impl Disk {
 
 #[async_trait::async_trait]
 impl DiskAPI for Disk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        match self {
+            Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+            Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+        }
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         match self {
             Self::Local(disk) => disk.supports_guarded_delete_versions().await,
@@ -548,6 +576,17 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
         }
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        match self {
+            Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+            Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -560,6 +599,35 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        match self {
+            Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+            Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        match self {
+            Self::Local(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+            Self::Remote(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -1506,6 +1574,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn host_name(&self) -> String;
     fn endpoint(&self) -> Endpoint;
     async fn close(&self) -> Result<()>;
+    /// Return candidates only while this disk maintains complete boot-local coverage.
+    /// Unsupported or unready disks must never return an authoritative empty page.
+    async fn list_multipart_index_roots(
+        &self,
+        _bucket: &str,
+        _marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        Err(DiskError::MethodNotAllowed)
+    }
     async fn get_disk_id(&self) -> Result<Option<Uuid>>;
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
 
@@ -1683,6 +1760,33 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     ) -> Result<DeleteRecoveryPage> {
         Err(DiskError::MethodNotAllowed)
     }
+    /// Bounded foreground inventory owns all traversal state until return or cancellation.
+    /// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
+    async fn discover_delete_transactions_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        Err(DiskError::DeleteRecoveryWorkLimit)
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _transaction: Uuid,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        Err(DiskError::DeleteRecoveryWorkLimit)
+    }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _request: BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        Err(DiskError::DeleteRecoveryWorkLimit)
+    }
     /// A read-only capability check; unsupported implementations must fail closed.
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         Err(DiskError::GuardedDeleteUnsupported)
@@ -2068,6 +2172,13 @@ pub struct WalkDirOptions {
     #[serde(default)]
     pub skip_hidden_prefix_check: bool,
 
+    /// Bound hidden metadata read while proving a non-recursive prefix visible.
+    #[serde(default)]
+    pub hidden_entry_budget: usize,
+    /// A prefix whose visibility was not established before the preceding cut.
+    #[serde(default)]
+    pub pending_prefix: Option<String>,
+
     // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
     pub report_notfound: bool,
 
@@ -2400,9 +2511,62 @@ pub enum DeleteRecoveryAction {
     AdoptCommit { authority: Box<DeleteRecoveryJournal> },
 }
 
+/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryDiscovery {
+    pub transactions: Vec<DeleteTransactionId>,
+    pub visited: u32,
+    pub complete: bool,
+}
+
+pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
+pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlement {
+    pub object: String,
+    pub transaction: Uuid,
+    pub expected_fingerprint: DeleteRecoveryDigest,
+    pub action: DeleteRecoveryAction,
+    pub max_entries: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub enum DeleteRecoverySettlementOutcome {
+    Inspected(Box<DeleteRecoveryInspection>),
+    ReceiptPublishedInspectionDeferred,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlementReply {
+    pub outcome: DeleteRecoverySettlementOutcome,
+    pub visited: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryInspection {
+    pub inspection: DeleteRecoveryInspection,
+    pub visited: u32,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub enum DeleteRecoveryRpcOperation {
+    SettleBounded(BoundedDeleteRecoverySettlement),
+    InspectBounded {
+        object: String,
+        transaction: Uuid,
+        max_entries: u32,
+    },
+    DiscoverBounded {
+        object: String,
+        max_entries: u32,
+    },
     InspectOrphanDataDir {
         path: String,
     },
@@ -2440,7 +2604,11 @@ impl DeleteRecoveryRpcOperation {
     /// Full metadata application and adoption share the existing mutation envelope.
     pub fn max_wire_payload_bytes(&self) -> usize {
         match self {
-            Self::ApplyVersion { .. }
+            Self::SettleBounded(BoundedDeleteRecoverySettlement {
+                action: DeleteRecoveryAction::AdoptCommit { .. },
+                ..
+            })
+            | Self::ApplyVersion { .. }
             | Self::ApplyVersions { .. }
             | Self::Settle {
                 action: DeleteRecoveryAction::AdoptCommit { .. },
@@ -2456,8 +2624,11 @@ impl DeleteRecoveryRpcOperation {
 pub enum DeleteRecoveryRpcReply {
     OrphanDataDir { plain: bool },
     Discovered(DeleteRecoveryPage),
+    DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
     Inspected(DeleteRecoveryInspection),
+    InspectedBounded(BoundedDeleteRecoveryInspection),
     Settled(DeleteRecoveryInspection),
+    SettledBounded(BoundedDeleteRecoverySettlementReply),
     AppliedVersion,
     AppliedVersions(Vec<Option<u32>>),
     Evidence { present: bool },
@@ -2935,6 +3106,8 @@ mod tests {
             recursive: true,
             incl_deleted: false,
             skip_hidden_prefix_check: false,
+            hidden_entry_budget: 0,
+            pending_prefix: None,
             report_notfound: false,
             filter_prefix: Some("prefix_".to_string()),
             forward_to: Some("object/path".to_string()),

crates/ecstore/src/disk/multipart_index.rs

@@ -0,0 +1,275 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! A candidate index never replaces multipart metadata or quorum validation.
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
+pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
+
+pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
+
+/// A bounded page from one disk's continuously maintained, boot-local index.
+/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
+/// pages from different epochs or physical disk identities.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexPage {
+    pub bucket: String,
+    pub disk_id: Uuid,
+    pub epoch: Uuid,
+    pub roots: Vec<String>,
+    pub next_marker: Option<String>,
+}
+
+pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
+pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
+
+/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
+/// Old peers may ignore this field, but their ordinary response cannot decode
+/// as the required signed, challenge-bound MultipartIndexRpcReply.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcRequest {
+    pub disk_id: String,
+    pub metrics: bool,
+    pub noop: bool,
+    pub multipart_index: MultipartIndexQuery,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexQuery {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub bucket: String,
+    pub marker: Option<String>,
+    pub page_cap: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcReply {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub page: MultipartIndexPage,
+    pub proof: Vec<u8>,
+}
+
+impl MultipartIndexRpcRequest {
+    pub fn new(bucket: &str, marker: Option<&str>) -> Self {
+        Self {
+            disk_id: String::new(),
+            metrics: false,
+            noop: true,
+            multipart_index: MultipartIndexQuery {
+                operation: "list_multipart_index_roots".into(),
+                version: 1,
+                challenge: Uuid::new_v4(),
+                bucket: bucket.to_owned(),
+                marker: marker.map(str::to_owned),
+                page_cap: MULTIPART_INDEX_PAGE_SIZE,
+            },
+        }
+    }
+
+    pub fn validate(&self) -> crate::disk::error::Result<()> {
+        let query = &self.multipart_index;
+        if !self.disk_id.is_empty()
+            || self.metrics
+            || !self.noop
+            || query.operation != "list_multipart_index_roots"
+            || query.version != 1
+            || query.challenge.is_nil()
+            || query.bucket.is_empty()
+            || query.bucket.len() > 255
+            || query.bucket.contains('/')
+            || matches!(query.bucket.as_str(), "." | "..")
+            || query.page_cap != MULTIPART_INDEX_PAGE_SIZE
+            || query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
+        {
+            return Err(crate::disk::error::DiskError::MethodNotAllowed);
+        }
+        Ok(())
+    }
+
+    pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
+    }
+
+    pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
+    }
+}
+
+fn valid_root(root: &str) -> bool {
+    root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
+}
+
+impl MultipartIndexPage {
+    pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        if expected_disk.is_nil()
+            || self.disk_id != expected_disk
+            || self.epoch.is_nil()
+            || self.bucket != bucket
+            || self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
+            || self
+                .roots
+                .iter()
+                .any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
+            || self.roots.windows(2).any(|pair| pair[0] >= pair[1])
+            || self
+                .next_marker
+                .as_ref()
+                .is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(())
+    }
+}
+
+impl MultipartIndexRpcReply {
+    /// This is structural validation only; the caller must also verify `proof`.
+    pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        let query = &request.multipart_index;
+        request.validate()?;
+        if self.operation != query.operation
+            || self.version != query.version
+            || self.challenge != query.challenge
+            || self.proof.len() != 32
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
+    }
+}
+
+/// A disk can attest coverage only after every page belongs to the same physical
+/// identity and index generation. An interrupted/changed stream contributes no
+/// partial candidates to the erasure set's coverage count.
+pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
+where
+    F: FnMut(Option<String>) -> Fut,
+    Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
+{
+    let mut epoch = None;
+    let mut marker = None;
+    let mut roots = Vec::new();
+    loop {
+        let page = fetch(marker.clone()).await?;
+        page.validate(bucket, marker.as_deref(), identity)?;
+        if epoch.is_some_and(|epoch| epoch != page.epoch) {
+            return Err(crate::disk::error::DiskError::DiskNotFound);
+        }
+        epoch = Some(page.epoch);
+        roots.extend(page.roots);
+        marker = page.next_marker;
+        if marker.is_none() {
+            return Ok(roots);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
+        let request = MultipartIndexRpcRequest::new("bucket", None);
+        let id = Uuid::new_v4();
+        let mut reply = MultipartIndexRpcReply {
+            operation: request.multipart_index.operation.clone(),
+            version: 1,
+            challenge: request.multipart_index.challenge,
+            page: MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch: Uuid::new_v4(),
+                roots: vec!["a".repeat(64)],
+                next_marker: None,
+            },
+            proof: vec![0; 32],
+        };
+        reply.validate(&request, id).expect("valid structural envelope");
+        assert!(
+            reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
+            "old challenge cannot replay"
+        );
+        assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
+        reply.page.next_marker = Some("a".repeat(64));
+        assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
+        reply.page.next_marker = None;
+        reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
+        assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
+    }
+
+    #[test]
+    fn multipart_index_response_proof_binds_the_complete_query_and_page() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let mut request = MultipartIndexRpcRequest::new("bucket", None);
+        let mut page = MultipartIndexPage {
+            bucket: "bucket".into(),
+            disk_id: Uuid::new_v4(),
+            epoch: Uuid::new_v4(),
+            roots: vec![],
+            next_marker: None,
+        };
+        let canonical = request.canonical_response("disk", &page).expect("canonical response");
+        let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
+        let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
+            crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                &request.canonical_response("disk", page).expect("canonical"),
+                &proof,
+            )
+        };
+        verify(&request, &page).expect("exact response");
+        request.multipart_index.marker = Some("0".repeat(64));
+        assert!(verify(&request, &page).is_err(), "cursor is signed");
+        request.multipart_index.marker = None;
+        page.epoch = Uuid::new_v4();
+        assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
+    }
+    #[tokio::test]
+    async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
+        let id = Uuid::new_v4();
+        let epoch = Uuid::new_v4();
+        for replace_disk in [false, true] {
+            let first = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch,
+                roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
+                next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
+            };
+            let second = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: if replace_disk { Uuid::new_v4() } else { id },
+                epoch: if replace_disk { epoch } else { Uuid::new_v4() },
+                roots: vec![],
+                next_marker: None,
+            };
+            let mut replies = std::collections::VecDeque::from([first, second]);
+            let result = collect_pages("bucket", id, |marker| {
+                let page = replies.pop_front().expect("bounded fixture pages");
+                if marker.is_some() {
+                    assert!(page.roots.is_empty());
+                }
+                std::future::ready(Ok(page))
+            })
+            .await;
+            assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
+            assert!(replies.is_empty(), "exercise the continuation response");
+        }
+    }
+}

crates/ecstore/src/error/mod.rs

@@ -269,6 +269,11 @@ pub enum StorageError {
     /// retained for operator inspection.
     #[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
     DeleteRecoveryLegacyAmbiguous,
+    /// Delete recovery exhausted the per-request work budget before it could
+    /// classify every residue. Retrying later makes progress, so callers treat
+    /// it like `SlowDown`; it stays distinct so the disk error round-trips.
+    #[error("delete recovery work limit reached; please retry")]
+    DeleteRecoveryWorkLimit,
     #[error("Decommission not started")]
     DecommissionNotStarted,
     #[error("Decommission already running")]
@@ -480,6 +485,7 @@ impl From<DiskError> for StorageError {
             DiskError::InvalidPath => StorageError::InvalidPath,
             DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
             DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
         }
     }
@@ -507,6 +513,7 @@ impl StorageError {
             StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
             StorageError::StorageFull => DiskError::DiskFull,
             StorageError::SlowDown => DiskError::TooManyOpenFiles,
+            StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
             StorageError::InconsistentDisk => DiskError::InconsistentDisk,
             StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
@@ -710,6 +717,7 @@ impl Clone for StorageError {
             StorageError::StorageFull => StorageError::StorageFull,
             StorageError::SlowDown => StorageError::SlowDown,
             StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
+            StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
             StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
             StorageError::InvalidUploadIDKeyCombination(a, b) => {
                 StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
@@ -852,6 +860,7 @@ impl StorageError {
             StorageError::StorageFull => StorageErrorCode::StorageFull,
             StorageError::SlowDown => StorageErrorCode::SlowDown,
             StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
+            StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
             StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
             StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
             StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
@@ -958,6 +967,7 @@ impl StorageError {
             StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
             StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
             StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
+            StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
             StorageErrorCode::PrefixAccessDenied => {
                 Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
             }

crates/ecstore/src/io_support/rio.rs

@@ -79,6 +79,14 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
     Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
 }
 
+/// Validate that replicated compression metadata can be read by this build.
+///
+/// # Errors
+/// Returns an error for an unsupported compression scheme.
+pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
+    compression_scheme_to_read_plan(scheme).map(|_| ())
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ReadEncryptionBackend {
     Legacy,
@@ -429,6 +437,7 @@ impl WritePlan {
         let checksum = reader.content_hash().clone();
         let trailer = reader.get_trailer().cloned();
 
+        let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
         let encrypted = self.encryption.is_some();
         if let Some(algorithm) = self.compression {
             reader = HashReader::from_reader(
@@ -443,12 +452,15 @@ impl WritePlan {
 
         if let Some(encryption) = self.encryption {
             let etag_nonce = match encryption.mode {
-                WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
-                    base_nonce
-                }
-                WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
+                WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
+                WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
+                // Each part write needs an independent fingerprint, including a
+                // retry at the same part number. This salt affects only the ETag
+                // MAC; encryption framing and its data nonce stay unchanged.
+                WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
             };
-            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
+            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
+                .with_inner_etag_delegation(delegate_inner_etag);
             reader = match encryption.mode {
                 WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]
@@ -556,6 +568,54 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
+        let plaintext = b"same confidential part";
+        let size = i64::try_from(plaintext.len()).expect("test length");
+        for object_key_mode in [false, true] {
+            let mut tags = std::collections::HashSet::new();
+            for part_number in [1, 2, 1] {
+                let encryption = if object_key_mode {
+                    WriteEncryption::multipart_object_key([7; 32], part_number)
+                } else {
+                    WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
+                };
+                let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
+                let mut reader = WritePlan::new()
+                    .with_encryption(encryption)
+                    .apply(source, size)
+                    .expect("part plan");
+                let mut ciphertext = Vec::new();
+                reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
+                let tag = reader.try_resolve_etag().expect("part fingerprint");
+                assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
+                assert!(
+                    tags.insert(tag),
+                    "different part numbers and retries must not identify matching plaintext"
+                );
+                #[cfg(feature = "rio-v2")]
+                let mut decrypt = if object_key_mode {
+                    DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
+                } else {
+                    DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
+                };
+                #[cfg(not(feature = "rio-v2"))]
+                let mut decrypt = DecryptReader::new_multipart(
+                    Cursor::new(ciphertext),
+                    [7; 32],
+                    if object_key_mode { [0; 12] } else { [9; 12] },
+                    vec![part_number as usize],
+                );
+                let mut decoded = Vec::new();
+                decrypt
+                    .read_to_end(&mut decoded)
+                    .await
+                    .expect("existing data nonce still decrypts");
+                assert_eq!(decoded, plaintext);
+            }
+        }
+    }
+
     async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
         let plaintext = b"checksum-context-through-write-plan".repeat(256);
         let actual_size = plaintext.len() as i64;

crates/ecstore/src/io_support/shard_integrity.rs

@@ -333,6 +333,34 @@ struct ProofCache {
     preferred: usize,
 }
 
+type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
+
+static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
+    std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
+
+fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .get(&(part.generation, part.root, stripe))
+        .cloned()
+}
+
+fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
+    let mut map = VERIFIED_STRIPE_RECORDS.lock();
+    if map.len() >= 4096
+        && let Some(evict_key) = map.keys().next().copied()
+    {
+        map.remove(&evict_key);
+    }
+    map.insert((part.generation, part.root, stripe), record);
+}
+
+pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .retain(|(stored_gen, _, _), _| *stored_gen != generation);
+}
+
 impl PartProofReader {
     pub(crate) async fn reconstruction_proof(
         &self,
@@ -410,6 +438,9 @@ impl PartProofReader {
 
     pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
         let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
+        if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
+            return Ok(bytes);
+        }
         let mut cached = self.cached.lock().await;
         if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
             return Ok(bytes.clone());
@@ -465,6 +496,9 @@ impl PartProofReader {
             bytes
         };
         self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
+        if self.part.stripes <= 4 {
+            put_verified_stripe_record(&self.part, stripe, record.clone());
+        }
         if cached.records.len() == 2 {
             cached.records.pop_front();
         }
@@ -863,4 +897,41 @@ mod tests {
             prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
         }
     }
+
+    #[tokio::test]
+    async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
+        let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
+        let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
+        let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
+        builder.push(shards.into_iter()).await.expect("stripe");
+        let prepared = builder.finish(8).await.expect("index");
+        let inline_bytes = prepared.inline_bytes().expect("inline bytes");
+        let part = prepared.part;
+
+        let warm_reader = PartProofReader {
+            part: part.clone(),
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: Some(inline_bytes),
+            cached: Mutex::new(ProofCache::default()),
+        };
+        warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
+        assert!(
+            get_verified_stripe_record(&part, 0).is_some(),
+            "small verified stripe record should populate the verified stripe cache"
+        );
+
+        let cached_reader = Arc::new(PartProofReader {
+            part,
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: None,
+            cached: Mutex::new(ProofCache::default()),
+        });
+        cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
+        assert!(
+            cached_reader.verify(0, 0, shards[1]).await.is_err(),
+            "cached record must still reject corrupt or mismatched shard payloads"
+        );
+    }
 }

crates/ecstore/src/object_api/types.rs

@@ -949,8 +949,65 @@ pub enum ShardIntegrityWriteMode {
     Protected,
 }
 
+/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
+/// Credits cover directory entries (including a minimum charge for an empty probe) and
+/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
+#[derive(Debug)]
+pub struct DeleteRecoveryWorkBudget {
+    remaining: std::sync::atomic::AtomicU32,
+}
+
+/// Entries one disk may reserve up front per recovery step (discovery plus
+/// inspection). Every disk of every set a request fans out to reserves this
+/// much before any refund arrives, so a budget must cover that many disks.
+const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
+
+impl Default for DeleteRecoveryWorkBudget {
+    /// Covers a full 16-drive set with headroom; `for_disks` scales it when a
+    /// request spans more drives than that.
+    fn default() -> Self {
+        Self::new(16 * 1024 * 1024)
+    }
+}
+
+impl DeleteRecoveryWorkBudget {
+    /// A per-request budget sized for `disks` concurrent up-front reservations.
+    pub(crate) fn for_disks(disks: usize) -> Self {
+        let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
+        let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
+        let default = u64::from(Self::default().remaining());
+        Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
+    }
+
+    pub(crate) fn new(entries: u32) -> Self {
+        Self {
+            remaining: std::sync::atomic::AtomicU32::new(entries),
+        }
+    }
+
+    pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
+        use std::sync::atomic::Ordering;
+        self.remaining
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
+                (remaining > 0).then(|| remaining.saturating_sub(maximum))
+            })
+            .map(|remaining| remaining.min(maximum))
+            .map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
+    }
+
+    pub(crate) fn remaining(&self) -> u32 {
+        self.remaining.load(std::sync::atomic::Ordering::Acquire)
+    }
+
+    pub(crate) fn refund(&self, unused: u32) {
+        self.remaining.fetch_add(unused, std::sync::atomic::Ordering::Release);
+    }
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
+    #[doc(hidden)]
+    pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
     // Use the maximum parity (N/2), used when saving server configuration files
     pub max_parity: bool,
     pub mod_time: Option<OffsetDateTime>,
@@ -1102,6 +1159,12 @@ pub struct ObjectOptions {
     pub expiration: ExpirationOptions,
     pub lifecycle_audit_event: LcAuditEvent,
 
+    /// In-process request gate evaluated on authoritative completion metadata.
+    /// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
+    /// implementations may own customer encryption keys.
+    #[doc(hidden)]
+    pub multipart_completion_authorization:
+        Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
     pub eval_metadata: Option<HashMap<String, String>>,
     /// Internal compare-and-set condition for replication workers publishing
     /// terminal status after remote I/O. Storage validates it while holding
@@ -1115,6 +1178,10 @@ pub struct ObjectOptions {
     /// Authoritative bucket Object Lock snapshot installed inside `ECStore`
     /// before a destructive commit reaches the set layer.
     pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
+    /// ECStore single PUTs acquire their destructive-write configuration guards
+    /// only after staging the client body. External lock owners supply a snapshot.
+    #[doc(hidden)]
+    pub defer_put_object_lock_snapshot: bool,
 
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,
@@ -1146,6 +1213,11 @@ pub enum ReplicationStatusWritebackMode {
 }
 
 impl ObjectOptions {
+    pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
+        self.delete_recovery_work_budget
+            .get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
+    }
+
     pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
         match self.shard_integrity_write_mode {
             Some(ShardIntegrityWriteMode::Protected) => true,

crates/ecstore/src/services/rebalance/worker.rs

@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     }
 }
 
+fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
+    match rebalance_error_source(err) {
+        Error::Lock(err) => is_rebalance_transient_lock_error(err),
+        // No metadata was written when lock admission failed. Each attempt
+        // must acquire its quorum again before loading or saving this run.
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket,
+            object,
+            ..
+        } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
+        _ => false,
+    }
+}
+
 fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return true;
@@ -385,9 +400,7 @@ where
         match result {
             Ok(value) => return Ok(value),
             Err(err) => {
-                if attempt.saturating_add(1) >= max_attempts.max(1)
-                    || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
-                {
+                if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
                     return Err(err);
                 }
                 #[cfg(test)]
@@ -719,6 +732,82 @@ mod error_source_tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
+        let mut attempts = 0;
+        retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(if attempts == 1 {
+                resolve_rebalance_meta_save_result(
+                    Err(rebalance_meta_lock_error(
+                        rustfs_lock::LockError::QuorumNotReached {
+                            required: 1,
+                            achieved: 0,
+                        },
+                        "write",
+                    )),
+                    "stop_rebalance",
+                )
+            } else {
+                Ok(())
+            })
+        })
+        .await
+        .expect("a later quorum-protected admission must be allowed to succeed");
+        assert_eq!(attempts, 2);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
+        for max_attempts in [0, 1, 3] {
+            let mut attempts = 0;
+            let result = retry_rebalance_metadata_access(None, max_attempts, || {
+                attempts += 1;
+                std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )))
+            })
+            .await;
+            let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
+            assert_eq!(attempts, max_attempts.max(1));
+            assert!(matches!(
+                rebalance_error_source(&error),
+                Error::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    required: 1,
+                    achieved: 0,
+                    ..
+                }
+            ));
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
+        let mut attempts = 0;
+        let result = retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(Err::<(), _>(if attempts == 1 {
+                rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )
+            } else {
+                Error::OperationCanceled
+            }))
+        })
+        .await;
+        assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
+        assert!(matches!(result, Err(Error::OperationCanceled)));
+    }
+
     #[tokio::test]
     async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
         for err in [
@@ -731,6 +820,20 @@ mod error_source_tests {
                 required: 3,
                 achieved: 2,
             },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "data-bucket".to_string(),
+                object: REBAL_META_NAME.to_string(),
+                required: 1,
+                achieved: 0,
+            },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
+                object: "other-metadata.bin".to_string(),
+                required: 1,
+                achieved: 0,
+            },
             Error::other("stale rebalance run rejected: lock acquisition timed out"),
             Error::other("rebalance distributed run fence lost"),
             Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),

crates/ecstore/src/services/tier/test_util.rs

@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
             bytes.len()
         };
 
-        Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
+        Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
     }
 
     async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {

crates/ecstore/src/services/tier/tier.rs

@@ -37,7 +37,6 @@ use std::{
     time::Duration,
 };
 use time::OffsetDateTime;
-use tokio::io::BufReader;
 use tokio::{
     select,
     sync::{Notify, RwLock},
@@ -8183,7 +8182,7 @@ mod tests {
         }
 
         async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
-            Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
+            Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
         }
 
         async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
@@ -8248,7 +8247,7 @@ mod tests {
             _opts: WarmBackendGetOpts,
         ) -> std::result::Result<ReadCloser, std::io::Error> {
             if self.healthy {
-                Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
+                Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
             } else {
                 Err(std::io::Error::other("mock get failed"))
             }
@@ -9895,7 +9894,7 @@ mod tests {
         }
 
         async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
-            Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
+            Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
         }
 
         async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -5340,6 +5340,14 @@ impl SetDisks {
         }
 
         let disks = self.get_disks_internal().await;
+        if disks.len() == self.set_drive_count
+            && disks.iter().all(|disk| {
+                disk.as_ref()
+                    .is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
+            })
+        {
+            return Ok(false);
+        }
         let transaction_path = part_transaction_path(dst_object);
         let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
 
@@ -5539,6 +5547,14 @@ impl SetDisks {
         write_quorum: usize,
     ) -> disk::error::Result<()> {
         let disks = self.get_disks_internal().await;
+        if disks.len() == self.set_drive_count
+            && disks.iter().all(|disk| {
+                disk.as_ref()
+                    .is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
+            })
+        {
+            return Ok(());
+        }
         let listings = join_all(disks.iter().map(|disk| {
             let disk = disk.clone();
             async move {
@@ -5746,7 +5762,7 @@ impl SetDisks {
         let all_prepared_succeeded = errs
             .iter()
             .zip(prepared_disks.iter())
-            .all(|(err, disk)| disk.is_none() || err.is_none());
+            .all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
         let committed = if all_prepared_succeeded {
             let settles = prepared_disks.iter().map(|disk| {
                 let disk = disk.clone();

crates/ecstore/src/set_disk/metadata.rs

@@ -161,7 +161,7 @@ impl SetDisks {
             )
         });
         let mut metadata = FileMeta::new();
-        for parts in versions.into_values() {
+        for (version_id, parts) in versions {
             let errors = parts
                 .iter()
                 .zip(&missing)
@@ -173,19 +173,36 @@ impl SetDisks {
                     }
                 })
                 .collect::<Vec<_>>();
-            let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
-                Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
+            let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
+                Ok((read, write)) => (
+                    usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
+                    usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
+                ),
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
                 Err(DiskError::ErasureReadQuorum) if complete => continue,
                 Err(error) => return Err(error),
             };
-            let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
-            let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
+            let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
+                // A null-version system record must carry the same commit proof
+                // as GET. Listing only needs the selected metadata, so no disk
+                // handles are retained from the shared read selector.
+                Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
+            } else {
+                let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
+                Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
+            };
             metadata.add_version(selected)?;
         }
         if metadata.versions.is_empty() {
-            return Ok(None);
+            // With no decodable version, the per-version quorum loop cannot
+            // reject corrupt or unavailable replicas. Only complete, readable
+            // evidence may establish that this key has no authoritative version.
+            return if complete {
+                Ok(None)
+            } else {
+                Err(DiskError::ErasureReadQuorum)
+            };
         }
         Ok(Some(MetaCacheEntry {
             name,

crates/ecstore/src/set_disk/mod.rs

@@ -357,8 +357,7 @@ use crate::io_support::rio::HashReader;
 
 pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
 pub const MAX_PARTS_COUNT: usize = 10000;
-pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
-pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
+pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
 pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
 const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
 
@@ -4696,6 +4695,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
             .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
 }
 
+// Reusing a held writer additionally requires the same quorum, so repeated
+// clients must not give different weight to an otherwise identical domain.
+fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    !left.is_empty()
+        && left.len() == right.len()
+        && left.iter().all(|client| {
+            left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+                && right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+        })
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5645,9 +5655,9 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            (true, true) => {
-                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
-            }
+            // Distributed lock clients receive the raw ObjectKey; the pool
+            // namespace is not part of the key stored by their backends.
+            (true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
             _ => false,
         }
     }
@@ -9038,6 +9048,13 @@ mod tests {
             "constructing an object lock must not clone each client Arc"
         );
 
+        // Check shared ownership before acquisition can spawn pending cleanup.
+        let read_lock = set
+            .new_ns_lock("bucket", "read-object")
+            .await
+            .expect("second namespace lock should be created");
+        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
+
         let write_error = write_lock
             .get_write_lock(Duration::from_millis(500))
             .await
@@ -9052,11 +9069,6 @@ mod tests {
             ),
             "the shared client representation must preserve the exact write quorum result: {write_error}"
         );
-        let read_lock = set
-            .new_ns_lock("bucket", "read-object")
-            .await
-            .expect("second namespace lock should be created");
-        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
         let read_guard = read_lock
             .get_read_lock(Duration::from_millis(500))
             .await
@@ -9880,7 +9892,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn namespace_lock_domain_includes_pool_namespace() {
+    async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
 
         let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9919,11 +9931,25 @@ mod tests {
             "sets in the same pool share the object namespace lock domain"
         );
         assert!(
-            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pool namespaces must not be deduplicated solely by identical clients"
+            same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
+            "different pools using identical backends lock the same raw object key"
         );
     }
 
+    #[test]
+    fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
+        let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
+        assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
+        assert!(!same_distributed_lock_quorum(
+            &[a.clone(), a.clone(), b.clone()],
+            &[a.clone(), b.clone(), b]
+        ));
+        assert!(!same_distributed_lock_quorum(&[], &[]));
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -141,6 +141,7 @@ pub(crate) struct DeleteRecoverySummary {
 #[derive(Clone)]
 pub(crate) struct DeleteRecoveryContext<'a> {
     pub incarnation: Option<Uuid>,
+    pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     pub dry_run: bool,
     pub explicit_delete: bool,
     /// Only the locked heal path may repair a live marker-creation source.
@@ -156,6 +157,7 @@ impl<'a> DeleteRecoveryContext<'a> {
     pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
         Self {
             incarnation,
+            work_budget: None,
             dry_run: false,
             explicit_delete: false,
             repair_marker_source: false,
@@ -230,19 +232,54 @@ async fn inspect(
     bucket: &str,
     object: &str,
     transaction: Uuid,
+    budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
 ) -> Vec<Result<DeleteRecoveryInspection>> {
     join_all(disks.iter().map(|disk| async move {
-        let state = disk
-            .as_ref()
-            .ok_or(DiskError::DiskNotFound)?
-            .inspect_delete_transaction(bucket, object, transaction)
-            .await?;
+        let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
+        let state = if let Some(budget) = budget {
+            let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
+            let reply = disk
+                .inspect_delete_transaction_bounded(bucket, object, transaction, reserved)
+                .await?;
+            if reply.visited > reserved {
+                return Err(DiskError::FileCorrupt);
+            }
+            budget.refund(reserved - reply.visited.max(1));
+            reply.inspection
+        } else {
+            disk.inspect_delete_transaction(bucket, object, transaction).await?
+        };
         validate(&state, object, transaction)?;
         Ok(state)
     }))
     .await
 }
 
+async fn discover_bounded(
+    disk: &DiskStore,
+    bucket: &str,
+    object: &str,
+    budget: &crate::object_api::DeleteRecoveryWorkBudget,
+) -> Result<Vec<DeleteTransactionId>> {
+    let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
+    let page = disk.discover_delete_transactions_bounded(bucket, object, reserved).await?;
+    if page.visited > reserved
+        || page.transactions.len() > 256
+        || page
+            .transactions
+            .iter()
+            .any(|id| id.object != object || id.transaction.is_nil())
+        || (!page.complete && !page.transactions.is_empty())
+    {
+        return Err(DiskError::FileCorrupt);
+    }
+    budget.refund(reserved - page.visited.max(1));
+    if !page.complete {
+        return Err(DiskError::DeleteRecoveryWorkLimit);
+    }
+    Ok(page.transactions)
+}
+
 async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
     let mut cursor = None;
     let mut ids = BTreeSet::new();
@@ -273,30 +310,92 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
         .collect())
 }
 
+#[derive(Clone, Copy)]
+struct RecoveryWorkScope<'a> {
+    namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
+    budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
+}
+
+async fn settle_outcome(
+    disk: &DiskStore,
+    bucket: &str,
+    state: &DeleteRecoveryInspection,
+    action: DeleteRecoveryAction,
+    can_commit: &(impl Fn() -> bool + Sync + ?Sized),
+    scope: RecoveryWorkScope<'_>,
+) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
+    use crate::disk::DeleteRecoverySettlementOutcome;
+    if !can_commit() {
+        return Err(DiskError::other("delete recovery namespace fence was lost"));
+    }
+    let outcome = if let Some(budget) = scope.budget {
+        let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
+        let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
+        let reply = disk
+            .settle_delete_transaction_bounded(
+                bucket,
+                crate::disk::BoundedDeleteRecoverySettlement {
+                    object: state.id.object.clone(),
+                    transaction: state.id.transaction,
+                    expected_fingerprint: state.fingerprint,
+                    action,
+                    max_entries: reserved,
+                },
+                scope.namespace_owner.cloned(),
+            )
+            .await?;
+        if reply.visited > reserved
+            || (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
+        {
+            return Err(DiskError::FileCorrupt);
+        }
+        budget.refund(reserved - reply.visited.max(1));
+        reply.outcome
+    } else {
+        DeleteRecoverySettlementOutcome::Inspected(Box::new(
+            disk.settle_delete_transaction_with_namespace_owner(
+                bucket,
+                &state.id.object,
+                state.id.transaction,
+                state.fingerprint,
+                action,
+                scope.namespace_owner.cloned(),
+            )
+            .await?,
+        ))
+    };
+    if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
+        validate(next, &state.id.object, state.id.transaction)?;
+    }
+    Ok(outcome)
+}
+
 async fn settle(
     disk: &DiskStore,
     bucket: &str,
     state: &DeleteRecoveryInspection,
     action: DeleteRecoveryAction,
     can_commit: &(impl Fn() -> bool + Sync + ?Sized),
-    namespace_owner: Option<&Arc<dyn Send + Sync>>,
+    scope: RecoveryWorkScope<'_>,
 ) -> Result<DeleteRecoveryInspection> {
-    if !can_commit() {
-        return Err(DiskError::other("delete recovery namespace fence was lost"));
+    match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
+        crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
+        crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
+            Err(DiskError::DeleteRecoveryWorkLimit)
+        }
+    }
+}
+
+fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
+    outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
+}
+
+fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
+    match outcome {
+        Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
+        Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
+        Err(_) => false,
     }
-    let result = disk
-        .settle_delete_transaction_with_namespace_owner(
-            bucket,
-            &state.id.object,
-            state.id.transaction,
-            state.fingerprint,
-            action,
-            namespace_owner.cloned(),
-        )
-        .await;
-    let next = result?;
-    validate(&next, &state.id.object, state.id.transaction)?;
-    Ok(next)
 }
 
 /// Each phase fans out across independent drives, and awaits all started work
@@ -307,23 +406,16 @@ async fn settle_group(
     copies: &mut [DeleteRecoveryInspection],
     action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
     can_commit: &(impl Fn() -> bool + Sync + ?Sized),
-    namespace_owner: Option<&Arc<dyn Send + Sync>>,
+    scope: RecoveryWorkScope<'_>,
     removed_versions: &mut HashSet<Uuid>,
 ) -> Result<bool> {
     let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
         let action = action(copy);
         async move {
             match action {
-                Some(action) => settle(
-                    disk.as_ref().ok_or(DiskError::DiskNotFound)?,
-                    bucket,
-                    copy,
-                    action,
-                    can_commit,
-                    namespace_owner,
-                )
-                .await
-                .map(Some),
+                Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
+                    .await
+                    .map(Some),
                 None => Ok(None),
             }
         }
@@ -439,16 +531,31 @@ impl SetDisks {
                     true
                 }
             };
+        let mut fast_receipt_quorum = false;
         if can_try_local_fast_path && can_commit() {
             let fast_published = join_all(disks.iter().map(|disk| async {
                 if !can_commit() {
                     return Err(DiskError::other("delete recovery namespace fence was lost"));
                 }
                 let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
-                disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
-                    .await
+                disk.try_publish_prepared_delete_local(
+                    bucket,
+                    object,
+                    transaction,
+                    context.namespace_owner.clone(),
+                    context.work_budget.clone(),
+                )
+                .await
             }))
             .await;
+            fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
+            if fast_receipt_quorum
+                && fast_published
+                    .iter()
+                    .any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
+            {
+                return Ok(());
+            }
             let all_fast_published = fast_published.len() == self.set_drive_count
                 && fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
             if all_fast_published && can_commit() {
@@ -456,6 +563,7 @@ impl SetDisks {
                 test_hooks::record_cleanup_attempt(bucket, object);
                 let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
                     let ns_owner = context.namespace_owner.clone();
+                    let work_budget = context.work_budget.clone();
                     async move {
                         if !can_commit() {
                             return Ok(false);
@@ -466,11 +574,24 @@ impl SetDisks {
                         let Ok(Some((journal, _, _))) = pub_res else {
                             return Ok(false);
                         };
-                        disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
-                            .await
+                        disk.try_finish_and_finalize_prepared_delete_local(
+                            bucket,
+                            object,
+                            transaction,
+                            journal,
+                            ns_owner,
+                            work_budget,
+                        )
+                        .await
                     }
                 }))
                 .await;
+                if finalized
+                    .iter()
+                    .any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
+                {
+                    return Ok(());
+                }
                 if finalized.iter().all(|res| matches!(res, Ok(true))) {
                     if let Some(ticket) = context.recovery_ticket.as_ref() {
                         self.ctx.delete_recovery_hints().ack(ticket);
@@ -479,7 +600,17 @@ impl SetDisks {
                 }
             }
         }
-        let states = inspect(disks, bucket, object, transaction).await;
+        let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
+        if states
+            .iter()
+            .any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
+        {
+            return if fast_receipt_quorum {
+                Ok(())
+            } else {
+                Err(DiskError::DeleteRecoveryWorkLimit)
+            };
+        }
         let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
         if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
             && states.iter().all(|state| {
@@ -499,25 +630,50 @@ impl SetDisks {
             if state.state == DeleteRecoveryState::LegacyAmbiguous {
                 return Err(DiskError::FileCorrupt);
             }
-            settle(
+            settle_outcome(
                 disk,
                 bucket,
                 state,
                 DeleteRecoveryAction::PublishCommit,
                 can_commit,
-                context.namespace_owner.as_ref(),
+                RecoveryWorkScope {
+                    namespace_owner: context.namespace_owner.as_ref(),
+                    budget: context.work_budget.as_deref(),
+                },
             )
             .await
         }))
         .await;
-        if published
-            .iter()
-            .filter(|state| matches!(state, Ok(state) if receipt(state.state)))
-            .count()
-            < write_quorum
-        {
-            return Err(DiskError::ErasureWriteQuorum);
+        if !has_receipt_quorum(&published, write_quorum) {
+            return Err(
+                if published
+                    .iter()
+                    .any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
+                {
+                    DiskError::DeleteRecoveryWorkLimit
+                } else {
+                    DiskError::ErasureWriteQuorum
+                },
+            );
         }
+        if published.iter().any(|outcome| {
+            matches!(
+                outcome,
+                Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
+            )
+        }) {
+            return Ok(());
+        }
+        let published = published
+            .into_iter()
+            .map(|outcome| match outcome {
+                Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
+                Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
+                    Err(DiskError::DeleteRecoveryWorkLimit)
+                }
+                Err(error) => Err(error),
+            })
+            .collect::<Vec<_>>();
         // Physical cleanup is retryable after the certified decision. An
         // unavailable participant retains the decision for its eventual return.
         let ticket = context.recovery_ticket.clone();
@@ -670,6 +826,7 @@ impl SetDisks {
         }
         let DeleteRecoveryContext {
             incarnation,
+            work_budget,
             dry_run,
             explicit_delete,
             repair_marker_source,
@@ -681,7 +838,13 @@ impl SetDisks {
         } = context;
         let disks = self.disk_inventory().await;
         let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
-        let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
+        let pages = join_all(disks.iter().flatten().map(|disk| async {
+            match work_budget.as_deref() {
+                Some(budget) => discover_bounded(disk, bucket, object, budget).await,
+                None => discover(disk, bucket, Some(object)).await,
+            }
+        }))
+        .await;
         let mut transactions = BTreeSet::new();
         let mut inspected = 0;
         let mut offline = !complete_inventory;
@@ -712,6 +875,7 @@ impl SetDisks {
             ..Default::default()
         };
         let mut all_states = BTreeMap::new();
+        let mut registered_evidence_hint = false;
         for transaction in transactions {
             let mut complete = Vec::with_capacity(disks.len());
             let inspections = if let Some((published_tx, published_states)) = published_hint
@@ -723,8 +887,14 @@ impl SetDisks {
             {
                 published_states.to_vec()
             } else {
-                inspect(&disks, bucket, object, transaction).await
+                inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
             };
+            if inspections
+                .iter()
+                .any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
+            {
+                return Err(DiskError::DeleteRecoveryWorkLimit);
+            }
             for state in inspections {
                 match state {
                     Ok(state) => {
@@ -736,6 +906,33 @@ impl SetDisks {
                                 summary.blocked += 1;
                                 return Ok(summary);
                             }
+                            // Candidate UUID directories are not evidence. Only a bound, inspected
+                            // journal can create a foreground hint; empty/partial scans rely on the
+                            // existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
+                            if work_budget.is_some()
+                                && !registered_evidence_hint
+                                && journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
+                                && journal.volume == bucket
+                                && journal.object == object
+                                && journal.transaction == transaction
+                                && matches!(
+                                    state.state,
+                                    DeleteRecoveryState::NewPrepared
+                                        | DeleteRecoveryState::QuorumCommitted
+                                        | DeleteRecoveryState::CommitFinished
+                                )
+                            {
+                                self.ctx.delete_recovery_hints().register(
+                                    crate::store::delete_recovery_runtime::DeleteRecoveryKey {
+                                        pool: self.pool_index,
+                                        set: self.set_index,
+                                        bucket: bucket.to_owned(),
+                                        object: object.to_owned(),
+                                        incarnation,
+                                    },
+                                );
+                                registered_evidence_hint = true;
+                            }
                         }
                         complete.push(state);
                     }
@@ -767,7 +964,7 @@ impl SetDisks {
         };
         if let Some((transaction, source)) = prerequisite {
             Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
-            let refreshed = inspect(&disks, bucket, object, transaction)
+            let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
                 .await
                 .into_iter()
                 .collect::<Result<Vec<_>>>()?;
@@ -853,7 +1050,7 @@ impl SetDisks {
             if mutated {
                 // A sibling may have restored metadata or reclaimed shared
                 // evidence. Refresh fingerprints before acting on this group.
-                *copies = inspect(&disks, bucket, object, transaction)
+                *copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
                     .await
                     .into_iter()
                     .collect::<Result<Vec<_>>>()?;
@@ -876,7 +1073,10 @@ impl SetDisks {
                             (copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
                         },
                         can_commit,
-                        namespace_owner.as_ref(),
+                        RecoveryWorkScope {
+                            namespace_owner: namespace_owner.as_ref(),
+                            budget: work_budget.as_deref(),
+                        },
                         &mut summary.removed_versions,
                     )
                     .await?
@@ -987,7 +1187,10 @@ impl SetDisks {
                         )
                     },
                     can_commit,
-                    namespace_owner.as_ref(),
+                    RecoveryWorkScope {
+                        namespace_owner: namespace_owner.as_ref(),
+                        budget: work_budget.as_deref(),
+                    },
                     &mut summary.removed_versions,
                 )
                 .await?;
@@ -1001,7 +1204,10 @@ impl SetDisks {
                                 .then_some(DeleteRecoveryAction::FinishCommit)
                         },
                         can_commit,
-                        namespace_owner.as_ref(),
+                        RecoveryWorkScope {
+                            namespace_owner: namespace_owner.as_ref(),
+                            budget: work_budget.as_deref(),
+                        },
                         &mut summary.removed_versions,
                     )
                     .await?;
@@ -1048,7 +1254,10 @@ impl SetDisks {
                     copies,
                     |copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
                     can_commit,
-                    namespace_owner.as_ref(),
+                    RecoveryWorkScope {
+                        namespace_owner: namespace_owner.as_ref(),
+                        budget: work_budget.as_deref(),
+                    },
                     &mut summary.removed_versions,
                 )
                 .await?
@@ -1076,7 +1285,7 @@ impl SetDisks {
                 settled.push(copies);
             } else {
                 for transaction in all_states.keys() {
-                    let fresh = inspect(&disks, bucket, object, *transaction).await;
+                    let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
                     if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
                         summary.pending += 1;
                         return Ok(summary);
@@ -1086,7 +1295,7 @@ impl SetDisks {
             }
             for (index, mut copies) in settled.into_iter().enumerate() {
                 if index > 0 {
-                    copies = inspect(&disks, bucket, object, copies[0].id.transaction)
+                    copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
                         .await
                         .into_iter()
                         .collect::<Result<Vec<_>>>()?;
@@ -1106,7 +1315,10 @@ impl SetDisks {
                     &mut copies,
                     |copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
                     can_commit,
-                    namespace_owner.as_ref(),
+                    RecoveryWorkScope {
+                        namespace_owner: namespace_owner.as_ref(),
+                        budget: work_budget.as_deref(),
+                    },
                     &mut summary.removed_versions,
                 )
                 .await?
@@ -1358,6 +1570,457 @@ mod tests {
         body
     }
 
+    #[tokio::test]
+    async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
+        let bucket = "recover-bounded-mixed-quorum";
+        let (_dirs, store, disks) = fixture(bucket).await;
+        put(&store, bucket, &vec![73; 4096], false).await;
+        let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
+        let mut outcomes = Vec::new();
+        for (index, disk) in disks.iter().enumerate() {
+            let state = disk
+                .inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
+                .await
+                .expect("preparation inspection");
+            let max_entries = match index {
+                0 => 65536,
+                1 | 2 => state.visited,
+                _ => 1,
+            };
+            outcomes.push(
+                disk.settle_delete_transaction_bounded(
+                    bucket,
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "object".to_owned(),
+                        transaction,
+                        expected_fingerprint: state.inspection.fingerprint,
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries,
+                    },
+                    None,
+                )
+                .await
+                .map(|reply| reply.outcome),
+            );
+        }
+        assert!(
+            matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
+        );
+        assert!(matches!(
+            &outcomes[1],
+            Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
+        ));
+        assert!(outcomes[3].is_err());
+        assert!(has_receipt_quorum(&outcomes, 3));
+        assert!(!has_receipt_quorum(&outcomes, 4));
+        outcomes[2] = Err(DiskError::FileAccessDenied);
+        assert!(
+            !has_receipt_quorum(&outcomes, 3),
+            "a write failure cannot substitute for a durable receipt"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
+        use crate::object_api::DeleteRecoveryWorkBudget;
+        use crate::storage_api_contracts::object::ObjectToDelete;
+        let bucket = "recover-bounded-request";
+        let (dirs, store, _) = fixture(bucket).await;
+        for dir in &dirs {
+            let prefix = dir.path().join(bucket).join("prefix");
+            std::fs::create_dir_all(&prefix).expect("physical prefix");
+            for index in 0..64 {
+                std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+            }
+        }
+        let error = store
+            .delete_object(
+                bucket,
+                "prefix",
+                ObjectOptions {
+                    delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("incomplete discovery is not absence");
+        assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
+        for incarnation in [
+            None,
+            Some(
+                store
+                    .bucket_incarnation_id_from_disk(bucket)
+                    .await
+                    .expect("bucket incarnation"),
+            ),
+        ] {
+            assert!(
+                !store
+                    .ctx
+                    .delete_recovery_hints()
+                    .is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
+                        pool: 0,
+                        set: 0,
+                        bucket: bucket.to_owned(),
+                        object: "prefix".to_owned(),
+                        incarnation,
+                    }),
+                "an unproven prefix must not move its scan into background recovery"
+            );
+        }
+        for distinct_versions in [false, true] {
+            let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
+            let objects = (0..1000)
+                .map(|_| ObjectToDelete {
+                    object_name: "prefix".to_owned(),
+                    version_id: distinct_versions.then(Uuid::new_v4),
+                    ..Default::default()
+                })
+                .collect();
+            let (_, errors) = store
+                .delete_objects(
+                    bucket,
+                    objects,
+                    ObjectOptions {
+                        delete_recovery_work_budget: Some(Arc::clone(&budget)),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(
+                errors
+                    .iter()
+                    .any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
+                "repeated actions must not each obtain a new allowance: {distinct_versions}"
+            );
+            assert!(
+                dirs.iter()
+                    .all(|dir| dir.path().join(bucket).join("prefix/child-63").is_dir())
+            );
+        }
+        let error = store
+            .delete_object(bucket, "prefix", ObjectOptions::default())
+            .await
+            .expect_err("ordinary missing key");
+        assert!(
+            crate::error::is_err_object_not_found(&error),
+            "small complete inventory retains no-op semantics: {error:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
+        use crate::layout::endpoint::Endpoint;
+        use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
+        use crate::runtime::instance::InstanceContext;
+        let mut dirs = Vec::new();
+        let mut pools = Vec::new();
+        for pool_index in 0..2 {
+            let mut endpoints = Vec::new();
+            for set_index in 0..2 {
+                for disk_index in 0..4 {
+                    let dir = tempfile::tempdir().expect("temporary disk");
+                    let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
+                    endpoint.set_pool_index(pool_index);
+                    endpoint.set_set_index(set_index);
+                    endpoint.set_disk_index(disk_index);
+                    endpoints.push(endpoint);
+                    dirs.push(dir);
+                }
+            }
+            pools.push(PoolEndpoints {
+                legacy: false,
+                set_count: 2,
+                drives_per_set: 4,
+                endpoints: Endpoints::from(endpoints),
+                cmd_line: format!("bounded-recovery-{pool_index}"),
+                platform: "test".to_owned(),
+            });
+        }
+        let endpoints = EndpointServerPools(pools);
+        let ctx = Arc::new(InstanceContext::new());
+        crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
+            .await
+            .expect("local disks");
+        let cancel = tokio_util::sync::CancellationToken::new();
+        let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
+            .await
+            .expect("two pools with two sets");
+        cancel.cancel();
+        init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "recover-bounded-fanout";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket");
+        let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
+        let mut keys = BTreeSet::new();
+        for pool in &store.pools {
+            for set in &pool.disk_set {
+                keys.insert(
+                    candidates
+                        .iter()
+                        .find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
+                        .expect("fixture key for set")
+                        .clone(),
+                );
+            }
+        }
+        let keys = keys.into_iter().collect::<Vec<_>>();
+        for pool in &store.pools {
+            for set in &pool.disk_set {
+                assert!(
+                    keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
+                    "fixture must cover every set"
+                );
+            }
+        }
+        for dir in &dirs {
+            for key in &keys {
+                let prefix = dir.path().join(bucket).join(key);
+                std::fs::create_dir_all(&prefix).expect("physical prefix");
+                for child in 0..8 {
+                    std::fs::create_dir(prefix.join(format!("child-{child}"))).expect("child");
+                }
+            }
+        }
+        let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
+        let objects = (0..1000)
+            .map(|index| crate::storage_api_contracts::object::ObjectToDelete {
+                object_name: keys[index % keys.len()].clone(),
+                version_id: Some(Uuid::new_v4()),
+                ..Default::default()
+            })
+            .collect();
+        let (_, errors) = store
+            .delete_objects(
+                bucket,
+                objects,
+                ObjectOptions {
+                    delete_recovery_work_budget: Some(Arc::clone(&budget)),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            errors
+                .iter()
+                .any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
+            "all pools and sets must share the finite allowance"
+        );
+        assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
+        assert!(
+            dirs.iter()
+                .all(|dir| dir.path().join(bucket).join(&keys[0]).join("child-7").is_dir())
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
+        let bucket = "recover-bounded-fast-commit";
+        let (_dirs, store, disks) = fixture(bucket).await;
+        put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
+        // Inline cleanup needs no directory enumeration and may legitimately
+        // finish with zero scan credits. Require real part files here.
+        for disk in &disks {
+            let info = disk
+                .read_version("", bucket, "object", "", &ReadOptions::default())
+                .await
+                .expect("read non-inline source");
+            assert!(!info.inline_data(), "the fixture must require physical part cleanup");
+            let data_dir = info.data_dir.expect("physical data directory");
+            let part = std::path::PathBuf::from(disk.endpoint().to_string())
+                .join(bucket)
+                .join("object")
+                .join(data_dir.to_string())
+                .join("part.1");
+            assert!(part.is_file(), "the cleanup scan must encounter a real part");
+        }
+        store
+            .delete_object(
+                bucket,
+                "object",
+                ObjectOptions {
+                    delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("a cached prepared receipt commits independently of cleanup budget");
+        assert!(
+            store
+                .ctx
+                .delete_recovery_hints()
+                .is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
+                    pool: 0,
+                    set: 0,
+                    bucket: bucket.to_owned(),
+                    object: "object".to_owned(),
+                    incarnation: Some(
+                        store
+                            .bucket_incarnation_id_from_disk(bucket)
+                            .await
+                            .expect("bucket incarnation")
+                    ),
+                })
+        );
+        assert!(
+            !disks[0]
+                .discover_delete_transactions(bucket, Some("object"), None, 128)
+                .await
+                .expect("retained recovery proof")
+                .transactions
+                .is_empty()
+        );
+        store
+            .recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
+            .await
+            .expect("background finishes committed cleanup");
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
+        let bucket = "recover-bounded-candidate";
+        let (dirs, store, _) = fixture(bucket).await;
+        let transaction = Uuid::new_v4();
+        for directory in &dirs {
+            let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
+            std::fs::create_dir_all(&candidate).expect("candidate path");
+            std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
+        }
+        assert!(
+            store
+                .delete_object(bucket, "candidate", ObjectOptions::default())
+                .await
+                .is_err()
+        );
+        for incarnation in [
+            None,
+            Some(
+                store
+                    .bucket_incarnation_id_from_disk(bucket)
+                    .await
+                    .expect("bucket incarnation"),
+            ),
+        ] {
+            assert!(
+                !store
+                    .ctx
+                    .delete_recovery_hints()
+                    .is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
+                        pool: 0,
+                        set: 0,
+                        bucket: bucket.to_owned(),
+                        object: "candidate".to_owned(),
+                        incarnation,
+                    })
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
+        let bucket = "recover-bounded-evidence";
+        let (_dirs, store, disks) = fixture(bucket).await;
+        let payload = vec![71; 4096];
+        put(&store, bucket, &payload, false).await;
+        let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
+        let before = disks[0]
+            .inspect_delete_transaction(bucket, "object", transaction)
+            .await
+            .expect("before refusal");
+        let error = store
+            .delete_object(
+                bucket,
+                "object",
+                ObjectOptions {
+                    delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("no incomplete recovery admission");
+        assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
+        let after = disks[0]
+            .inspect_delete_transaction(bucket, "object", transaction)
+            .await
+            .expect("after refusal");
+        assert_eq!(before.fingerprint, after.fingerprint);
+        store
+            .recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
+            .await
+            .expect("background-compatible recovery");
+        assert_eq!(body(&store, bucket, None).await, payload);
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
+        let bucket = "recover-bounded-commit";
+        let (_dirs, store, disks) = fixture(bucket).await;
+        put(&store, bucket, &vec![72; 4096], false).await;
+        let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
+        let set = &store.pools[0].disk_set[0];
+        let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
+            pool: 0,
+            set: 0,
+            bucket: bucket.to_owned(),
+            object: "object".to_owned(),
+            incarnation: None,
+        };
+        let ticket = set.ctx.delete_recovery_hints().register(key.clone());
+        let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
+        let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        // This hook disables local fast cleanup and runs only after receipt
+        // publication and the shared recovery path's key-wide GC inspection.
+        // Exhausting preparation instead would correctly reject the delete.
+        test_hooks::before_gc_finalize(bucket, "object", {
+            let budget = Arc::clone(&budget);
+            let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
+            Arc::new(move || {
+                let budget = Arc::clone(&budget);
+                let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
+                Box::pin(async move {
+                    assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
+                    exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
+                })
+            })
+        });
+        set.publish_delete_commit(
+            bucket,
+            "object",
+            transaction,
+            &set.disk_inventory().await,
+            set.set_drive_count / 2 + 1,
+            DeleteRecoveryContext {
+                work_budget: Some(Arc::clone(&budget)),
+                recovery_ticket: ticket,
+                ..DeleteRecoveryContext::new(None, &|| true)
+            },
+        )
+        .await
+        .expect("the durable decision still succeeds");
+        assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
+        assert_eq!(budget.remaining(), 0);
+        assert!(set.ctx.delete_recovery_hints().is_pending(&key));
+        let state = disks[0]
+            .inspect_delete_transaction(bucket, "object", transaction)
+            .await
+            .expect("retained decision");
+        assert!(receipt(state.state));
+        store
+            .recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
+            .await
+            .expect("eventual cleanup");
+        assert!(
+            disks[0]
+                .discover_delete_transactions(bucket, Some("object"), None, 128)
+                .await
+                .expect("cleaned proofs")
+                .transactions
+                .is_empty()
+        );
+    }
+
     #[tokio::test]
     async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
         let bucket = "recover-undecided";

crates/ecstore/src/set_disk/ops/locking.rs

@@ -95,13 +95,8 @@ impl SetDisks {
         bucket: &str,
         object: &str,
     ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
-        let resource = ObjectKey::with_version(
-            bucket,
-            object_name_disk_identity(object).as_ref(),
-            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
-        );
         let guard = self
-            .namespace_lock_for_key(resource)
+            .namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
             .await
             .get_write_lock(super::super::get_lock_acquire_timeout())
             .await
@@ -111,6 +106,94 @@ impl SetDisks {
         Ok(Arc::new(guard))
     }
 
+    fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
+        ObjectKey::with_version(
+            bucket,
+            object_name_disk_identity(object).as_ref(),
+            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
+        )
+    }
+
+    /// Batched write-lock acquisition for the whole key set of one bulk
+    /// mutation: one lock round instead of one quorum round-trip per key.
+    ///
+    /// Bulk delete is the only multi-key write-lock holder. The batched round
+    /// is deadlock-free by construction (a single `lock_batch` RPC per lock
+    /// client, and every lock server orders concurrent batches by the same
+    /// global (shard, key) order), so unlike an optimistic probe wave it
+    /// needs no canonical serial fallback. Outcomes are per key and use the
+    /// same error shape as the single-key paths.
+    async fn namespace_write_locks_batch(
+        &self,
+        bucket: &str,
+        resources: &[ObjectKey],
+        timeout: std::time::Duration,
+        mode: &'static str,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        if resources.is_empty() {
+            return Vec::new();
+        }
+        let lock = self.namespace_lock_for_key(resources[0].clone()).await;
+        lock.get_write_locks_batch(resources, timeout)
+            .await
+            .into_iter()
+            .zip(resources)
+            .map(|(result, resource)| {
+                result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
+            })
+            .collect()
+    }
+
+    /// Batched raw namespace write locks for a bulk delete's key set. The
+    /// resource keys use the same disk-identity folding as `new_ns_lock`, so
+    /// aliases (`a\b` vs `a/b`) still share one lock.
+    pub(crate) async fn acquire_object_write_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| ObjectKey {
+                bucket: Arc::from(bucket),
+                object: Arc::from(object_name_disk_identity(object).as_ref()),
+                version: None,
+            })
+            .collect();
+        self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
+    }
+
+    /// Batched delete-transaction locks for a bulk delete's key set; see
+    /// [`Self::acquire_delete_transaction_lock`] for the resource shape.
+    pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| self.delete_transaction_resource(bucket, object))
+            .collect();
+        let mut guards = Vec::with_capacity(resources.len());
+        for (object, result) in objects.iter().zip(
+            self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
+                .await,
+        ) {
+            match result {
+                Ok(guard) => {
+                    #[cfg(test)]
+                    super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
+                    let _ = object;
+                    guards.push(Ok(Arc::new(guard)));
+                }
+                Err(error) => guards.push(Err(error)),
+            }
+        }
+        guards
+    }
+
     #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
     pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
         match err {

crates/ecstore/src/set_disk/ops/object.rs

@@ -3914,6 +3914,7 @@ impl SetDisks {
                 bucket,
                 object,
                 DeleteRecoveryContext {
+                    work_budget: opts.delete_recovery_work_budget.clone(),
                     explicit_delete: true,
                     transaction_guard: transaction_guard.cloned(),
                     namespace_owner: Some(Arc::new((
@@ -4534,7 +4535,7 @@ impl SetDisks {
 
             let stream = mem::replace(
                 &mut data.stream,
-                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
+                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
             );
 
             let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
@@ -4789,6 +4790,27 @@ impl SetDisks {
                 .await?;
             }
 
+            // Lock order: lifecycle, metadata transaction, then object namespace.
+            // No body reads occur below this point. The snapshot must describe the
+            // configuration at commit, not the pre-body advisory object lookup.
+            let mut commit_opts;
+            let opts = if opts.defer_put_object_lock_snapshot {
+                commit_opts = opts.clone();
+                let expected = source_bucket_incarnation_id
+                    .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
+                let store = metadata_sys::object_store_in(&self.ctx).await?;
+                let snapshot = store.object_lock_config_snapshot_for_single_put(bucket).await?;
+                if !snapshot.is_valid_for_destructive_put(store.id, bucket, expected) {
+                    return Err(Error::other("PUT bucket generation or configuration changed while staging"));
+                }
+                snapshot.add_lock_fences(&mut commit_opts);
+                commit_opts.object_lock_config_snapshot = Some(snapshot);
+                commit_opts.defer_put_object_lock_snapshot = false;
+                &commit_opts
+            } else {
+                opts
+            };
+
             if let Some(store) = opts.decommission_capacity_admission.as_ref() {
                 #[cfg(test)]
                 {
@@ -4958,7 +4980,20 @@ impl SetDisks {
                 );
             }
 
-            if let Some(version_id) = opts.version_id.as_deref()
+            // A write without a version id to a key that is not versioning-Enabled
+            // replaces the key's null version in place, so the WORM gate must judge
+            // that version too. The request-layer pre-check reads the latest
+            // version, which is not the one replaced when a UUID version or a
+            // delete marker sits above a retained null version.
+            let worm_gate_version_id = opts.version_id.clone().or_else(|| {
+                (!opts.versioned
+                    && opts
+                        .object_lock_config_snapshot
+                        .as_ref()
+                        .is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
+                .then(|| Uuid::nil().to_string())
+            });
+            if let Some(version_id) = worm_gate_version_id.as_deref()
                 && !is_meta_bucketname(bucket)
             {
                 let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
@@ -4978,9 +5013,10 @@ impl SetDisks {
                     .await;
                 match current {
                     Ok(existing) => {
-                        let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
-                            Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
-                        })?;
+                        let object_lock_config = opts
+                            .object_lock_config_snapshot
+                            .as_deref()
+                            .ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
                         // The WORM gate protects the locked version from local
                         // overwrites; an authorized replication write passes it
                         // only when the LWW merge below will judge every
@@ -5011,6 +5047,8 @@ impl SetDisks {
                             }
                         }
                     }
+                    // A null delete marker has no protected payload to replace.
+                    Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
                 }
@@ -5266,6 +5304,7 @@ impl SetDisks {
             let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
             let commit_publication_guard = publication_commit_guard.take();
             let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
+            let commit_object_lock_config_snapshot = opts.object_lock_config_snapshot.clone();
             let commit_decommission_capacity_guard = decommission_capacity_guard.take();
             let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
             // A scanner publication scope owns the movement permit until the
@@ -5328,6 +5367,7 @@ impl SetDisks {
                 let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
                 let mut _publication_guard = commit_publication_guard;
                 let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
+                let mut _object_lock_config_snapshot = commit_object_lock_config_snapshot;
                 let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
                 let mut quota_reservation = quota_reservation;
                 let rename_stage_start = Instant::now();
@@ -5661,6 +5701,7 @@ impl SetDisks {
                         let object_lock_guard = _object_lock_guard.take();
                         let publication_guard = _publication_guard.take();
                         let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
+                        let object_lock_config_snapshot = _object_lock_config_snapshot.take();
                         let decommission_object_lock_guard = _decommission_object_lock_guard.take();
                         let decommission_capacity_guard = _decommission_capacity_guard.take();
                         let cleanup_bucket = commit_bucket.clone();
@@ -5683,6 +5724,7 @@ impl SetDisks {
                                 write_context,
                                 publication_guard,
                                 bucket_lifecycle_guard,
+                                object_lock_config_snapshot,
                                 decommission_object_lock_guard,
                                 decommission_capacity_guard,
                             ),
@@ -5704,6 +5746,7 @@ impl SetDisks {
                                 write_context,
                                 publication_guard,
                                 bucket_lifecycle_guard,
+                                object_lock_config_snapshot,
                                 decommission_object_lock_guard,
                                 decommission_capacity_guard,
                             ),
@@ -5717,6 +5760,7 @@ impl SetDisks {
                                 drop(write_context);
                                 drop(publication_guard);
                                 drop(bucket_lifecycle_guard);
+                                drop(object_lock_config_snapshot);
                                 cleanup_runtime::run(cleanup_reservation, async move {
                                     let zero_target_cleanup = targets.is_empty();
                                     rustfs_io_metrics::record_put_object_stage_duration(
@@ -6129,7 +6173,11 @@ impl SetDisks {
                     );
                 }
 
-                if !tail_owns_tmp_cleanup {
+                let all_local_disks_cleaned_tmp = all_commit_disks_local
+                    && online_disks.len() == commit_set.set_drive_count
+                    && online_disks.iter().all(Option::is_some)
+                    && !issue3031_diag_enabled();
+                if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
                     let cleanup_set = commit_set.clone();
                     let cleanup_tmp_dir = commit_tmp_dir.clone();
                     cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
@@ -9127,6 +9175,9 @@ impl SetDisks {
                 &disks,
                 write_quorum,
                 DeleteRecoveryContext {
+                    work_budget: commit_context
+                        .options
+                        .and_then(|options| options.delete_recovery_work_budget.clone()),
                     recovery_ticket,
                     transaction_guard: commit_context.transaction_guard.clone(),
                     namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
@@ -9300,6 +9351,60 @@ impl SetDisks {
 }
 
 impl SetDisks {
+    /// Judge the version a versionless destructive delete removes.
+    ///
+    /// For a key that is not versioning-`Enabled`, a delete without a version
+    /// id removes the `null` version, while the preceding Object Lock check
+    /// judged the key's latest entry. When a UUID version or delete marker
+    /// sits above a retained `null` version, the two differ and the retained
+    /// version must be judged on its own. The caller holds the object's write
+    /// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
+    /// lookup: their latest entry is the version removed.
+    ///
+    /// The lookup future is boxed from a synchronous wrapper so it leaves the
+    /// delete futures' frames: `delete_object` already nests the boxed batch
+    /// path, and inlining a second `get_object_info` overflowed the 4 MiB test
+    /// stack on Linux.
+    fn check_null_version_object_lock_delete<'a>(
+        &'a self,
+        bucket: &'a str,
+        object: &'a str,
+        latest: &'a ObjectInfo,
+        opts: &'a ObjectOptions,
+    ) -> impl Future<Output = Result<()>> + Send + 'a {
+        Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
+    }
+
+    async fn check_null_version_object_lock_delete_inner(
+        &self,
+        bucket: &str,
+        object: &str,
+        latest: &ObjectInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()> {
+        if is_meta_bucketname(bucket)
+            || opts.version_id.is_some()
+            || set_disk_delete_creates_delete_marker(opts)
+            || opts.transition.expire_restored
+            || latest.version_id.is_none_or(|version| version.is_nil())
+        {
+            return Ok(());
+        }
+        let null_opts = ObjectOptions {
+            version_id: Some(Uuid::nil().to_string()),
+            no_lock: true,
+            metadata_cache_safe: false,
+            ..opts.clone()
+        };
+        match self.get_object_info(bucket, object, &null_opts).await {
+            Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
+            // A null delete marker has no protected payload to remove.
+            Err(StorageError::MethodNotAllowed) => Ok(()),
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
+            Err(err) => Err(err),
+        }
+    }
+
     /// Read current metadata without cache admission or read-side repair. The
     /// outer batch owner already holds the sorted namespace write-lock set.
     pub(crate) async fn conditional_delete_current_snapshot(
@@ -9334,6 +9439,76 @@ where
     futures::future::try_join_all(probes).await.map(|_| ()).map_err(Error::from)
 }
 
+/// Per-key source-read inputs for the bulk-delete admission loop. The bounded
+/// metadata pre-fetch and the loop itself must resolve these identically, so
+/// both sides derive them here.
+struct BatchDeleteSourceInputs {
+    replication_object_name: String,
+    explicit_null_version: bool,
+    version_id: Option<Uuid>,
+    versioned: bool,
+    version_suspended: bool,
+    check_opts: ObjectOptions,
+    object_lock_check_required: bool,
+    replicate_delete: bool,
+    needs_source_read: bool,
+}
+
+fn batch_delete_source_inputs(
+    bucket: &str,
+    dobj: &ObjectToDelete,
+    opts: &ObjectOptions,
+    delete_config_snapshot: &DeleteReplicationConfigSnapshot,
+    object_lock_config: &Arc<ObjectLockConfigSnapshot>,
+    guarded: bool,
+) -> BatchDeleteSourceInputs {
+    let replication_object_name = decode_dir_object(&dobj.object_name);
+    let explicit_null_version = is_explicit_null_version(dobj.version_id);
+    let version_id = delete_file_info_version_id(dobj.version_id);
+    let (versioned, version_suspended) = delete_config_snapshot
+        .versioning_config()
+        .delete_state(replication_object_name.as_str());
+    let check_opts = ObjectOptions {
+        version_id: dobj.version_id.map(|version_id| version_id.to_string()),
+        versioned,
+        version_suspended,
+        object_lock_delete: opts.object_lock_delete.clone(),
+        object_lock_config_snapshot: Some(Arc::clone(object_lock_config)),
+        no_lock: true,
+        suppress_read_repair: guarded,
+        ..Default::default()
+    };
+    // A missing bucket Object Lock configuration does not prove that
+    // persisted object metadata is clean: migrated or corrupt data may
+    // still carry an explicit retention or legal hold. Only a real
+    // delete-marker creation can skip the object-level WORM check.
+    let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
+    let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
+    let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
+    let replication_needs_source = replicate_delete
+        && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
+    // Guarded explicit-version deletes are rejected before any read and must
+    // not join the pre-fetch wave.
+    let admission_error = guarded && dobj.version_id.is_some();
+    let needs_source_read = !admission_error
+        && (object_lock_check_required
+            || replication_needs_source
+            || opts.tier_delete_journal_api.is_some()
+            || dobj.expected_identity.is_some()
+            || dobj.version_id.is_some());
+    BatchDeleteSourceInputs {
+        replication_object_name,
+        explicit_null_version,
+        version_id,
+        versioned,
+        version_suspended,
+        check_opts,
+        object_lock_check_required,
+        replicate_delete,
+        needs_source_read,
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
     type Error = Error;
@@ -9566,9 +9741,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         if let Some(etag) = &src_info.etag {
             replacement_metadata.insert("etag".to_owned(), etag.clone());
         }
+        // ObjectInfo externalizes these fields; copy callers supply the effective
+        // values, including empty tags/absent expiry for explicit replacement.
+        if !src_info.user_tags.is_empty() {
+            replacement_metadata
+                .entry(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string())
+                .or_insert_with(|| (*src_info.user_tags).clone());
+        }
+        if let Some(expires) = src_info.expires
+            && !replacement_metadata.contains_key("expires")
+        {
+            replacement_metadata.insert(
+                "expires".to_string(),
+                expires
+                    .format(&time::format_description::well_known::Rfc3339)
+                    .map_err(Error::other)?,
+            );
+        }
         fi.metadata = replacement_metadata.clone();
 
-        let mod_time = OffsetDateTime::now_utc();
+        let mod_time = dst_opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc);
         fi.mod_time = Some(mod_time);
         fi.version_id = version_id;
         fi.versioned = src_opts.versioned || src_opts.version_suspended;
@@ -9691,6 +9883,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         objects: Vec<ObjectToDelete>,
         mut opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        opts.ensure_delete_recovery_work_budget();
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
         let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
@@ -9732,7 +9925,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let unique_lock_count = batch.requests.len();
 
         let mut failed_map = HashMap::new();
-        let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
+        let mut local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
         let mut locked_objects = HashSet::new();
 
         // Instance-scoped, not the ambient facade (backlog#1052) — see
@@ -9749,7 +9942,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             (failed_map, locked_objects, dist_batch_lock_ids) = self.acquire_dist_delete_object_locks_batch(&batch).await;
         } else {
             let batch_result = self.local_lock_manager.acquire_locks_batch(batch).await;
-            _local_batch_guards = batch_result.guards;
+            local_batch_guards = batch_result.guards;
 
             for key in batch_result.successful_locks {
                 locked_objects.insert(key.object.as_ref().to_string());
@@ -9791,8 +9984,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // Every raw batch lock is acquired first. Canonical sorting also
             // gives aliases of one disk key exactly one transaction owner.
             let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
-            for key in keys {
-                match self.acquire_delete_transaction_lock(bucket, &key).await {
+            // Bulk delete is the only multi-key transaction-lock holder; one
+            // batched lock round replaces one quorum round-trip per key and is
+            // deadlock-free by construction (a single `lock_batch` RPC per lock
+            // client, global (shard, key) order at each server). Per-key
+            // failures keep the set layer's per-key error reporting.
+            let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
+            let acquired = self
+                .acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
+                .await;
+            for (key, result) in keys.into_iter().zip(acquired) {
+                match result {
                     Ok(guard) => {
                         opts.add_namespace_lock_guard(&guard);
                         transaction_guards.insert(key, guard);
@@ -9828,6 +10030,54 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 },
             }
         };
+        // Per-key read inputs are resolved exactly once; the pre-fetch and the
+        // admission loop below consume this table so the two cannot disagree
+        // on eligibility or read options.
+        let mut source_inputs: Vec<Option<BatchDeleteSourceInputs>> = Vec::with_capacity(objects.len());
+        for dobj in &objects {
+            let guarded = current_generations.is_some_and(|generations| generations.contains_key(&dobj.object_name));
+            source_inputs.push(Some(batch_delete_source_inputs(
+                bucket,
+                dobj,
+                &opts,
+                &delete_config_snapshot,
+                &object_lock_config,
+                guarded,
+            )));
+        }
+        // Bounded-parallel pre-fetch of the source metadata each admitted key
+        // needs below. These reads are pure quorum lookups under the locks
+        // already held; sequencing one metadata fan-out per key made a
+        // 1000-key batch pay a full round of drive reads at a time.
+        const DELETE_OBJECTS_SOURCE_READ_CONCURRENCY: usize = 16;
+        let mut source_reads: HashMap<usize, (ObjectInfo, FileInfo, Option<Error>)> = HashMap::with_capacity(objects.len());
+        {
+            let mut pending = Vec::new();
+            for (i, source) in source_inputs.iter().enumerate() {
+                if del_errs[i].is_some() {
+                    continue;
+                }
+                let Some(source) = source else {
+                    continue;
+                };
+                if !source.needs_source_read {
+                    continue;
+                }
+                let object_name = objects[i].object_name.clone();
+                let check_opts = &source.check_opts;
+                pending.push(async move {
+                    let (goi, file_info, _write_quorum, gerr) = self
+                        .get_object_info_fileinfo_and_quorum(bucket, &object_name, check_opts)
+                        .await;
+                    (i, (goi, file_info, gerr))
+                });
+            }
+            let reads = futures::stream::iter(pending)
+                .buffer_unordered(DELETE_OBJECTS_SOURCE_READ_CONCURRENCY)
+                .collect::<Vec<_>>()
+                .await;
+            source_reads.extend(reads);
+        }
         let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
         let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
         let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
@@ -9849,41 +10099,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 ));
                 continue;
             }
-            let replication_object_name = decode_dir_object(&dobj.object_name);
-            let explicit_null_version = is_explicit_null_version(dobj.version_id);
-            let version_id = delete_file_info_version_id(dobj.version_id);
-            let (versioned, version_suspended) = delete_config_snapshot
-                .versioning_config()
-                .delete_state(replication_object_name.as_str());
-            let check_opts = ObjectOptions {
-                version_id: dobj.version_id.map(|version_id| version_id.to_string()),
+            let Some(source) = source_inputs[i].take() else {
+                continue;
+            };
+            let BatchDeleteSourceInputs {
+                replication_object_name,
+                explicit_null_version,
+                version_id,
                 versioned,
                 version_suspended,
-                object_lock_delete: opts.object_lock_delete.clone(),
-                object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)),
-                no_lock: true,
-                suppress_read_repair: guarded,
-                ..Default::default()
-            };
-            // A missing bucket Object Lock configuration does not prove that
-            // persisted object metadata is clean: migrated or corrupt data may
-            // still carry an explicit retention or legal hold. Only a real
-            // delete-marker creation can skip the object-level WORM check.
-            let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
-            let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
-            let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
-            let replication_needs_source = replicate_delete
-                && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
-            let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required
-                || replication_needs_source
-                || opts.tier_delete_journal_api.is_some()
-                || dobj.expected_identity.is_some()
-                || dobj.version_id.is_some()
-            {
-                let (goi, file_info, _write_quorum, gerr) = self
-                    .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
-                    .await;
-                (goi, file_info, gerr)
+                check_opts,
+                object_lock_check_required,
+                replicate_delete,
+                needs_source_read,
+            } = source;
+            let (mut goi, mut authoritative_file_info, mut gerr) = if needs_source_read {
+                match source_reads.remove(&i) {
+                    Some(read) => read,
+                    None => {
+                        let (goi, file_info, _write_quorum, gerr) = self
+                            .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
+                            .await;
+                        (goi, file_info, gerr)
+                    }
+                }
             } else {
                 (ObjectInfo::default(), FileInfo::default(), None)
             };
@@ -9900,11 +10139,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 && !guarded
                 && goi.name.is_empty()
             {
+                // Recovery can restore or retire versions of this key
+                // (rolling back an interrupted prepared delete brings a
+                // retained version back). A pre-fetched read for a later
+                // duplicate of the same name would still report the stale
+                // miss and skip its object-lock check — bypassing retention —
+                // so evict those entries and let the loop re-read them live.
+                for (other_index, other) in objects.iter().enumerate() {
+                    if other_index != i && other.object_name == dobj.object_name {
+                        source_reads.remove(&other_index);
+                    }
+                }
                 let recovery = self
                     .recover_delete_key(
                         bucket,
                         &dobj.object_name,
                         DeleteRecoveryContext {
+                            work_budget: opts.delete_recovery_work_budget.clone(),
                             dry_run: !opts.no_lock,
                             transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
                             namespace_owner: opts.delete_mutation_owner.clone(),
@@ -10018,6 +10269,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 del_errs[i] = Some(err);
                 continue;
             }
+            // Not gated on `source_missing`: with a delete marker on top the
+            // latest-version lookup reports the key as missing while the null
+            // version below it may still be retained.
+            if object_lock_check_required
+                && let Err(err) = self
+                    .check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
+                    .await
+            {
+                del_errs[i] = Some(err);
+                continue;
+            }
 
             if opts.tier_delete_journal_api.is_some()
                 && !source_missing
@@ -10473,6 +10735,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // response can hide a persisted decision and must never roll back.
             let groups = groups.collect::<Vec<_>>();
             published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
+            // Journal publish fans out per object per drive; keep enough
+            // publications in flight to feed the drives' durable-mutation
+            // lanes (8 workers each) across a 16-drive set.
+            const DELETE_JOURNAL_PUBLISH_CONCURRENCY: usize = 32;
             let publications = futures::stream::iter(groups.into_iter().map(|index: usize| {
                 let group = &vers[index];
                 let recovery_ticket = recovery_tickets[index].clone();
@@ -10498,6 +10764,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                             disks,
                             self.set_drive_count / 2 + 1,
                             DeleteRecoveryContext {
+                                work_budget: opts.delete_recovery_work_budget.clone(),
                                 recovery_ticket,
                                 namespace_owner,
                                 transaction_guard,
@@ -10508,7 +10775,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     (index, result)
                 }
             }))
-            .buffer_unordered(16)
+            .buffer_unordered(DELETE_JOURNAL_PUBLISH_CONCURRENCY)
             .collect::<Vec<_>>()
             .await;
             for (index, result) in publications {
@@ -10648,9 +10915,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         // Deferred recovery takes fresh raw and transaction locks. Discard the
         // completed leases' deadlines while retaining every caller-owned fence.
         opts.namespace_lock_fence = outer_namespace_lock_fence;
-        // Release inner owners before reacquiring any raw key lock.
+        // Release inner owners before reacquiring any raw key lock. The raw
+        // batch guards must go too: deferred recovery takes a fresh write lock
+        // on the same keys below, and a still-held local batch guard would
+        // deadlock it against itself (the distributed path releases its batch
+        // ids in the same region).
         drop(namespace_owner);
         drop(transaction_guards);
+        drop(local_batch_guards);
 
         // TODO(backlog): support partial object deletion for multi-part objects
 
@@ -10676,6 +10948,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     bucket,
                     &object.object_name,
                     DeleteRecoveryContext {
+                        work_budget: opts.delete_recovery_work_budget.clone(),
                         explicit_delete,
                         namespace_owner: Some(Arc::new((
                             self.ctx.begin_namespace_commit(),
@@ -10734,6 +11007,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
     #[tracing::instrument(skip(self, opts))]
     async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
+        opts.ensure_delete_recovery_work_budget();
         let _scope_outcome_guard = opts
             .scanner_publication_commit_scope
             .clone()
@@ -11030,6 +11304,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     bucket,
                     object,
                     DeleteRecoveryContext {
+                        work_budget: opts.delete_recovery_work_budget.clone(),
                         transaction_guard: transaction_guard.clone(),
                         namespace_owner: Some(Arc::new((
                             self.ctx.begin_namespace_commit(),
@@ -11090,6 +11365,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             opts.precondition_check(&goi)?;
             check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
         }
+        self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
+            .await?;
         let delete_marker_purge_candidate = opts
             .version_id
             .as_deref()
@@ -11639,7 +11916,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         // as a second object header set: forwarding SSE intent or wrapped DEKs
         // would request a second encryption pass and disclose local envelope
         // material to the remote provider.
-        transition_meta.retain(|key, _| !rustfs_utils::http::is_replication_stripped_encryption_key(key));
+        transition_meta.retain(|key, _| {
+            !rustfs_utils::http::is_replication_stripped_encryption_key(key)
+                && !rustfs_utils::http::is_snowball_transport_key(key)
+        });
         transition_meta.insert("name".to_string(), object.to_string());
         rustfs_utils::http::metadata_compat::insert_str(
             &mut transition_meta,
@@ -16115,6 +16395,170 @@ mod metadata_mutation_generation_tests {
         assert_retired(&set_disks, &metadata_key).await;
     }
 
+    #[tokio::test]
+    async fn restore_metadata_copy_preserves_tags_expiry_and_version_order() {
+        use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+        use s3s::dto::RestoreRequest;
+        use time::format_description::well_known::Rfc3339;
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "restore-metadata-preservation";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
+        let latest_time = old_time + time::Duration::days(1);
+        let expires = latest_time + time::Duration::days(365);
+        for deleted in [false, true] {
+            let object = if deleted { "deleted" } else { "live" };
+            let tags = "classification=secret&owner=alice";
+            let old = set
+                .put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(b"old protected content".to_vec()),
+                    &ObjectOptions {
+                        versioned: true,
+                        mod_time: Some(old_time),
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        user_defined: HashMap::from([
+                            (AMZ_OBJECT_TAGGING.to_string(), tags.to_string()),
+                            ("x-amz-tagging".to_string(), tags.to_string()),
+                            ("expires".to_string(), expires.format(&Rfc3339).expect("expiry")),
+                        ]),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("write historical version");
+            let latest = if deleted {
+                let marker = FileInfo {
+                    version_id: Some(Uuid::new_v4()),
+                    mod_time: Some(latest_time),
+                    deleted: true,
+                    versioned: true,
+                    ..Default::default()
+                };
+                for disk in &disks {
+                    disk.write_metadata(bucket, bucket, object, marker.clone())
+                        .await
+                        .expect("publish current delete marker");
+                }
+                marker.version_id
+            } else {
+                set.put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(b"latest content".to_vec()),
+                    &ObjectOptions {
+                        versioned: true,
+                        mod_time: Some(latest_time),
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("write latest version")
+                .version_id
+            };
+            let opts = ObjectOptions {
+                version_id: old.version_id.map(|id| id.to_string()),
+                versioned: true,
+                mod_time: Some(old_time),
+                ..Default::default()
+            };
+            let mut source = set.get_object_info(bucket, object, &opts).await.expect("read old metadata");
+            assert_eq!(source.user_tags.as_str(), tags);
+            assert_eq!(source.expires, Some(expires));
+            assert!(!source.user_defined.contains_key(AMZ_OBJECT_TAGGING));
+            assert!(source.user_defined.contains_key("x-amz-tagging"));
+            source.metadata_only = true;
+            Arc::make_mut(&mut source.user_defined)
+                .insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
+            set.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
+                .await
+                .expect("restore metadata update");
+            let restored = set
+                .get_object_info(bucket, object, &opts)
+                .await
+                .expect("read restored metadata");
+            assert_eq!(restored.user_tags.as_str(), tags);
+            assert_eq!(restored.expires, Some(expires));
+            assert_eq!(restored.mod_time, Some(old_time));
+            for disk in &disks {
+                let bytes = disk
+                    .read_all(bucket, &format!("{object}/xl.meta"))
+                    .await
+                    .expect("persisted metadata");
+                let meta = FileMeta::load(&bytes).expect("decode versions");
+                let versions = meta.list_versions(bucket, object, false).expect("list versions");
+                assert_eq!(versions.len(), 2);
+                assert_eq!(versions[0].version_id, latest);
+                assert_eq!(versions[0].mod_time, Some(latest_time));
+                assert_eq!(versions[0].deleted, deleted);
+                assert_eq!(versions[1].version_id, old.version_id);
+                assert_eq!(versions[1].mod_time, Some(old_time));
+            }
+            let restore_opts = put_restore_opts(bucket, object, &RestoreRequest::default(), &restored)
+                .await
+                .expect("background restore options");
+            assert_eq!(restore_opts.user_defined.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some(tags));
+            assert_eq!(restore_opts.user_defined.get("expires"), Some(&expires.format(&Rfc3339).expect("expiry")));
+            assert_eq!(restore_opts.mod_time, Some(old_time));
+        }
+    }
+
+    #[tokio::test]
+    async fn metadata_copy_honors_explicit_tag_and_expiry_replacement() {
+        use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+        use time::format_description::well_known::Rfc3339;
+
+        let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "copy-metadata-replacement";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
+        let expires = old_time + time::Duration::days(365);
+        set.put_object(
+            bucket,
+            "object",
+            &mut PutObjReader::from_vec(b"content".to_vec()),
+            &ObjectOptions {
+                mod_time: Some(old_time),
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("write original");
+        let opts = ObjectOptions::default();
+        let mut source = set.get_object_info(bucket, "object", &opts).await.expect("original metadata");
+        source.metadata_only = true;
+        source.user_tags = Arc::new("old=value".to_string());
+        source.expires = Some(expires - time::Duration::days(1));
+        let metadata = Arc::make_mut(&mut source.user_defined);
+        metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
+        metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
+        set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
+            .await
+            .expect("explicit metadata wins over externalized fields");
+        let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
+        assert_eq!(copied.user_tags.as_str(), "new=value");
+        assert_eq!(copied.expires, Some(expires));
+        assert!(copied.mod_time.is_some_and(|time| time > old_time));
+
+        copied.metadata_only = true;
+        copied.user_tags = Arc::new(String::new());
+        copied.expires = None;
+        set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
+            .await
+            .expect("explicit empty replacement");
+        let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
+        assert!(cleared.user_tags.is_empty());
+        assert!(cleared.expires.is_none());
+    }
+
     #[tokio::test]
     async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -16567,6 +17011,19 @@ mod transition_commit_failure_tests {
             .collect();
         user_defined.insert("x-amz-meta-owner".to_string(), "finance".to_string());
 
+        let snowball_keys = [
+            "snowball-auto-extract",
+            "x-amz-meta-snowball-auto-extract",
+            "x-amz-snowball-auto-extract",
+            "snowball-prefix",
+            "x-amz-meta-rustfs-snowball-prefix",
+            "x-amz-meta-minio-snowball-ignore-dirs",
+            "other-vendor-snowball-ignore-errors",
+        ];
+        for key in snowball_keys {
+            user_defined.insert(key.to_string(), "true".to_string());
+        }
+
         let mut reader = PutObjReader::from_vec(b"stored bytes the tier keeps opaque ".repeat(64));
         set_disks
             .put_object(
@@ -16616,6 +17073,10 @@ mod transition_commit_failure_tests {
             .await
             .expect("the tier must have received the object");
 
+        for key in snowball_keys {
+            assert!(!remote_metadata.contains_key(key), "tier must receive opaque bytes, not {key}");
+            assert_eq!(transitioned.user_defined.get(key).map(String::as_str), Some("true"));
+        }
         for (key, _) in encryption_metadata {
             assert!(
                 !remote_metadata.keys().any(|stored| stored.eq_ignore_ascii_case(key)),
@@ -24988,6 +25449,131 @@ mod delete_objects_lock_gating_tests {
             .expect("plain object should be written");
     }
 
+    /// Regression for the PR9155 review finding: a pre-fetched source-missing
+    /// read of a later duplicate must not skip the object-lock check after an
+    /// earlier entry's recovery rolls back an interrupted prepared delete and
+    /// restores a GOVERNANCE-retained version. Without the eviction the later
+    /// duplicate deletes the restored version, bypassing retention.
+    #[tokio::test]
+    async fn duplicate_entries_recheck_object_lock_after_recovery_restores_retained_version() {
+        let (_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "recovery-retention-dup";
+        let object = "retained.bin";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+
+        // A versioned put, so the batch can address the version explicitly.
+        let mut reader = PutObjReader::from_vec(vec![7u8; 256]);
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("versioned object should be written");
+        let version = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read back the version identity")
+            .version_id
+            .expect("versioned put assigns a version id");
+
+        // GOVERNANCE retention on that version, persisted the way
+        // PutObjectRetention persists it.
+        let retain_until = OffsetDateTime::now_utc() + time::Duration::days(60);
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    version_id: Some(version.to_string()),
+                    eval_metadata: Some(HashMap::from([
+                        (metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
+                        (
+                            metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
+                            retain_until
+                                .format(&time::format_description::well_known::Rfc3339)
+                                .expect("retention date formats"),
+                        ),
+                    ])),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("governance retention should persist");
+
+        // Interrupted authorized delete: the recoverable per-disk delete
+        // applies the version removal and leaves the prepared journal
+        // undecided — exactly a crash between apply and publish.
+        let transaction = Uuid::new_v4();
+        for disk in &disk_stores {
+            let fi = disk
+                .read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("version exists before the interrupted delete");
+            let errors = disk
+                .delete_versions_recoverable(
+                    bucket,
+                    vec![FileInfoVersions {
+                        name: object.to_string(),
+                        versions: vec![fi],
+                        ..Default::default()
+                    }],
+                    DeleteOptions {
+                        recoverable_delete: true,
+                        old_data_dir: Some(transaction),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(errors.iter().all(Option::is_none), "the plant must apply: {errors:?}");
+        }
+
+        // A no-bypass batch naming the retained version twice: the first
+        // entry rolls the prepared delete back (restoring the retained
+        // version) and must be denied; the second must re-read and stay
+        // denied too instead of reusing the stale pre-fetched miss.
+        let victim = ObjectToDelete {
+            object_name: object.to_string(),
+            version_id: Some(version),
+            ..Default::default()
+        };
+        let (_deleted, errors) = set_disks
+            .delete_objects(
+                bucket,
+                vec![victim.clone(), victim],
+                ObjectOptions {
+                    versioned: true,
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            errors.iter().all(|error| error
+                .as_ref()
+                .is_some_and(|error| matches!(error, StorageError::PrefixAccessDenied(_, _)))),
+            "both duplicates must be denied by the restored retention: {errors:?}"
+        );
+
+        // The retained version survives the batch.
+        for disk in &disk_stores {
+            disk.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("the retained version must survive the batch");
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial(shard_integrity_rollout)]
     async fn conditional_delete_snapshot_preserves_multipart_etag() {

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
 
 pub(crate) mod list {
     pub(crate) use rustfs_storage_api::{
-        ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
-        ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
-        WalkVersionsSortOrder,
+        AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
+        ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
+        WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
     };
 }
 
 pub(crate) mod multipart {
     pub(crate) use rustfs_storage_api::{
-        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
+        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
+        MultipartUploadInspection, MultipartUploadResult, PartInfo,
     };
 }
 

crates/ecstore/src/store/bucket.rs

@@ -14,7 +14,7 @@
 
 use super::*;
 use crate::bucket::{
-    metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
+    metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
     utils::is_meta_bucketname,
 };
 use crate::error::is_err_bucket_not_found;
@@ -145,11 +145,41 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
     }
 }
 
-fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
-    [
-        table_bucket_catalog_metadata_prefix(bucket),
-        format!("{BUCKET_META_PREFIX}/{bucket}"),
-    ]
+fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
+    use crate::bucket::metadata::*;
+
+    let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
+    let root = format!("{BUCKET_META_PREFIX}/{bucket}");
+    if bucket != "ahm-replacement" {
+        prefixes.push(root);
+        return prefixes;
+    }
+
+    // This valid user bucket shares its metadata directory with raw drive
+    // replacement intents, seals and recovery proofs. Delete only bucket-owned
+    // paths, including legacy standalone configuration, never the shared root.
+    // S3 objects live in the separate user volume; configuration APIs write the
+    // aggregate metadata file, so clients cannot populate arbitrary siblings.
+    prefixes.extend(
+        [
+            BUCKET_METADATA_FILE,
+            BUCKET_INCARNATION_FILE,
+            BUCKET_POLICY_CONFIG,
+            BUCKET_QUOTA_CONFIG_FILE,
+            BUCKET_TARGETS_FILE,
+            BUCKET_ACL_CONFIG,
+            BUCKET_TABLE_CONFIG,
+            BUCKET_DURABILITY_CONFIG,
+            BUCKET_ON_DEMAND_MIGRATION_CONFIG,
+            ".replication",
+            "integrity-jobs",
+            crate::data_usage::DATA_USAGE_CACHE_NAME,
+        ]
+        .into_iter()
+        .chain(XML_BUCKET_CONFIG_FILES)
+        .map(|file| format!("{root}/{file}")),
+    );
+    prefixes
 }
 
 fn bucket_deleted_marker_prefix(bucket: &str) -> String {
@@ -160,41 +190,48 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
     format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
 }
 
-pub(crate) async fn await_bucket_namespace_operation<T, F>(
-    guard: Option<&rustfs_lock::NamespaceLockGuard>,
-    bucket: &str,
+// Box the child before constructing either lease wrapper. Keeping F inline in
+// nested select futures multiplies poll-frame temporaries along DELETE -> quota
+// cleanup -> metadata GET. This boundary must not inline back into that chain.
+#[inline(never)]
+pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
+    guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    bucket: &'a str,
     operation: &'static str,
     future: F,
-) -> Result<T>
+) -> impl Future<Output = Result<T>> + 'a
 where
-    F: Future<Output = Result<T>>,
+    F: Future<Output = Result<T>> + 'a,
 {
-    let Some(guard) = guard else {
-        return future.await;
-    };
-    if guard.is_lock_lost() {
-        return Err(StorageError::other(format!(
-            "bucket namespace lock was lost before {operation}: {bucket}"
-        )));
-    }
-    tokio::select! {
-        biased;
-        _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
-            "bucket namespace lock was lost during {operation}: {bucket}"
-        ))),
-        result = future => result,
+    let future = Box::pin(future);
+    async move {
+        let Some(guard) = guard else {
+            return future.await;
+        };
+        if guard.is_lock_lost() {
+            return Err(StorageError::other(format!(
+                "bucket namespace lock was lost before {operation}: {bucket}"
+            )));
+        }
+        tokio::select! {
+            biased;
+            _ = guard.lock_lost_notified() => Err(StorageError::other(format!(
+                "bucket namespace lock was lost during {operation}: {bucket}"
+            ))),
+            result = future => result,
+        }
     }
 }
 
-async fn await_bucket_lifecycle_operation<T, F>(
-    lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
-    namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
-    bucket: &str,
+fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
+    lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    bucket: &'a str,
     operation: &'static str,
     future: F,
-) -> Result<T>
+) -> impl Future<Output = Result<T>> + 'a
 where
-    F: Future<Output = Result<T>>,
+    F: Future<Output = Result<T>> + 'a,
 {
     await_bucket_namespace_operation(
         lifecycle_guard,
@@ -202,7 +239,6 @@ where
         operation,
         await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
     )
-    .await
 }
 
 fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
@@ -478,10 +514,18 @@ impl ECStore {
         let mut tags = HashMap::with_capacity(tagging.tag_set.len());
         for tag in &tagging.tag_set {
             let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
-                return Err(Error::other("stored bucket tag is missing a key or value"));
+                return Err(crate::bucket::metadata::unreadable_config_error(
+                    bucket,
+                    BUCKET_TAGGING_CONFIG,
+                    metadata.tagging_config_xml.len(),
+                ));
             };
             if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
-                return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
+                return Err(crate::bucket::metadata::unreadable_config_error(
+                    bucket,
+                    BUCKET_TAGGING_CONFIG,
+                    metadata.tagging_config_xml.len(),
+                ));
             }
         }
         Ok(tags)
@@ -523,7 +567,22 @@ impl ECStore {
     where
         F: FnOnce() -> Result<()> + Send,
     {
-        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
+        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
+    }
+
+    /// Validate an external fence and the bucket incarnation in one metadata transaction.
+    pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
+        &self,
+        bucket: &str,
+        config_file: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: Uuid,
+        validate: F,
+    ) -> Result<OffsetDateTime>
+    where
+        F: FnOnce() -> Result<()> + Send,
+    {
+        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
     }
 
     pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
@@ -1658,7 +1717,17 @@ impl ECStore {
                 ns_guard.as_ref(),
                 bucket,
                 "bucket retirement publication",
-                crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
+                crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
+            )
+            .await?;
+            // The lifecycle fence excludes live reservation owners. A crash-left
+            // V1 ledger must not block quota admission after same-name recreation.
+            await_bucket_lifecycle_operation(
+                bucket_lifecycle_guard.as_ref(),
+                ns_guard.as_ref(),
+                bucket,
+                "retired bucket quota ledger cleanup",
+                crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
             )
             .await?;
         }
@@ -1809,6 +1878,20 @@ mod tests {
         assert!(!xlmeta.diagnostic_truncated);
     }
 
+    #[tokio::test]
+    async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
+        let payload = std::hint::black_box([7u8; 128 * 1024]);
+        let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
+            tokio::task::yield_now().await;
+            Ok(std::hint::black_box(payload)[0])
+        });
+        assert!(
+            std::mem::size_of_val(&operation) < 1024,
+            "nested lease wrappers must not retain the concrete storage future inline"
+        );
+        assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
+    }
+
     #[tokio::test(start_paused = true)]
     async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
         let ttl = Duration::from_millis(20);
@@ -2225,7 +2308,13 @@ mod tests {
             let result = store.get_bucket_tags_for_policy(bucket).await;
             match expected {
                 Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
-                None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
+                None => assert!(
+                    crate::bucket::metadata::unreadable_config_refusal(
+                        &result.expect_err("corrupt tags must not become an untagged bucket")
+                    )
+                    .is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
+                    "corrupt tags must carry a typed tagging refusal: {xml:?}"
+                ),
             }
         }
     }
@@ -2427,6 +2516,187 @@ mod tests {
         assert!(prefixes.contains(&"buckets/analytics".to_string()));
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
+        use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
+
+        let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "ahm-replacement";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create legal bucket");
+        let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
+        for (config, bytes) in [
+            (BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
+            (
+                BUCKET_TAGGING_CONFIG,
+                b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
+            ),
+            (
+                BUCKET_VERSIONING_CONFIG,
+                b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
+            ),
+        ] {
+            metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
+                .await
+                .expect("persist old configuration");
+        }
+        let ledger_path = format!("config/quota-ledger/{bucket}.json");
+        // V1 precharges outstanding growth: four live bytes plus the
+        // crash-left reservation's 100 bytes must already be accounted.
+        let ledger = serde_json::to_vec(&serde_json::json!({
+            "version": 1, "bucket_incarnation": old_incarnation,
+            "quota_revision_unix_nanos": 0, "accounted_usage": 104,
+            "reservations": {Uuid::new_v4().to_string(): {
+                "object": "crashed-put", "old_size": 0, "new_size": 100,
+                "created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
+            }}, "reconcile_required": true
+        }))
+        .expect("encode crash-left ledger");
+
+        // The storage API requires the same legacy-quota admission supplied
+        // by the S3 handler, even for this small fixture object.
+        let mut live_options = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+        assert!(live_options.set_quota_admission(0, 1024));
+        // Failed deletion must leave both the live incarnation and its ledger alone.
+        store
+            .put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
+            .await
+            .expect("write live object");
+        crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
+            .await
+            .expect("persist old ledger");
+        assert!(matches!(
+            store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
+            Err(StorageError::BucketNotEmpty(_))
+        ));
+        assert_eq!(
+            crate::config::com::read_config(store.clone(), &ledger_path)
+                .await
+                .expect("live ledger remains"),
+            ledger
+        );
+        // Remove every version; the configured bucket is otherwise still empty.
+        let versions = store
+            .clone()
+            .list_object_versions(bucket, "", None, None, None, 100)
+            .await
+            .expect("list live versions");
+        for object in versions.objects {
+            store
+                .delete_object(
+                    bucket,
+                    &object.name,
+                    ObjectOptions {
+                        version_id: object.version_id.map(|id| id.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("remove live version");
+        }
+
+        let root = format!("{BUCKET_META_PREFIX}/{bucket}");
+        let task = Uuid::new_v4();
+        let recovery_names = [
+            "ahm_replacement_intent.json",
+            "ahm_replacement_intent_seal",
+            "ahm_replacement_completion_proof.json",
+        ]
+        .map(|suffix| format!("{task}_{suffix}"));
+        let disk_paths = store.pools[0].disk_set[0]
+            .disks
+            .read()
+            .await
+            .iter()
+            .flatten()
+            .map(|disk| PathBuf::from(disk.endpoint().to_string()))
+            .collect::<Vec<_>>();
+        for disk in &disk_paths {
+            let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
+            for name in &recovery_names {
+                tokio::fs::write(directory.join(name), b"durable heal record")
+                    .await
+                    .expect("seed raw recovery record");
+            }
+            // Legacy standalone settings and bucket-owned subtrees must not survive.
+            for name in [
+                "quota.json",
+                "versioning.xml",
+                "policy.json",
+                "bucket-targets.json",
+                "integrity-jobs/job.json",
+                ".replication/resync.bin",
+                ".usage-cache.bin",
+            ] {
+                let path = directory.join(name);
+                tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
+                    .await
+                    .expect("legacy parent");
+                tokio::fs::write(path, b"old bucket configuration")
+                    .await
+                    .expect("seed legacy configuration");
+            }
+        }
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete legal same-name bucket");
+        assert!(matches!(
+            crate::config::com::read_config(store.clone(), &ledger_path).await,
+            Err(StorageError::ConfigNotFound)
+        ));
+        for disk in &disk_paths {
+            let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
+            for name in &recovery_names {
+                assert_eq!(
+                    tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
+                    b"durable heal record"
+                );
+            }
+            for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
+                .into_iter()
+                .filter(|p| p.starts_with(&root))
+            {
+                assert!(
+                    !disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
+                    "bucket-owned metadata must be removed"
+                );
+            }
+        }
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate legal bucket");
+        let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
+        assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
+        assert!(fresh.quota_config_json.is_empty());
+        assert!(fresh.tagging_config_xml.is_empty());
+        assert!(fresh.versioning_config_xml.is_empty());
+        assert!(fresh.policy_config_json.is_empty());
+        // The strict purge/recovery cleanup must share the same ownership boundary.
+        store
+            .cleanup_deleted_bucket_metadata_strict(bucket, true)
+            .await
+            .expect("strict metadata cleanup");
+        for disk in &disk_paths {
+            for name in &recovery_names {
+                assert_eq!(
+                    tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
+                        .await
+                        .expect("strict cleanup preserves recovery"),
+                    b"durable heal record"
+                );
+            }
+        }
+    }
+
     #[test]
     fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
         assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");

crates/ecstore/src/store/heal.rs

@@ -368,7 +368,7 @@ impl ECStore {
             Some(expected) => expected,
             None => self.bucket_incarnation_id_from_disk(bucket).await?,
         };
-        let object = object.map(str::to_owned);
+        let object = object.map(encode_dir_object);
         let prefix = prefix.map(str::to_owned);
         self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
             let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
@@ -430,7 +430,7 @@ impl ECStore {
                     HealItemType::Bucket
                 }
                 .to_string(),
-                object: object.or(prefix).unwrap_or_default(),
+                object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
                 bucket,
                 detail: format!(
                     "delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
@@ -656,12 +656,48 @@ impl ECStore {
         Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
     }
 
+    /// Read the published bucket identity without fabricating or migrating metadata.
+    /// Absence is distinct from quorum, corruption and transaction-fence failures.
+    pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
+        let transaction =
+            crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
+        let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
+        if transaction.is_lock_lost() {
+            return Err(Error::PreconditionFailed);
+        }
+        Ok(incarnation)
+    }
+
     /// Restore the bucket configuration and incarnation before retiring a replacement marker.
     pub async fn heal_replacement_bucket_metadata(
         self: &Arc<Self>,
         bucket: &str,
         opts: &HealOpts,
         targets: &[String],
+    ) -> Result<()> {
+        // Reject an invalid request before touching the drives so callers keep
+        // the PreconditionFailed contract for bad buckets and options.
+        if opts.pool.is_none()
+            || opts.set.is_none()
+            || targets.is_empty()
+            || opts.dry_run
+            || opts.no_lock
+            || check_valid_bucket_name_strict(bucket).is_err()
+        {
+            return Err(Error::PreconditionFailed);
+        }
+        let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
+        self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
+            .await
+    }
+
+    /// Keep a replacement pass bound to its admitted bucket through metadata commit.
+    pub async fn heal_replacement_bucket_metadata_at_incarnation(
+        self: &Arc<Self>,
+        bucket: &str,
+        expected: uuid::Uuid,
+        opts: &HealOpts,
+        targets: &[String],
     ) -> Result<()> {
         use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
         use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
@@ -698,9 +734,8 @@ impl ECStore {
 
         // Read the persisted identity without lazy migration: a replacement
         // must never turn lost configuration into newly fabricated defaults.
-        let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
         let targets = targets.to_vec();
-        self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
+        self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
             // Match config writers: lifecycle, metadata transaction, object locks.
             // Keep the transaction stable through target readback, including when
             // the caller is cancelled and the storage owner finishes its write.
@@ -1022,6 +1057,7 @@ impl ECStore {
         set_index: usize,
         targets: &[String],
     ) -> Result<bool> {
+        let object = encode_dir_object(object);
         let pool = self
             .pools
             .get(pool_index)
@@ -1034,7 +1070,7 @@ impl ECStore {
             )
         })?;
 
-        set.replacement_targets_have_version(bucket, object, version_id, targets)
+        set.replacement_targets_have_version(bucket, &object, version_id, targets)
             .await
             .map_err(Into::into)
     }
@@ -2859,6 +2895,105 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
+        let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
+        let bucket = "directory-replacement-readback";
+        let object = "folder/";
+        let encoded = encode_dir_object(object);
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut PutObjReader::from_vec(Vec::new()),
+                &ObjectOptions {
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write directory object through the public store entry");
+        let set = &store.pools[0].disk_set[0];
+        let targets = vec![set.set_endpoints[0].to_string()];
+        assert!(
+            store
+                .replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
+                .await
+                .expect("readback")
+        );
+        let disk = set.disks.read().await[0].clone().expect("target disk");
+        disk.delete(
+            bucket,
+            &encoded,
+            DeleteOptions {
+                recursive: true,
+                immediate: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("remove target directory shard");
+        assert!(
+            !store
+                .replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
+                .await
+                .expect("missing target")
+        );
+        let opts = HealOpts {
+            pool: Some(0),
+            set: Some(0),
+            ..Default::default()
+        };
+        let (result, error) = store
+            .heal_object(bucket, object, "", &opts)
+            .await
+            .expect("heal directory object");
+        assert!(error.is_none(), "heal failed: {error:?}");
+        assert_eq!(result.object, object);
+        assert!(
+            store
+                .replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
+                .await
+                .expect("healed readback")
+        );
+        assert!(
+            store
+                .replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
+                .await
+                .expect("encoded readback")
+        );
+
+        // Exact-object recovery must inspect the same physical namespace and
+        // reject corrupt proof rather than silently report no transactions.
+        let metadata = disk
+            .read_all(bucket, &format!("{encoded}/xl.meta"))
+            .await
+            .expect("healed metadata");
+        disk.write_all(
+            bucket,
+            &format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
+            b"invalid proof".to_vec().into(),
+        )
+        .await
+        .expect("seed corrupt recovery proof");
+        store
+            .recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
+            .await
+            .expect_err("directory recovery must discover and reject the corrupt proof");
+        assert_eq!(
+            disk.read_all(bucket, &format!("{encoded}/xl.meta"))
+                .await
+                .expect("unchanged metadata"),
+            metadata
+        );
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
         let (root, store, shutdown) = multi_pool_heal_store().await;
@@ -3692,6 +3827,177 @@ mod tests {
         );
     }
 
+    // The RPC lock service uses the raw ObjectKey and one backend per node,
+    // even when the requesting sets carry different pool namespace strings.
+    async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
+        use rustfs_lock::client::{LockClient, local::LocalClient};
+
+        let clients = || {
+            (0..2)
+                .map(|_| {
+                    Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
+                })
+                .collect::<Vec<_>>()
+        };
+        let first = clients();
+        let second = if shared { first.clone() } else { clients() };
+        let mut pools = source.pools.clone();
+        for (pool, lockers) in pools.iter_mut().zip([first, second]) {
+            for set in &mut Arc::make_mut(pool).disk_set {
+                Arc::make_mut(set).lockers = lockers.clone();
+            }
+        }
+        let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
+        let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
+        Arc::new(ECStore {
+            id: Uuid::new_v4(),
+            disk_map: Default::default(),
+            pools,
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
+            pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
+            rebalance_meta: tokio::sync::RwLock::new(None),
+            decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(state),
+            ctx: Arc::clone(&source.ctx),
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
+        let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let first_lock = first
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("first pool lock");
+        let second_lock = second
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("second pool lock");
+        let guard = first_lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("first writer");
+        assert!(
+            second_lock
+                .get_write_lock(std::time::Duration::from_millis(150))
+                .await
+                .is_err(),
+            "different pool namespaces still conflict on the same backend and object key"
+        );
+        drop(guard);
+        second_lock
+            .get_write_lock(std::time::Duration::from_secs(3))
+            .await
+            .expect("second writer should acquire after the first releases");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    dry_run: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
+        .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let missing = remove_pool_meta_shard(&store, 1).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(10),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("cross-pool repair must not reenter the shared writer")
+        .expect("pool metadata repair should return a result");
+        assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+        assert!(
+            missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "the selected missing shard must be restored"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
+        let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let lock = target
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("target lock");
+        let guard = lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("hold target lock");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    POOL_META_NAME,
+                    "",
+                    &HealOpts {
+                        pool: Some(1),
+                        dry_run: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            barrier.wait_until_external_heal_target_lock_attempted(),
+        )
+        .await
+        .expect("inspection must attempt the independent target lock");
+        assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
+        drop(guard);
+        let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
+            .await
+            .expect("inspection should progress after target lock release")
+            .expect("heal task should not panic")
+            .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {

crates/ecstore/src/store/init.rs

@@ -5155,6 +5155,11 @@ mod tests {
         let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
         let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
         assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
+        if !sharded {
+            crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
+                .await
+                .expect("bootstrap quota with one disk offline");
+        }
 
         let payload = vec![0x51; 4096];
         let mut reader = PutObjReader::from_vec(payload.clone());
@@ -21007,6 +21012,299 @@ mod tests {
         );
     }
 
+    async fn seed_retained_null_version(
+        store: &Arc<crate::store::ECStore>,
+        bucket: &str,
+        object: &str,
+        protection: &str,
+        latest_is_marker: bool,
+    ) {
+        let metadata = if protection == "legal-hold" {
+            HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
+        } else {
+            HashMap::from([
+                ("x-amz-object-lock-mode".to_string(), protection.to_string()),
+                ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
+            ])
+        };
+        let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut original,
+                &ObjectOptions {
+                    user_defined: metadata,
+                    mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed retained null version");
+        let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut latest,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed newer unprotected UUID version");
+        if latest_is_marker {
+            store
+                .delete_object(
+                    bucket,
+                    object,
+                    ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create a UUID delete marker above the retained null version");
+        }
+    }
+
+    async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
+        let mut reader = store
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(Uuid::nil().to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("retained null version must remain readable");
+        let mut body = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut body)
+            .await
+            .expect("read retained null version");
+        assert_eq!(body, b"protected null version");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_delete_checks_actual_target() {
+        use crate::bucket::replication::DeleteReplicationConfigSnapshot;
+        use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
+
+        let temp = tempfile::tempdir().expect("create null-version delete store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        let configs = [
+            VersioningConfiguration::default(),
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+                ..Default::default()
+            },
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                excluded_prefixes: Some(vec![ExcludedPrefix {
+                    prefix: Some("archive/".to_string()),
+                }]),
+                ..Default::default()
+            },
+        ];
+        for (case, config) in configs.into_iter().enumerate() {
+            let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
+            for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+                for latest_is_marker in [false, true] {
+                    let object = format!("archive/{case}-{protection}-{latest_is_marker}");
+                    seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                    let (_, errors) = store
+                        .delete_objects(
+                            &bucket,
+                            vec![ObjectToDelete {
+                                object_name: object.clone(),
+                                ..Default::default()
+                            }],
+                            ObjectOptions {
+                                delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                ..Default::default()
+                            },
+                        )
+                        .await;
+                    assert!(
+                        matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
+                        "batch delete must judge the retained null version: {object}: {errors:?}"
+                    );
+                    let error = store
+                        .delete_object(
+                            &bucket,
+                            &object,
+                            ObjectOptions {
+                                version_suspended: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect_err("suspended single delete must also judge the retained null version");
+                    assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                    assert_retained_null_version_body(&store, &bucket, &object).await;
+                    if protection == "GOVERNANCE" {
+                        let (_, errors) = store
+                            .delete_objects(
+                                &bucket,
+                                vec![ObjectToDelete {
+                                    object_name: object.clone(),
+                                    ..Default::default()
+                                }],
+                                ObjectOptions {
+                                    delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                    object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
+                                        bypass_governance: true,
+                                    }),
+                                    ..Default::default()
+                                },
+                            )
+                            .await;
+                        assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
+                        let removed = store
+                            .get_object_info(
+                                &bucket,
+                                &object,
+                                &ObjectOptions {
+                                    version_id: Some(Uuid::nil().to_string()),
+                                    ..Default::default()
+                                },
+                            )
+                            .await;
+                        match removed {
+                            Ok(marker) => {
+                                assert!(marker.delete_marker, "only a null delete marker may replace the payload");
+                                assert_eq!(marker.size, 0);
+                                assert!(marker.data_dir.is_none());
+                            }
+                            Err(error) => assert!(
+                                matches!(
+                                    error,
+                                    StorageError::MethodNotAllowed
+                                        | StorageError::ObjectNotFound(_, _)
+                                        | StorageError::VersionNotFound(_, _, _)
+                                ),
+                                "unexpected bypass result: {error:?}"
+                            ),
+                        }
+                        assert!(
+                            store
+                                .get_object_reader(
+                                    &bucket,
+                                    &object,
+                                    None,
+                                    HeaderMap::new(),
+                                    &ObjectOptions {
+                                        version_id: Some(Uuid::nil().to_string()),
+                                        ..Default::default()
+                                    }
+                                )
+                                .await
+                                .is_err(),
+                            "authorized deletion must make the retained null payload unreadable"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
+        let temp = tempfile::tempdir().expect("create null-version overwrite store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+            for latest_is_marker in [false, true] {
+                let object = format!("{protection}-{latest_is_marker}");
+                seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
+                let error = store
+                    .put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
+                    .await
+                    .expect_err("PUT must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+
+                let upload = store
+                    .clone()
+                    .new_multipart_upload(&bucket, &object, &ObjectOptions::default())
+                    .await
+                    .expect("initiate multipart upload");
+                let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
+                let part = store
+                    .clone()
+                    .put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
+                    .await
+                    .expect("stage multipart replacement");
+                let error = store
+                    .clone()
+                    .complete_multipart_upload(
+                        &bucket,
+                        &object,
+                        &upload.upload_id,
+                        vec![crate::storage_api_contracts::multipart::CompletePart {
+                            part_num: part.part_num,
+                            etag: part.etag,
+                            ..Default::default()
+                        }],
+                        &ObjectOptions::default(),
+                    )
+                    .await
+                    .expect_err("multipart completion must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+
+                let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
+                store
+                    .put_object(
+                        &bucket,
+                        &object,
+                        &mut next_version,
+                        &ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("a new UUID version must remain allowed");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+            }
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {

crates/ecstore/src/store/mod.rs

@@ -423,8 +423,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
     !is_meta_bucketname(&oi.bucket)
 }
 
-const MAX_UPLOADS_LIST: usize = 10000;
-
 mod bucket;
 mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;

crates/ecstore/src/store/multipart.rs

@@ -19,10 +19,28 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
 use crate::storage_api_contracts::object::ObjectOperations as _;
 use futures::{StreamExt, stream};
+use rustfs_utils::path::encode_dir_object_ref;
 use std::collections::HashSet;
 
 const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
 
+#[async_trait::async_trait]
+impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
+    type Error = Error;
+
+    async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
+        let (_, lifecycle_guard) = self
+            .guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
+            .await?;
+        for pool in &self.pools {
+            for set in &pool.disk_set {
+                set.ensure_no_multipart_uploads(bucket).await?;
+            }
+        }
+        ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
+    }
+}
+
 #[cfg(test)]
 static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
     std::sync::OnceLock::new();
@@ -255,6 +273,28 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
     }
 }
 
+async fn pool_has_multipart_upload_for_key(
+    pool: &crate::core::sets::Sets,
+    bucket: &str,
+    object: &str,
+    expected_incarnation_id: Option<Uuid>,
+) -> Result<bool> {
+    let results = stream::iter(pool.disk_set.iter().cloned())
+        .map(|set| async move {
+            set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
+                .await
+        })
+        .buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
+        .collect::<Vec<_>>()
+        .await;
+    let mut found = false;
+    for result in results {
+        // Preserve errors from every set instead of hiding them behind a hit.
+        found |= result?;
+    }
+    Ok(found)
+}
+
 async fn list_pool_multipart_uploads_for_incarnation(
     pool: &crate::core::sets::Sets,
     bucket: &str,
@@ -472,6 +512,8 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<ListPartsInfo> {
         check_list_parts_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         let opts = &opts;
 
@@ -602,6 +644,8 @@ impl ECStore {
         mutation_fence: Option<&ObjectLockDiagGuard>,
     ) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
         check_new_object_args(bucket, object, opts.data_movement)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
 
         if self.single_pool() {
@@ -657,21 +701,7 @@ impl ECStore {
             if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
                 continue;
             }
-            let res = list_pool_multipart_uploads_for_incarnation(
-                pool,
-                bucket,
-                &MultipartUploadListRequest {
-                    prefix: object.to_string(),
-                    key_marker: None,
-                    upload_id_marker: None,
-                    delimiter: None,
-                    max_uploads: MAX_UPLOADS_LIST,
-                    expected_incarnation_id: opts.expected_bucket_incarnation_id,
-                },
-            )
-            .await?;
-
-            if !res.uploads.is_empty() {
+            if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
                 self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
                     .await;
                 let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
@@ -754,6 +784,8 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<PartInfo> {
         check_put_object_part_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         if let Some(guard) = bucket_lifecycle_guard {
             opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
@@ -788,6 +820,8 @@ impl ECStore {
             .part_number
             .ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
         check_put_object_part_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         if !opts.data_movement {
             return Err(Error::other("targeted multipart upload requires data_movement options"));
         }
@@ -820,6 +854,8 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<MultipartInfo> {
         check_list_parts_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         let opts = &opts;
         if self.single_pool() {
@@ -853,6 +889,8 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<()> {
         check_abort_multipart_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
 
         for pool_idx in self.existing_multipart_pool_order().await {
@@ -1034,6 +1072,8 @@ impl ECStore {
         for upload_id in upload_ids {
             check_abort_multipart_args(bucket, object, upload_id)?;
         }
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         if !opts.data_movement {
             return Err(Error::other("targeted multipart abort requires data_movement options"));
         }
@@ -1168,11 +1208,9 @@ impl ECStore {
         let oi = if require_all_pools {
             lookup_opts.metadata_cache_safe = false;
             lookup_opts.suppress_read_repair = true;
-            if let Err(err) = check_object_args(bucket, object) {
-                return Some(err);
-            }
-            let encoded_object = encode_dir_object(object);
-            self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
+            // Only completion uses this branch, after validating bucket and object.
+            let encoded_object = encode_dir_object_ref(object);
+            self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
                 .await
                 .and_then(|(info, _)| {
                     lookup_opts.precondition_check(&info)?;
@@ -1194,7 +1232,34 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<ObjectInfo> {
         check_complete_multipart_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
+            // The set-level commit judges the null version this completion
+            // replaces; give it the same authoritative snapshot a PUT carries.
+            let expected_incarnation_id = opts
+                .expected_bucket_incarnation_id
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
+            let lifecycle_fence = opts
+                .bucket_lifecycle_lock_fence
+                .as_ref()
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
+            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
+                Some(snapshot) => Arc::clone(snapshot),
+                None => {
+                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
+                        .await?
+                }
+            };
+            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
+                return Err(Error::other(
+                    "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
+                ));
+            }
+            snapshot.add_lock_fences(&mut opts);
+            opts.object_lock_config_snapshot = Some(snapshot);
+        }
         if !self.single_pool() {
             opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
         }
@@ -1303,6 +1368,8 @@ impl ECStore {
         let request_opts = opts;
         let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         if !opts.data_movement {
             return Err(Error::other("targeted multipart completion requires data_movement options"));
         }
@@ -1424,6 +1491,362 @@ mod tests {
     };
     use std::time::Duration;
 
+    #[tokio::test]
+    async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
+        use crate::bucket::metadata_sys::init_bucket_metadata_sys;
+        use crate::runtime::instance::InstanceContext;
+
+        let mut dirs = Vec::new();
+        let mut pools = Vec::new();
+        for pool_index in 0..2 {
+            let mut endpoints = Vec::new();
+            for disk_index in 0..4 {
+                let directory = tempfile::tempdir().expect("temporary disk");
+                let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
+                endpoint.set_pool_index(pool_index);
+                endpoint.set_set_index(0);
+                endpoint.set_disk_index(disk_index);
+                dirs.push(directory);
+                endpoints.push(endpoint);
+            }
+            pools.push(PoolEndpoints {
+                legacy: false,
+                set_count: 1,
+                drives_per_set: 4,
+                endpoints: Endpoints::from(endpoints),
+                cmd_line: format!("exact-multipart-pool-{pool_index}"),
+                platform: "test".to_owned(),
+            });
+        }
+        let endpoints = EndpointServerPools(pools);
+        let ctx = Arc::new(InstanceContext::new());
+        crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
+            .await
+            .expect("initialize local disks");
+        let cancel = tokio_util::sync::CancellationToken::new();
+        let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
+            .await
+            .expect("initialize two pools");
+        init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "exact-pool-selection";
+        let other_bucket = "other-pool-selection";
+        for bucket in [bucket, other_bucket] {
+            store
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("create bucket");
+        }
+        for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
+            let (opts, _guard) = store
+                .guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
+                .await
+                .expect("pin upload incarnation");
+            store.pools[0]
+                .new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
+                .await
+                .expect("seed nonmatching upload in first pool");
+        }
+        let (opts, _guard) = store
+            .guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
+            .await
+            .expect("pin target incarnation");
+        assert!(
+            !pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
+                .await
+                .expect("first pool contains only other keys")
+        );
+        store.pools[1]
+            .new_multipart_upload(bucket, "report", &opts)
+            .await
+            .expect("seed exact upload in second pool");
+        let (created, pool_index, incarnation) = store
+            .handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
+            .await
+            .expect("create another upload for the exact key");
+        assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
+        assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
+        assert!(!created.upload_id.is_empty());
+        assert!(
+            !pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
+                .await
+                .expect("new upload must not appear in first pool")
+        );
+        assert!(
+            !pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
+                .await
+                .expect("old bucket incarnations must not affect placement")
+        );
+        cancel.cancel();
+    }
+
+    #[tokio::test]
+    async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::object::ObjectIO as _;
+        use sha2::Digest;
+        use tokio::io::AsyncReadExt;
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        for versioned in [false, true] {
+            let bucket = if versioned {
+                "directory-mpu-versioned"
+            } else {
+                "directory-mpu-unversioned"
+            };
+            store
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("create bucket");
+            if versioned {
+                store
+                    .update_bucket_metadata_config(
+                        bucket,
+                        "versioning.xml",
+                        br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
+                    )
+                    .await
+                    .expect("enable versioning");
+            }
+            let opts = ObjectOptions {
+                versioned,
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                ..Default::default()
+            };
+            let original = store
+                .put_object(
+                    bucket,
+                    "protected",
+                    &mut PutObjReader::from_vec(b"private original".to_vec()),
+                    &ObjectOptions {
+                        user_defined: HashMap::from([(
+                            rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
+                            "classification=secret".to_string(),
+                        )]),
+                        ..opts.clone()
+                    },
+                )
+                .await
+                .expect("write protected bare object");
+            let upload = store
+                .new_multipart_upload(bucket, "protected/", &opts)
+                .await
+                .expect("create directory upload");
+            let part = store
+                .put_object_part(
+                    bucket,
+                    "protected/",
+                    &upload.upload_id,
+                    1,
+                    &mut PutObjReader::from_vec(b"directory payload".to_vec()),
+                    &opts,
+                )
+                .await
+                .expect("write directory part");
+            store
+                .put_object_part(
+                    bucket,
+                    "protected/",
+                    &upload.upload_id,
+                    2,
+                    &mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
+                    &opts,
+                )
+                .await
+                .expect("write an unselected part");
+            let upload_sha = hex_simd::encode_to_string(
+                sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
+                hex_simd::AsciiCase::Lower,
+            );
+            let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
+            for dir in &dirs {
+                let staging = dir
+                    .path()
+                    .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
+                    .join(&upload_sha)
+                    .join(&upload_uuid);
+                let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
+                    .expect("decode upload metadata");
+                let versions = metadata
+                    .get_file_info_versions(bucket, "protected__XLDIR__", true)
+                    .expect("upload file info");
+                let data_dir = versions
+                    .versions
+                    .first()
+                    .expect("upload version")
+                    .data_dir
+                    .expect("staging data directory");
+                assert!(
+                    staging.join(data_dir.to_string()).join("part.2").is_file(),
+                    "unselected part exists before completion"
+                );
+            }
+            assert_eq!(
+                store
+                    .get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
+                    .await
+                    .expect("upload info")
+                    .object,
+                "protected/"
+            );
+            assert_eq!(
+                store
+                    .list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
+                    .await
+                    .expect("list parts")
+                    .object,
+                "protected/"
+            );
+            let uploads = store
+                .list_multipart_uploads(bucket, "protected/", None, None, None, 10)
+                .await
+                .expect("list logical prefix");
+            assert_eq!(uploads.uploads.len(), 1);
+            assert_eq!(uploads.uploads[0].object, "protected/");
+            assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
+            let completed = store
+                .clone()
+                .complete_multipart_upload(
+                    bucket,
+                    "protected/",
+                    &upload.upload_id,
+                    vec![CompletePart {
+                        part_num: part.part_num,
+                        etag: part.etag,
+                        ..Default::default()
+                    }],
+                    &opts,
+                )
+                .await
+                .expect("complete directory upload");
+            assert_eq!(completed.name, "protected/");
+            for (key, expected) in [
+                ("protected", b"private original".as_slice()),
+                ("protected/", b"directory payload".as_slice()),
+            ] {
+                let mut reader = store
+                    .get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("read distinct object");
+                let mut bytes = Vec::new();
+                reader.stream.read_to_end(&mut bytes).await.expect("read body");
+                assert_eq!(bytes, expected);
+            }
+            let preserved = store
+                .get_object_info(bucket, "protected", &ObjectOptions::default())
+                .await
+                .expect("bare metadata");
+            assert_eq!(preserved.version_id, original.version_id);
+            assert_eq!(preserved.etag, original.etag);
+            assert_eq!(preserved.user_tags.as_str(), "classification=secret");
+            for dir in &dirs {
+                assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
+                assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
+                let data = dir
+                    .path()
+                    .join(bucket)
+                    .join("protected__XLDIR__")
+                    .join(completed.data_dir.expect("multipart data directory").to_string());
+                assert!(data.join("part.1").is_file());
+                assert!(!data.join("part.2").exists(), "unselected part must never be published");
+                assert!(
+                    !dir.path()
+                        .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
+                        .join(&upload_sha)
+                        .join(&upload_uuid)
+                        .exists(),
+                    "original staging must be cleaned"
+                );
+            }
+            let abandoned = store
+                .new_multipart_upload(bucket, "protected/", &opts)
+                .await
+                .expect("create abort control");
+            store
+                .abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
+                .await
+                .expect("abort encoded upload");
+            assert!(
+                store
+                    .get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
+                    .await
+                    .is_err()
+            );
+            assert!(
+                store
+                    .get_object_info(bucket, "protected/", &ObjectOptions::default())
+                    .await
+                    .is_ok()
+            );
+        }
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::object::ObjectIO as _;
+        use tokio::io::AsyncReadExt;
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "xml-key-admission";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
+            let key = format!("bad-{}", char::from(byte));
+            assert!(
+                store
+                    .put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
+                    .await
+                    .is_err()
+            );
+            assert!(
+                store
+                    .new_multipart_upload(bucket, &key, &ObjectOptions::default())
+                    .await
+                    .is_err()
+            );
+            for dir in &dirs {
+                assert!(!dir.path().join(bucket).join(&key).exists());
+            }
+        }
+        let legacy_key = "legacy-\u{1}";
+        store.pools[0]
+            .get_disks_by_key(legacy_key)
+            .put_object(
+                bucket,
+                legacy_key,
+                &mut PutObjReader::from_vec(b"legacy payload".to_vec()),
+                &ObjectOptions {
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed a pre-upgrade object below new-key admission");
+        let mut reader = store
+            .get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("legacy GET stays usable");
+        let mut bytes = Vec::new();
+        reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
+        assert_eq!(bytes, b"legacy payload");
+        drop(reader);
+        store
+            .delete_object(bucket, legacy_key, ObjectOptions::default())
+            .await
+            .expect("legacy DELETE stays usable");
+        assert!(
+            store
+                .get_object_info(bucket, legacy_key, &ObjectOptions::default())
+                .await
+                .is_err()
+        );
+    }
+
     fn mp(object: &str, upload_id: &str) -> MultipartInfo {
         MultipartInfo {
             bucket: "bucket".to_string(),

crates/ecstore/tests/metadata_read_lock_reuse_test.rs

@@ -20,14 +20,230 @@ use std::sync::Arc;
 use std::time::Duration;
 use storage_api::contract_compat::ECStore;
 use storage_api::metadata_lock::{
-    BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
-    ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
-    isolated_store_over_temp_disks,
+    BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
+    NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
+    init_bucket_metadata_sys, isolated_store_over_temp_disks,
 };
 use tokio::io::AsyncReadExt;
 use tokio::time::timeout;
 use uuid::Uuid;
 
+struct ObservedPutBody {
+    inner: tokio::io::DuplexStream,
+    polled: Arc<tokio::sync::Notify>,
+}
+
+impl tokio::io::AsyncRead for ObservedPutBody {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        self.polled.notify_one();
+        std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
+    }
+}
+
+#[tokio::test]
+async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
+    use tokio::io::AsyncWriteExt;
+    for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "deferred-put-locks";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket");
+        let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
+        let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
+        let (mut sender, inner) = tokio::io::duplex(16);
+        let polled = Arc::new(tokio::sync::Notify::new());
+        let body = ObservedPutBody {
+            inner,
+            polled: polled.clone(),
+        };
+        let put_store = store.clone();
+        let put = tokio::spawn(async move {
+            let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
+            put_store
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        version_id,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        timeout(Duration::from_secs(5), polled.notified())
+            .await
+            .expect("PUT must be waiting for client data");
+        let lock = store
+            .new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
+            .await
+            .expect("metadata lock");
+        drop(
+            timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
+                .await
+                .expect("body must not block metadata writers")
+                .expect("metadata write guard"),
+        );
+        drop(
+            timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
+                .await
+                .expect("body must not retain lifecycle guard")
+                .expect("lifecycle writer"),
+        );
+        sender.write_all(b"body").await.expect("finish body");
+        sender.shutdown().await.expect("EOF");
+        timeout(Duration::from_secs(5), barrier.wait_until_paused())
+            .await
+            .expect("staged PUT reaches commit");
+        let writer = lock.get_write_lock(Duration::from_secs(5));
+        tokio::pin!(writer);
+        assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
+        let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
+        tokio::pin!(lifecycle);
+        assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
+        barrier.release();
+        let release_metadata = async {
+            drop(writer.await.expect("metadata writer"));
+        };
+        let release_lifecycle = async {
+            drop(lifecycle.await.expect("lifecycle writer"));
+        };
+        timeout(Duration::from_secs(5), async {
+            let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
+            put.expect("owner task").expect("commit");
+        })
+        .await
+        .expect("PUT and queued writers finish");
+    }
+}
+
+#[tokio::test]
+async fn staged_put_cannot_publish_into_a_recreated_bucket() {
+    use tokio::io::AsyncWriteExt;
+    let (_dirs, store) = isolated_store_over_temp_disks().await;
+    init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+    let bucket = "deferred-put-generation";
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("bucket");
+    let (mut sender, inner) = tokio::io::duplex(16);
+    let polled = Arc::new(tokio::sync::Notify::new());
+    let body = ObservedPutBody {
+        inner,
+        polled: polled.clone(),
+    };
+    let put_store = store.clone();
+    let put = tokio::spawn(async move {
+        let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
+        put_store
+            .put_object(bucket, "object", &mut reader, &ObjectOptions::default())
+            .await
+    });
+    timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
+    store
+        .delete_bucket(
+            bucket,
+            &DeleteBucketOptions {
+                force: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("remove old bucket while PUT stalls");
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("recreate bucket");
+    sender.write_all(b"body").await.expect("finish body");
+    sender.shutdown().await.expect("EOF");
+    let error = timeout(Duration::from_secs(5), put)
+        .await
+        .expect("commit gate")
+        .expect("owner")
+        .expect_err("old request generation must not publish");
+    assert!(
+        error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
+        "{error:?}"
+    );
+    assert!(
+        store
+            .get_object_info(bucket, "object", &ObjectOptions::default())
+            .await
+            .is_err()
+    );
+}
+
+#[tokio::test]
+async fn staged_put_rechecks_current_object_lock_at_commit() {
+    use tokio::io::AsyncWriteExt;
+    let (_dirs, store) = isolated_store_over_temp_disks().await;
+    init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+    let bucket = "deferred-put-retention";
+    store
+        .make_bucket(
+            bucket,
+            &MakeBucketOptions {
+                lock_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("locked bucket");
+    let mut old = PutObjReader::from_vec(b"old".to_vec());
+    store
+        .put_object(bucket, "object", &mut old, &ObjectOptions::default())
+        .await
+        .expect("unlocked predecessor");
+    let (mut sender, inner) = tokio::io::duplex(16);
+    let polled = Arc::new(tokio::sync::Notify::new());
+    let body = ObservedPutBody {
+        inner,
+        polled: polled.clone(),
+    };
+    let put_store = store.clone();
+    let put = tokio::spawn(async move {
+        let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
+        put_store
+            .put_object(bucket, "object", &mut reader, &ObjectOptions::default())
+            .await
+    });
+    timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
+    let mut protected = PutObjReader::from_vec(b"protected".to_vec());
+    store
+        .put_object(
+            bucket,
+            "object",
+            &mut protected,
+            &ObjectOptions {
+                user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("install a protected predecessor while the client stalls");
+    sender.write_all(b"body").await.expect("finish stale body");
+    sender.shutdown().await.expect("EOF");
+    let error = timeout(Duration::from_secs(5), put)
+        .await
+        .expect("commit gate")
+        .expect("task")
+        .expect_err("pre-body state cannot admit an overwrite of the newly protected version");
+    assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
+    let info = store
+        .get_object_info(bucket, "object", &ObjectOptions::default())
+        .await
+        .expect("protected object remains");
+    assert_eq!(info.size, 9);
+}
+
 #[tokio::test]
 async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
     replica_write_with_waiting_metadata_writer(false).await;

crates/ecstore/tests/storage_api.rs

@@ -9,12 +9,14 @@ use rustfs_storage_api as storage_contracts;
 #[cfg(feature = "test-util")]
 pub(crate) mod metadata_lock {
     pub(crate) use super::storage_contracts::{
-        BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
+        BucketOperations, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO,
+        ObjectOperations,
     };
     pub(crate) use super::{Error, ObjectOptions, PutObjReader};
     pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
         init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
     };
+    pub(crate) use rustfs_ecstore::api::rio::HashReader;
     pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
 }
 

crates/filemeta/src/filemeta.rs

@@ -3930,6 +3930,34 @@ mod test {
         assert!(versions_with_free.versions.iter().all(|version| version.num_versions == 2));
     }
 
+    #[test]
+    fn add_version_rejects_mod_times_that_cannot_round_trip() {
+        for nanos in [0, i128::from(i64::MAX) + 1, i128::from(i64::MIN) - 1, 1_i128 << 64] {
+            for deleted in [false, true] {
+                let mut meta = FileMeta::new();
+                let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
+                info.deleted = deleted;
+                info.mod_time = Some(OffsetDateTime::from_unix_timestamp_nanos(nanos).expect("time within supported date range"));
+                assert!(
+                    meta.add_version(info).is_err(),
+                    "unrepresentable time {nanos} must be rejected before publication"
+                );
+            }
+        }
+        for nanos in [-1, 1, i64::MIN, i64::MAX] {
+            let mut meta = FileMeta::new();
+            let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
+            let time = OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).expect("valid codec time");
+            info.mod_time = Some(time);
+            meta.add_version(info).expect("representable times remain supported");
+            let reopened = FileMeta::load(&meta.marshal_msg().expect("serialize version")).expect("reopen version");
+            let versions = reopened
+                .get_file_info_versions("bucket", "object", false)
+                .expect("read version history");
+            assert_eq!(versions.versions[0].mod_time, Some(time));
+        }
+    }
+
     #[test]
     fn test_data_integrity_validation() {
         // Test data integrity checks

crates/filemeta/src/filemeta/version.rs

@@ -858,13 +858,24 @@ impl FileMetaVersion {
             VersionType::Object => self
                 .object
                 .as_ref()
-                .map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some())
+                .map(|v| {
+                    v.erasure_algorithm.valid()
+                        && v.bitrot_checksum_algo.valid()
+                        && v.mod_time.is_some_and(|time| {
+                            let nanos = time.unix_timestamp_nanos();
+                            nanos != 0 && i64::try_from(nanos).is_ok()
+                        })
+                })
                 .unwrap_or_default(),
             VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::valid).unwrap_or_default(),
             VersionType::Delete => self
                 .delete_marker
                 .as_ref()
-                .map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH)
+                .map(|v| {
+                    v.mod_time.is_some_and(|time| {
+                        time > OffsetDateTime::UNIX_EPOCH && i64::try_from(time.unix_timestamp_nanos()).is_ok()
+                    })
+                })
                 .unwrap_or_default(),
             _ => false,
         }

crates/filemeta/src/metacache.rs

@@ -925,6 +925,13 @@ impl MetaCacheEntriesSorted {
 
 const METACACHE_STREAM_VERSION: u8 = 2;
 const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
+/// Upper bound on one incremental reservation while accumulating a single
+/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
+/// up to this bound so a legitimately large xl.meta does not pay one small
+/// read per chunk, while a corrupt length prefix still cannot force a
+/// reservation larger than the payload that has already arrived (see
+/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
+const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
 /// msgpack `false`: the entry-stream terminator, and the only byte a legacy
 /// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
 const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1157,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         // A whole xl.meta can legitimately exceed the limit on its individual
         // msgpack values. Grow only as payload arrives, so a corrupt length
         // prefix cannot cause a speculative allocation of the advertised size.
+        let mut chunk = METACACHE_READ_CHUNK_SIZE;
         while self.buf.len() < read_size {
             let start = self.buf.len();
             let extra = if lookahead {
                 let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
-                (read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).clamp(min_chunk, chunk)
             } else {
-                (read_size - start).min(METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).min(chunk)
             };
             if let Err(e) = self.buf.try_reserve(extra) {
                 let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1194,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
                     return Err(err);
                 }
             }
+            chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
         }
 
         self.pos = read_size;
@@ -2155,6 +2164,76 @@ mod tests {
         }
     }
 
+    /// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
+    /// decode completely even when the aggregate stream exceeds the per-value
+    /// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
+    /// element.
+    #[tokio::test]
+    async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
+        // ~2.9 KiB of metadata per entry matches the per-entry cost observed in
+        // the #9120 production report; 6,000 entries push the stream past 16 MiB.
+        const ENTRIES: usize = 6_000;
+        let metadata = vec![0xab; 2_880];
+        let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
+            .map(|i| MetaCacheEntry {
+                name: format!("bigdir/object-{i:05}"),
+                metadata: metadata.clone(),
+                ..Default::default()
+            })
+            .collect();
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("wide listing must encode");
+        writer.close().await.expect("stream must close");
+        let bytes = output.into_inner();
+        assert!(
+            bytes.len() > 16 << 20,
+            "fixture must exceed the per-value limit, got {} bytes",
+            bytes.len()
+        );
+        let mut reader = MetacacheReader::new(Cursor::new(bytes));
+        let decoded = reader.read_all().await.expect("wide listing must decode");
+        assert_eq!(decoded, entries);
+    }
+
+    /// The #9120 failure shape seen in production: one entry whose metadata is
+    /// an xl.meta that grew past 16 MiB with its version history, listed among
+    /// ordinary neighbors and skipped past without losing the framing.
+    #[tokio::test]
+    async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
+        let small = |i: u8| MetaCacheEntry {
+            name: format!("bigdir/object-{i}"),
+            metadata: vec![i; 64],
+            ..Default::default()
+        };
+        let oversized = MetaCacheEntry {
+            name: "bigdir/huge-versions".to_string(),
+            metadata: vec![0xcd; (16 << 20) + 12_345],
+            ..Default::default()
+        };
+        let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("mixed listing must encode");
+        writer.close().await.expect("stream must close");
+
+        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
+        assert_eq!(
+            reader.peek().await.expect("first entry").expect("present"),
+            entries[0],
+            "an ordinary entry before the oversized one must decode"
+        );
+        reader.skip(1).await.expect("skip the first entry");
+        assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
+        assert_eq!(
+            reader.peek().await.expect("oversized entry").expect("present"),
+            oversized,
+            "an entry past the per-value limit must round-trip whole"
+        );
+        reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
+        assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
+    }
+
     #[test]
     fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
         let version_id = Uuid::new_v4();
@@ -3775,6 +3854,74 @@ mod tests {
         assert!(corrupt_crc_entry.is_latest_delete_marker());
     }
 
+    #[test]
+    fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
+        let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
+            .expect("load realistic xl.meta");
+        let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
+        let mut free_meta_sys = HashMap::new();
+        rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
+        let free = FileMetaShallowVersion::try_from(FileMetaVersion {
+            version_type: VersionType::Delete,
+            delete_marker: Some(MetaDeleteMarker {
+                version_id: Some(Uuid::from_u128(42)),
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
+                meta_sys: free_meta_sys,
+            }),
+            write_version: 1,
+            ..Default::default()
+        })
+        .expect("encode cleanup record");
+        fm.versions.insert(0, free);
+        let entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scanner version decode");
+        assert_eq!(scanned.free_versions.len(), 1);
+        assert_eq!(scanned.versions.len(), 1);
+        assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
+        assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
+        let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
+        assert_eq!(listed.version_id, object.version_id);
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
+        assert_eq!(listed.size, scanned.versions[0].size);
+        assert!(!listed.deleted);
+
+        let marker_id = Uuid::from_u128(43);
+        fm.add_version(FileInfo {
+            volume: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(marker_id),
+            deleted: true,
+            mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
+            ..Default::default()
+        })
+        .expect("add current delete marker");
+        let mut marker_entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode marker metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scan marker stack");
+        assert_eq!(scanned.versions.len(), 2);
+        assert_eq!(scanned.versions[0].version_id, Some(marker_id));
+        assert!(scanned.versions[0].deleted);
+        assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
+        assert!(marker_entry.is_latest_delete_marker());
+        let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert!(listed.deleted);
+    }
+
     #[test]
     fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
         let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))

crates/heal/src/error.rs

@@ -77,6 +77,9 @@ pub enum Error {
     #[error("Replacement target is not ready: {0}")]
     ReplacementTargetNotReady(String),
 
+    #[error("Replacement bucket namespace changed; a fresh identity-bound pass is required")]
+    ReplacementNamespaceChanged,
+
     #[error("replacement recovery retry budget exhausted")]
     ReplacementRetryBudgetExhausted,
 
@@ -128,6 +131,23 @@ impl Error {
         Error::TransientSkip { message: message.into() }
     }
 
+    /// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
+    /// Callers must cut their own concurrency instead of retrying at full speed.
+    pub(crate) fn is_slow_down(&self) -> bool {
+        match self {
+            Error::Storage(error) => matches!(
+                error,
+                EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles | EcstoreError::DeleteRecoveryWorkLimit
+            ),
+            Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
+            Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
+                message.contains("Please reduce your request rate")
+            }
+            Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
+            _ => false,
+        }
+    }
+
     /// Whether a heal operation can be retried without changing its inputs.
     pub(crate) fn is_recoverable_heal(&self) -> bool {
         match classify_heal_error(self) {
@@ -215,9 +235,10 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
         Error::DanglingDeleteDeferred { retry_not_before } => HealErrorClass::DanglingGrace {
             retry_not_before: Some(*retry_not_before),
         },
-        Error::ReplacementTargetNotReady(_) | Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => {
-            HealErrorClass::Transient
-        }
+        Error::ReplacementTargetNotReady(_)
+        | Error::ReplacementNamespaceChanged
+        | Error::TransientSkip { .. }
+        | Error::BucketDiscoveryIncomplete => HealErrorClass::Transient,
         Error::LegacyMrfUnverified { .. } => HealErrorClass::Permanent,
         Error::Storage(error) => classify_storage_error(error),
         Error::Disk(error) => classify_disk_error(error),
@@ -271,6 +292,7 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
         | EcstoreError::FaultyDisk
         | EcstoreError::FaultyRemoteDisk
         | EcstoreError::SlowDown
+        | EcstoreError::DeleteRecoveryWorkLimit
         | EcstoreError::OperationCanceled
         | EcstoreError::RemoteClientUnavailable(_) => HealErrorClass::Transient,
         _ if error.is_quorum_error() => HealErrorClass::Transient,
@@ -449,6 +471,22 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
+    #[test]
+    fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
+        assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
+        assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
+        assert!(
+            Error::TaskExecutionFailed {
+                message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
+            }
+            .is_slow_down()
+        );
+        assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
+        assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
+    }
+
     #[test]
     fn pool_metadata_failures_keep_their_typed_recovery_classification() {
         use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};

crates/heal/src/heal/erasure_healer.rs

@@ -14,6 +14,7 @@
 
 use crate::error::{HealErrorClass, classify_heal_error};
 use crate::heal::{
+    pacing::HealPageRate,
     progress::{HealProgress, add_bytes, increment_counter},
     resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -128,12 +129,12 @@ pub struct ErasureSetHealer {
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
     replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
     work_parent: Option<super::manager::HealWork>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 async fn acquire_page_permit(
     semaphore: Arc<Semaphore>,
-    pacer: Option<&super::pacing::MainlinePacer>,
+    rate: &HealPageRate,
     cancel: &tokio_util::sync::CancellationToken,
 ) -> Result<tokio::sync::OwnedSemaphorePermit> {
     let acquire = || async {
@@ -146,18 +147,16 @@ async fn acquire_page_permit(
     let mut paid_pause = false;
     loop {
         let permit = acquire().await?;
-        if let Some(pacer) = pacer {
-            // Keep the real permit on the low-pressure path. Every acquisition
-            // gets a fresh decision, including a waiter that queued a second
-            // time. One completed pause is a bounded minimum-progress grant.
-            match pacer.admission_decision() {
-                super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
-                    drop(permit);
-                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
-                    continue;
-                }
-                _ => {}
+        // Keep the real permit on the low-pressure path. Every acquisition
+        // gets a fresh decision, including a waiter that queued a second
+        // time. One completed pause is a bounded minimum-progress grant.
+        match rate.admission_decision() {
+            super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
+                drop(permit);
+                paid_pause = rate.wait_after_admission(cancel, pressure).await?;
+                continue;
             }
+            _ => {}
         }
         return Ok(permit);
     }
@@ -166,7 +165,7 @@ async fn acquire_page_permit(
 #[cfg(test)]
 mod mainline_pacing_tests {
     use super::*;
-    use crate::heal::pacing::{MainlinePacer, TestPressure};
+    use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
     use tokio_util::sync::CancellationToken;
@@ -180,15 +179,15 @@ mod mainline_pacing_tests {
             .await
             .expect("existing object owns capacity");
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
-        let pacer = Arc::new(
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
-        );
+        let pacer =
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
+        let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
         let cancel = CancellationToken::new();
         let waiting = tokio::spawn({
             let semaphore = semaphore.clone();
-            let pacer = pacer.clone();
+            let rate = rate.clone();
             let cancel = cancel.clone();
-            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
+            async move { acquire_page_permit(semaphore, &rate, &cancel).await }
         });
         tokio::task::yield_now().await;
         provider.active.store(100, Ordering::SeqCst);
@@ -200,12 +199,13 @@ mod mainline_pacing_tests {
         assert_eq!(semaphore.available_permits(), 1);
         let deadline = tokio::time::timeout(
             Duration::from_millis(10),
-            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
+            acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
         )
         .await;
         assert!(deadline.is_err());
         assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
-        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
+        let unpaced = HealPageRate::new(1, None);
+        let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
             .await
             .expect("unpaced admission");
         assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,10 +221,11 @@ mod mainline_pacing_tests {
             let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
             let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let rate = HealPageRate::new(1, Some(pacer));
             let cancel = CancellationToken::new();
-            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
-            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
             assert!(futures::poll!(first.as_mut()).is_pending());
             assert!(futures::poll!(second.as_mut()).is_pending());
             drop(occupied);
@@ -329,6 +330,20 @@ impl ErasureSetHealer {
         }
     }
 
+    /// Page fan-out for one whole erasure-set pass (stable across buckets).
+    fn pass_page_concurrency_limit(&self) -> usize {
+        if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
+            // Versions of the same object must not race without its namespace lock.
+            1
+        } else {
+            Self::effective_heal_page_object_concurrency_for_source(
+                self.source,
+                self.heal_opts.scan_mode,
+                self.replacement_task_id.is_some(),
+            )
+        }
+    }
+
     /// Map the shared heal error class onto this healer's per-object actions.
     ///
     /// `classify_heal_error` is the only classifier; this adapter decides what
@@ -373,7 +388,7 @@ impl ErasureSetHealer {
         }
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }
@@ -453,7 +468,16 @@ impl ErasureSetHealer {
         let task_id = self.get_or_create_task_id(set_disk_id).await?;
 
         // 2. initialize or resume resume state
-        let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
+        let (resume_manager, mut checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
+        let replacement_buckets = if self.replacement_task_id.is_some() {
+            let current = resume_manager.prepare_replacement_bucket_pass(self.storage.as_ref()).await?;
+            // Preparation may have durably rewound the companion checkpoint.
+            checkpoint_manager = CheckpointManager::load_from_disk(self.disk.clone(), &task_id).await?;
+            Some(current)
+        } else {
+            None
+        };
+        let buckets = replacement_buckets.as_deref().unwrap_or(buckets);
 
         let admin_buckets = if self.admin_task.is_some() {
             Some(
@@ -464,12 +488,7 @@ impl ErasureSetHealer {
             None
         };
         let buckets = admin_buckets.as_deref().unwrap_or(buckets);
-        if checkpoint_manager
-            .get_checkpoint()
-            .await
-            .admin
-            .is_some_and(|admin| admin.completed)
-        {
+        if checkpoint_manager.admin_completed().await {
             self.initialize_progress(buckets, &resume_manager.get_state().await).await;
             return Ok(());
         }
@@ -479,6 +498,9 @@ impl ErasureSetHealer {
             .execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
             .await;
 
+        if self.replacement_task_id.is_some() {
+            resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
+        }
         result?;
         self.verify_replacement_identity_fence("completion").await?;
 
@@ -490,14 +512,7 @@ impl ErasureSetHealer {
         }
 
         if self.admin_task.is_some() {
-            let mut admin = checkpoint_manager
-                .get_checkpoint()
-                .await
-                .admin
-                .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
-            admin.completed = true;
-            admin.outcome.finish(None);
-            checkpoint_manager.set_admin(admin).await?;
+            checkpoint_manager.complete_admin().await?;
         } else {
             checkpoint_manager.cleanup().await?;
             resume_manager.cleanup().await?;
@@ -718,7 +733,10 @@ impl ErasureSetHealer {
             }
             return Err(Error::DanglingDeleteDeferred { retry_not_before });
         }
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        // One rate controller for the whole pass: SlowDown cool-down and page
+        // fan-out persist across buckets instead of resetting at each one.
+        let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
+        let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
 
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -726,14 +744,17 @@ impl ErasureSetHealer {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
             set_disk_id,
-            current_bucket_index = checkpoint.current_bucket_index,
-            current_object_index = checkpoint.current_object_index,
+            current_bucket_index,
+            current_object_index,
             state = "resuming",
             "Erasure set resumed"
         );
 
         // 2. initialize progress
         self.initialize_progress(buckets, &state).await;
+        // Snapshot before any baseline write: set_progress_baseline overwrites
+        // the saved generation, which would hide a real generation mismatch.
+        let checkpoint = checkpoint_manager.progress_snapshot().await;
         let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
             let baseline = self.progress.read().await;
             (
@@ -760,17 +781,7 @@ impl ErasureSetHealer {
         let mut successful_objects = state.successful_objects;
         let mut failed_objects = state.failed_objects;
         let mut skipped_objects = state.skipped_objects;
-        let checkpoint_has_progress = checkpoint.baseline_known
-            || checkpoint.successful_objects > 0
-            || checkpoint.failed_object_count > 0
-            || checkpoint.skipped_object_count > 0
-            || checkpoint.skipped_new_versions > 0
-            || checkpoint.skipped_ilm_expired > 0
-            || checkpoint.processed_bytes > 0
-            || checkpoint.total_objects > 0
-            || checkpoint.total_bytes > 0
-            || checkpoint.baseline_generation.is_some()
-            || checkpoint.counter_unknown;
+        let checkpoint_has_progress = checkpoint.has_progress();
         let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
         let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
         if checkpoint_has_progress {
@@ -854,6 +865,7 @@ impl ErasureSetHealer {
                     resume_manager,
                     checkpoint_manager,
                     state.start_time,
+                    &page_rate,
                 )
                 .await;
 
@@ -967,9 +979,16 @@ impl ErasureSetHealer {
                 // Recheck even resumed buckets: their user-object cursor does not
                 // prove that the replacement holds the internal bucket records.
                 for bucket in buckets {
-                    self.storage
-                        .heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
-                        .await?;
+                    if self.replacement_task_id.is_some() {
+                        let expected = resume_manager.replacement_bucket_incarnation(bucket).await?;
+                        self.storage
+                            .heal_replacement_bucket_metadata_at_incarnation(bucket, expected, &self.heal_opts, targets)
+                            .await?;
+                    } else {
+                        self.storage
+                            .heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
+                            .await?;
+                    }
                 }
             }
             self.heal_replacement_pool_metadata(
@@ -1019,15 +1038,16 @@ impl ErasureSetHealer {
         // never hot-retried in place here.
         if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
             if self.replacement_task_id.is_some() {
+                resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
                 let state = resume_manager.get_state().await;
-                let checkpoint = checkpoint_manager.get_checkpoint().await;
+                let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
                 if state.can_retry()
                     && failed_objects == 0
                     && failed_buckets == 0
                     && skipped_objects > 0
-                    && skipped_objects == checkpoint.dangling_delete_grace_objects
-                    && !checkpoint.counter_unknown
-                    && let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
+                    && skipped_objects == grace_objects
+                    && !counter_unknown
+                    && let Some(retry_not_before) = retry_not_before
                 {
                     // Publish the generation's rewind and wait first. If the
                     // checkpoint reset is interrupted, initialization reapplies
@@ -1167,14 +1187,9 @@ impl ErasureSetHealer {
 
         let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
         let checkpoint_key = compose_key(&object_key, None);
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
-        if checkpoint.processed_objects.contains(&checkpoint_key)
-            || checkpoint.failed_objects.contains(&checkpoint_key)
-            || checkpoint.skipped_objects.contains(&checkpoint_key)
-        {
+        if checkpoint_manager.is_object_settled(&checkpoint_key).await {
             return Ok(());
         }
-        drop(checkpoint);
 
         self.verify_replacement_identity_fence("pool metadata").await?;
         resume_manager
@@ -1428,7 +1443,7 @@ impl ErasureSetHealer {
 
     /// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
     #[allow(clippy::too_many_arguments)]
-    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
+    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
     async fn heal_bucket_with_resume(
         &self,
         bucket: &str,
@@ -1442,6 +1457,7 @@ impl ErasureSetHealer {
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
         started_at_secs: u64,
+        page_rate: &Arc<HealPageRate>,
     ) -> Result<bool> {
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -1456,7 +1472,11 @@ impl ErasureSetHealer {
             "Erasure set bucket started"
         );
 
-        let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
+        let incarnation = if self.replacement_task_id.is_some() {
+            Some(resume_manager.replacement_bucket_incarnation(bucket).await?)
+        } else {
+            self.admin_bucket_incarnation(checkpoint_manager, bucket).await?
+        };
         let unresolved_delete_recovery = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
         if unresolved_delete_recovery && !self.heal_opts.dry_run {
             resume_manager.set_delete_recovery_rescan_pending(true).await?;
@@ -1491,17 +1511,6 @@ impl ErasureSetHealer {
         // detects a non-advancing cursor even though next_marker embeds a
         // transient cache id that changes between identical pages.
         let mut previous_page_last: Option<(String, Option<String>)> = None;
-        let page_concurrency_limit =
-            if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
-                // Versions of the same object must not race without its namespace lock.
-                1
-            } else {
-                Self::effective_heal_page_object_concurrency_for_source(
-                    self.source,
-                    self.heal_opts.scan_mode,
-                    self.replacement_task_id.is_some(),
-                )
-            };
         let in_flight = Arc::new(AtomicUsize::new(0));
         // Per-bucket sample caps for per-object warn! lines: a flapping rebuild
         // disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1523,9 +1532,7 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
-            if let Some(pacer) = &self.mainline_pacer {
-                pacer.wait(&self.cancel_token).await?;
-            }
+            page_rate.wait_before_page(&self.cancel_token).await?;
             self.verify_replacement_identity_fence("page scan").await?;
             if let Some(expected) = incarnation {
                 self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1547,9 +1554,8 @@ impl ErasureSetHealer {
                     .await?
             };
             let page_is_empty = objects.is_empty();
-            let checkpoint = checkpoint_manager.get_checkpoint().await;
             let page_resume_index = *current_object_index;
-            let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
+            let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
             let mut page_tasks = FuturesUnordered::new();
             let mut completed_in_page = 0usize;
 
@@ -1563,10 +1569,7 @@ impl ErasureSetHealer {
 
                 // Per-version dedup identity — the single canonical key.
                 let key = compose_key(&item.name, item.version_id.as_deref());
-                if checkpoint.processed_objects.contains(&key)
-                    || checkpoint.failed_objects.contains(&key)
-                    || checkpoint.skipped_objects.contains(&key)
-                {
+                if checkpoint_manager.is_object_settled(&key).await {
                     continue;
                 }
 
@@ -1724,7 +1727,7 @@ impl ErasureSetHealer {
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
                 let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
-                let mainline_pacer = self.mainline_pacer.clone();
+                let page_rate = page_rate.clone();
 
                 let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
                 let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1736,14 +1739,15 @@ impl ErasureSetHealer {
                     .map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
                 let work = async move {
                     let mut retries = 0;
+                    let mut saw_slow_down = false;
                     let mut admin_identity = admin_identity;
                     let mut admin_task_id = admin_task_id;
                     loop {
-                        let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
+                        let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
 
                         let permit = match permit {
                             Ok(permit) => permit,
-                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
+                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
                         };
 
                         let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1751,7 +1755,8 @@ impl ErasureSetHealer {
                         if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
                             let (result, canonical) =
                                 admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
-                            return (dedup_key, object_name, version_id, result, canonical);
+                            let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
+                            return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
                         }
 
                         // Always go through heal_object. Genuine absence flows through
@@ -1763,15 +1768,20 @@ impl ErasureSetHealer {
                             (0, Err(Error::TaskCancelled))
                         } else {
                             let result = if replacement_object {
-                                storage
-                                    .heal_object_with_task_cancel(
-                                        &bucket_name,
-                                        &object_name,
-                                        version_id.as_deref(),
-                                        &heal_opts,
-                                        &cancel_token,
-                                    )
-                                    .await
+                                match incarnation {
+                                    Some(expected) => storage
+                                        .heal_replacement_object_at_incarnation(
+                                            &bucket_name,
+                                            &object_name,
+                                            version_id.as_deref(),
+                                            expected,
+                                            &heal_opts,
+                                            &cancel_token,
+                                        )
+                                        .await
+                                        .map(|result| (result.item, result.error)),
+                                    None => Err(Error::ReplacementNamespaceChanged),
+                                }
                             } else {
                                 storage
                                     .heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
@@ -1853,18 +1863,23 @@ impl ErasureSetHealer {
 
                         drop(in_flight_guard);
                         drop(permit);
+                        if matches!(&result.1, Err(err) if err.is_slow_down()) {
+                            saw_slow_down = true;
+                        }
                         if retry_replacement
                             && retries < REPLACEMENT_OBJECT_MAX_RETRIES
                             && matches!(&result.1, Err(Error::TransientSkip { .. }))
                         {
                             retries += 1;
                             tokio::select! {
-                                _ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
+                                _ = cancel_token.cancelled() => {
+                                    return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
+                                }
                                 _ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
                             }
                             continue;
                         }
-                        return (dedup_key, object_name, version_id, result, None);
+                        return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
                     }
                 };
                 page_tasks.push(async move {
@@ -1875,7 +1890,7 @@ impl ErasureSetHealer {
                         };
                         match result {
                             Ok(result) => result,
-                            Err(error) => (key, object, version, (0, Err(error)), None),
+                            Err(error) => (key, object, version, (0, Err(error), false), None),
                         }
                     } else {
                         work.await
@@ -1884,7 +1899,16 @@ impl ErasureSetHealer {
             }
 
             while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
-                let (object_size, result) = result;
+                let (object_size, result, saw_slow_down) = result;
+                // Exactly one rate observation per finished object, including
+                // SlowDown that later succeeded on a replacement retry.
+                page_rate.observe_object_outcome(
+                    &match &result {
+                        Ok(_) => Ok(true),
+                        Err(err) => Err(err),
+                    },
+                    saw_slow_down,
+                );
                 let mut telemetry_unknown = false;
                 let checkpoint_outcome = match result {
                     Ok(true) => {
@@ -2482,6 +2506,7 @@ mod resume_loop_tests {
     use super::{
         ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
     };
+    use crate::heal::pacing::HealPageRate;
     use crate::heal::progress::HealProgress;
     use crate::heal::resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -2642,6 +2667,7 @@ mod resume_loop_tests {
     struct FakeStorage {
         admin_disk: Mutex<Option<DiskStore>>,
         bucket_discovery_incomplete: AtomicBool,
+        bucket_discovery_error: Mutex<Option<EcstoreError>>,
         bucket_discovery_scopes: Mutex<Vec<Option<crate::heal::storage_api::storage::DiskSetSelector>>>,
         bucket_pages: Mutex<HashMap<String, Page>>,
         incarnations: Mutex<HashMap<String, uuid::Uuid>>,
@@ -2733,6 +2759,9 @@ mod resume_loop_tests {
             if self.bucket_discovery_incomplete.load(Ordering::SeqCst) {
                 return Err(Error::BucketDiscoveryIncomplete);
             }
+            if let Some(error) = self.bucket_discovery_error.lock().expect("discovery failure").clone() {
+                return Err(Error::Storage(error));
+            }
             self.list_buckets().await
         }
 
@@ -2909,6 +2938,16 @@ mod resume_loop_tests {
             }
             Ok(())
         }
+        async fn heal_replacement_bucket_metadata_at_incarnation(
+            &self,
+            bucket: &str,
+            expected: uuid::Uuid,
+            opts: &HealOpts,
+            targets: &[String],
+        ) -> Result<()> {
+            self.validate_bucket_incarnation(bucket, Some(expected)).await?;
+            self.heal_replacement_bucket_metadata(bucket, opts, targets).await
+        }
         async fn replacement_targets_have_version(
             &self,
             _bucket: &str,
@@ -3025,6 +3064,16 @@ mod resume_loop_tests {
         let temp = TempDir::new().unwrap();
         let disk = make_disk(&temp).await;
         let storage = Arc::new(FakeStorage::default());
+        if !target_endpoints.is_empty() {
+            storage.bucket_pages.lock().expect("replacement inventory").insert(
+                "b".to_string(),
+                Page {
+                    items: Vec::new(),
+                    next: None,
+                    truncated: false,
+                },
+            );
+        }
         let task_id = ResumeUtils::generate_task_id();
         let healer = ErasureSetHealer::new(
             storage.clone(),
@@ -3044,7 +3093,17 @@ mod resume_loop_tests {
         )
         .await
         .unwrap();
-        let checkpoint = CheckpointManager::new(disk, task_id.clone()).await.unwrap();
+        if !healer.target_endpoints.is_empty() {
+            resume
+                .prepare_replacement_bucket_pass(storage.as_ref())
+                .await
+                .expect("replacement fixture identity admission");
+        }
+        let checkpoint = if CheckpointManager::has_checkpoint(&disk, &task_id).await {
+            CheckpointManager::load_from_disk(disk, &task_id).await.unwrap()
+        } else {
+            CheckpointManager::new(disk, task_id.clone()).await.unwrap()
+        };
         Env {
             healer,
             storage,
@@ -3063,6 +3122,7 @@ mod resume_loop_tests {
         let mut successful = 0u64;
         let mut failed = 0u64;
         let mut skipped = 0u64;
+        let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
         let result = env
             .healer
             .heal_bucket_with_resume(
@@ -3077,11 +3137,66 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 state.start_time,
+                &page_rate,
             )
             .await;
         (processed, successful, failed, skipped, result.map(|_| ()))
     }
 
+    #[tokio::test]
+    async fn replacement_bucket_pass_rejects_incomplete_inventory_without_reusing_or_clearing_progress() {
+        let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
+        env.resume
+            .set_resume_cursor(Some("held-cursor".to_string()))
+            .await
+            .expect("cursor before discovery failure");
+        env.checkpoint
+            .add_processed_object("held-key".to_string())
+            .await
+            .expect("dedup before discovery failure");
+        // Model a page whose dedup identities must survive until cursor
+        // publication. complete_page deliberately prunes that identity set.
+        env.checkpoint
+            .advance_page(0, 7)
+            .await
+            .expect("persist the retained page identities");
+        let before = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .expect("durable checkpoint before discovery failure")
+            .get_checkpoint()
+            .await;
+        assert_eq!(before.current_object_index, 7);
+        assert!(before.processed_objects.contains("held-key"));
+        let state = env.resume.get_state().await;
+        env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
+        assert!(matches!(
+            env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
+            Err(Error::BucketDiscoveryIncomplete)
+        ));
+        assert_eq!(env.resume.get_state().await, state, "partial discovery must never publish a fresh plan");
+        let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+            .await
+            .expect("durable checkpoint")
+            .get_checkpoint()
+            .await;
+        assert_eq!(checkpoint.current_object_index, 7);
+        assert!(checkpoint.processed_objects.contains("held-key"));
+        env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst);
+        *env.storage.bucket_discovery_error.lock().expect("inject quorum failure") =
+            Some(EcstoreError::InsufficientReadQuorum("bucket".to_string(), "inventory".to_string()));
+        assert!(matches!(
+            env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
+            Err(Error::Storage(EcstoreError::InsufficientReadQuorum(..)))
+        ));
+        assert_eq!(env.resume.get_state().await, state, "quorum failure is not an empty namespace");
+        *env.storage.bucket_discovery_error.lock().expect("restore inventory") = None;
+        env.resume
+            .prepare_replacement_bucket_pass(env.storage.as_ref())
+            .await
+            .expect("unchanged complete inventory resumes existing pass");
+        assert_eq!(env.resume.get_state().await.resume_cursor.as_deref(), Some("held-cursor"));
+    }
+
     #[tokio::test]
     async fn test_empty_bucket_no_panic() {
         let env = make_env().await;
@@ -3144,6 +3259,7 @@ mod resume_loop_tests {
         let mut skipped = 0;
         let started_at = env.resume.get_state().await.start_time;
 
+        let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
         let error = healer
             .heal_bucket_with_resume(
                 "b",
@@ -3157,6 +3273,7 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 started_at,
+                &page_rate,
             )
             .await
             .expect_err("a remounted target must not begin a new page scan");
@@ -4055,7 +4172,7 @@ mod resume_loop_tests {
                 (rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
             ],
             async {
-                let mut env = make_env().await;
+                let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
                 env.healer.source = HealRequestSource::AutoHeal;
                 env.healer = env
                     .healer
@@ -4110,7 +4227,7 @@ mod resume_loop_tests {
                 (rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
             ],
             async {
-                let mut env = make_env().await;
+                let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
                 env.healer.source = HealRequestSource::AutoHeal;
                 env.healer.heal_opts.no_lock = true;
                 env.healer = env
@@ -4167,7 +4284,7 @@ mod resume_loop_tests {
 
     #[tokio::test]
     async fn auto_replacement_retries_a_transient_object_before_rewinding_the_scan() {
-        let mut env = make_env().await;
+        let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
         env.healer.source = HealRequestSource::AutoHeal;
         env.healer = env
             .healer
@@ -4199,7 +4316,7 @@ mod resume_loop_tests {
                 (rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("1")),
             ],
             async {
-                let mut env = make_env().await;
+                let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
                 env.healer.source = HealRequestSource::AutoHeal;
                 env.healer = env
                     .healer
@@ -4231,7 +4348,7 @@ mod resume_loop_tests {
 
     #[tokio::test]
     async fn auto_replacement_cancellation_interrupts_object_retry() {
-        let mut env = make_env().await;
+        let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
         env.healer.source = HealRequestSource::AutoHeal;
         env.healer = env
             .healer
@@ -4247,14 +4364,18 @@ mod resume_loop_tests {
         env.storage.set_outcome("retry", None, HealOutcome::Transient);
         let cancel = env.healer.cancel_token.clone();
         let observe = async {
-            loop {
-                let notified = env.storage.heal_started.notified();
-                if !env.storage.calls().is_empty() {
-                    break;
+            let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
+                loop {
+                    let notified = env.storage.heal_started.notified();
+                    if !env.storage.calls().is_empty() {
+                        break;
+                    }
+                    notified.await;
                 }
-                notified.await;
-            }
+            })
+            .await;
             cancel.cancel();
+            started.expect("replacement object retry must start");
         };
 
         let ((_, successful, _, _, result), ()) = tokio::join!(run(&env), observe);
@@ -4272,7 +4393,7 @@ mod resume_loop_tests {
                 (rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
             ],
             async {
-                let mut env = make_env().await;
+                let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
                 env.healer.source = HealRequestSource::AutoHeal;
                 let execution = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
                 env.healer = env

crates/heal/src/heal/erasure_healer/resume_loop_tests/dangling_grace.rs

@@ -51,6 +51,25 @@ async fn replacement_env() -> Env {
     )
     .await
     .expect("replacement intent should persist before scanning");
+    env.resume
+        .prepare_replacement_bucket_pass(env.storage.as_ref())
+        .await
+        .expect("the new replacement generation must admit its bucket identities before direct execution");
+    // Admission durably resets the companion checkpoint. Reload its current
+    // revision instead of retaining the previous generation's CAS instance.
+    env.checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
+        .await
+        .expect("reload the admitted replacement pass checkpoint");
+    assert_eq!(
+        env.resume
+            .replacement_bucket_incarnation("b")
+            .await
+            .expect("the fixture bucket must belong to the replacement pass"),
+        env.storage
+            .admit_bucket_incarnation("b")
+            .await
+            .expect("fixture bucket identity")
+    );
     env.healer = ErasureSetHealer::new(
         env.storage.clone(),
         Arc::new(RwLock::new(HealProgress::new())),

crates/heal/src/heal/manager.rs

@@ -810,7 +810,14 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
 fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
     active_heals
         .iter()
-        .find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
+        .find(|(_, task)| {
+            PriorityHealQueue::make_dedup_key_for_owner(
+                &task.heal_type,
+                &task.options,
+                task.source,
+                task.expected_mrf_bucket_incarnation_id,
+            ) == key
+        })
         .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
 }
 

crates/heal/src/heal/manager/queue.rs

@@ -611,7 +611,104 @@ impl PriorityHealQueue {
 
     /// Create a deduplication key from a heal request
     pub(super) fn make_dedup_key(request: &HealRequest) -> String {
-        Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
+        Self::make_dedup_key_for_owner(
+            &request.heal_type,
+            &request.options,
+            request.source,
+            request.expected_mrf_bucket_incarnation_id,
+        )
+    }
+
+    /// Bound MRF work can only merge with work for the same bucket lifetime.
+    /// Length-prefix user-controlled components so a key cannot forge another
+    /// version, scope, or incarnation through delimiter characters.
+    pub(super) fn make_dedup_key_for_owner(
+        heal_type: &HealType,
+        options: &HealOptions,
+        source: HealRequestSource,
+        incarnation: Option<Uuid>,
+    ) -> String {
+        if source == HealRequestSource::Mrf
+            && let Some(incarnation) = incarnation
+        {
+            use std::fmt::Write as _;
+
+            let (pool, set) = heal_scope_indices(heal_type, options);
+            let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
+            let mut part = |value: &str| {
+                // Formatting into a String is infallible.
+                let _ = write!(key, "{}:{value}", value.len());
+            };
+            match heal_type {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("object");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::Metadata { bucket, object } => {
+                    part("metadata");
+                    part(bucket);
+                    part(object);
+                }
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("ecdecode");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::DeleteMarkerPurge {
+                    bucket,
+                    object,
+                    version_id,
+                    purge,
+                } => {
+                    part("delete-marker-purge");
+                    part(bucket);
+                    part(object);
+                    part(version_id);
+                    part(&purge.bucket_incarnation_id.to_string());
+                    part(&purge.marker_incarnation_id.to_string());
+                    part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
+                }
+                HealType::Bucket { bucket } => {
+                    part("bucket");
+                    part(bucket);
+                }
+                HealType::Prefix { bucket, prefix } => {
+                    part("prefix");
+                    part(bucket);
+                    part(prefix);
+                }
+                // Cluster and erasure-set work are not bucket-bound; preserve
+                // their existing keys, including contains_erasure_set lookups.
+                HealType::Cluster | HealType::ErasureSet { .. } => {
+                    return Self::make_dedup_key_for_scope(heal_type, options);
+                }
+            }
+            return key;
+        }
+        Self::make_dedup_key_for_scope(heal_type, options)
     }
 
     pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {

crates/heal/src/heal/manager/scheduler.rs

@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
     request: &HealRequest,
     config: &HealConfig,
     provider: &Option<WorkloadSnapshotProviderRef>,
-) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
+) -> Option<crate::heal::pacing::ForegroundPacing> {
     let pace_running = request.source == HealRequestSource::Admin
         || (request.source == HealRequestSource::AutoHeal
             && matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,14 +43,13 @@ pub(super) fn running_mainline_pacer(
         return None;
     }
     provider.as_ref().and_then(|provider| {
-        crate::heal::pacing::MainlinePacer::new(
+        crate::heal::pacing::ForegroundPacing::new(
             provider.clone(),
             config.mainline_read_utilization_high_percent,
             config.mainline_write_utilization_high_percent,
             config.mainline_max_sleep,
             request.source,
         )
-        .map(Arc::new)
     })
 }
 

crates/heal/src/heal/manager/tests.rs

@@ -117,6 +117,453 @@ async fn process_manager_queue_once(manager: &HealManager) {
 
 struct MockStorage;
 
+fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
+    let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
+    request.source = HealRequestSource::Mrf;
+    request.expected_mrf_bucket_incarnation_id = Some(incarnation);
+    request.options.pool_index = Some(0);
+    request.options.set_index = Some(0);
+    request
+}
+
+async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
+    let old_incarnation = Uuid::new_v4();
+    let new_incarnation = Uuid::new_v4();
+    let old_request = incarnation_bound_mrf_fallback(old_incarnation);
+    let old_id = old_request.id.clone();
+    match existing_state {
+        "queued" => {
+            let receipt = manager
+                .submit_heal_request_with_receipt(old_request)
+                .await
+                .expect("old fallback must be admitted");
+            assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+        }
+        "active" => {
+            manager
+                .active_heals
+                .lock()
+                .await
+                .insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
+        }
+        "retrying" => {
+            manager.retrying_heals.lock().await.insert(
+                old_id.clone(),
+                RetryingHeal {
+                    request: old_request,
+                    error: "retry pending".to_owned(),
+                    cancel_token: CancellationToken::new(),
+                },
+            );
+        }
+        other => panic!("unknown fixture state: {other}"),
+    }
+
+    let new_request = incarnation_bound_mrf_fallback(new_incarnation);
+    let new_id = new_request.id.clone();
+    assert_ne!(new_id, old_id, "the fallback must use a new request ID");
+    let receipt = manager
+        .submit_heal_request_with_receipt(new_request)
+        .await
+        .expect("new incarnation admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
+    );
+    assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
+    let queue = manager.heal_queue.lock().await;
+    assert!(
+        queue
+            .requests()
+            .any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
+    );
+    if existing_state == "queued" {
+        assert!(
+            queue
+                .requests()
+                .any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
+        );
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("active").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
+}
+
+fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
+    let mut request = incarnation_bound_mrf_fallback(incarnation);
+    if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
+        *id = version_id.map(str::to_owned);
+    } else {
+        unreachable!("fallback fixture must be an object request");
+    }
+    request
+}
+
+#[tokio::test]
+async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
+    for version in [Uuid::nil(), Uuid::new_v4()] {
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+        let incarnation = Uuid::new_v4();
+        let version_id = version.to_string();
+        let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
+        let original_id = original.id.clone();
+        let accepted = manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first queued fallback");
+        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
+        let merged = manager
+            .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
+            .await
+            .expect("duplicate queued fallback");
+        assert_eq!(merged.result, HealAdmissionResult::Merged);
+        assert_eq!(merged.task_id, original_id);
+        assert_eq!(manager.heal_queue.lock().await.len(), 1);
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let original = incarnation_bound_mrf_fallback(incarnation);
+    let original_id = original.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let same = manager
+        .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
+        .await
+        .expect("same-incarnation fallback");
+    assert_eq!(same.result, HealAdmissionResult::Merged);
+    assert_eq!(same.task_id, original_id);
+
+    let mut versioned = incarnation_bound_mrf_fallback(incarnation);
+    versioned.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "object".to_owned(),
+        version_id: Some(Uuid::new_v4().to_string()),
+    };
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(versioned)
+            .await
+            .expect("different version")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
+    other_scope.options.pool_index = Some(1);
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(other_scope)
+            .await
+            .expect("different pool")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    same_id_other_incarnation.id = original_id;
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(same_id_other_incarnation)
+            .await
+            .expect("same-ID conflict must be decided")
+            .result,
+        HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
+    );
+}
+
+#[test]
+fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
+    let incarnation = Uuid::new_v4();
+    let bound = incarnation_bound_mrf_fallback(incarnation);
+    let mut unbound = bound.clone();
+    unbound.expected_mrf_bucket_incarnation_id = None;
+    let mut non_mrf = bound.clone();
+    non_mrf.source = HealRequestSource::ReadRepair;
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key(&non_mrf),
+        PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
+        "an unrelated source retains its previous scope key even if the MRF-only field is set"
+    );
+    assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
+
+    let mut left = bound.clone();
+    left.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part:a".to_owned(),
+        version_id: Some("b".to_owned()),
+    };
+    let mut right = bound.clone();
+    right.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part".to_owned(),
+        version_id: Some("a:b".to_owned()),
+    };
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
+        PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
+        "the legacy delimiter-only key demonstrates the collision"
+    );
+    assert_ne!(
+        PriorityHealQueue::make_dedup_key(&left),
+        PriorityHealQueue::make_dedup_key(&right),
+        "length-prefixed object and version components must not alias"
+    );
+    let mut forged = unbound;
+    forged.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: PriorityHealQueue::make_dedup_key(&bound),
+        version_id: None,
+    };
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
+}
+
+#[test]
+fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
+    let incarnation = Uuid::new_v4();
+    for decode in [false, true] {
+        let make_request = |object: &str, version_id: Option<&str>| {
+            let mut request = incarnation_bound_mrf_fallback(incarnation);
+            let bucket = "mrf-recreated-bucket".to_owned();
+            let object = object.to_owned();
+            let version_id = version_id.map(str::to_owned);
+            request.heal_type = if decode {
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            } else {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            };
+            request
+        };
+        let absent = make_request("object", None);
+        let empty = make_request("object", Some(""));
+        let forged_object = make_request("object4:some0:", None);
+        let forged_version = make_request("object", Some("4:none"));
+        let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
+        for (index, key) in keys.iter().enumerate() {
+            assert!(keys.iter().skip(index + 1).all(|other| key != other));
+        }
+
+        let mut queue = PriorityHealQueue::new();
+        assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
+        assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
+        assert_eq!(queue.len(), 2);
+
+        for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
+            let mut legacy_absent = absent.clone();
+            let mut legacy_empty = empty.clone();
+            legacy_absent.source = source;
+            legacy_empty.source = source;
+            if source == HealRequestSource::Mrf {
+                legacy_absent.expected_mrf_bucket_incarnation_id = None;
+                legacy_empty.expected_mrf_bucket_incarnation_id = None;
+            }
+            assert_eq!(
+                PriorityHealQueue::make_dedup_key(&legacy_absent),
+                PriorityHealQueue::make_dedup_key(&legacy_empty),
+                "unbound and non-MRF work retain the legacy empty-version key"
+            );
+        }
+    }
+}
+
+#[test]
+fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
+    let mut queue = PriorityHealQueue::new();
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let old_id = old.id.clone();
+    let old_key = PriorityHealQueue::make_dedup_key(&old);
+    let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let new_id = new.id.clone();
+    let new_key = PriorityHealQueue::make_dedup_key(&new);
+    let mut forced_old = old.clone();
+    forced_old.id = Uuid::new_v4().to_string();
+    forced_old.force_start = true;
+    let forced_id = forced_old.id.clone();
+
+    assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    queue
+        .remove_request_id(&old_id)
+        .expect("remove the old incarnation representative");
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
+    assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
+    assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
+    assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
+}
+
+#[tokio::test]
+async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let incarnation = Uuid::new_v4();
+    let request = incarnation_bound_mrf_fallback(incarnation);
+    let request_id = request.id.clone();
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from("object"),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let receipt = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(request, target)
+        .await
+        .expect("durable admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a durable new incarnation needs its own owner"
+    );
+    assert_eq!(receipt.task_id, request_id);
+    assert!(
+        manager
+            .heal_queue
+            .lock()
+            .await
+            .requests()
+            .any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
+    );
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
+#[tokio::test]
+async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let object = format!("same-generation-{}", Uuid::new_v4());
+    let make_request = || {
+        let mut request = incarnation_bound_mrf_fallback(incarnation);
+        request.heal_type = HealType::Object {
+            bucket: "mrf-recreated-bucket".to_owned(),
+            object: object.clone(),
+            version_id: None,
+        };
+        request
+    };
+    let old = make_request();
+    let old_id = old.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from(object.as_str()),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let merged = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
+        .await
+        .expect("durable same-incarnation admission");
+    assert_eq!(merged.result, HealAdmissionResult::Merged);
+    assert_eq!(merged.task_id, old_id);
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
         task_id: "fixture-token".to_string(),

crates/heal/src/heal/manager/tests/auto_scan.rs

@@ -881,6 +881,13 @@ async fn auto_format_fixture(
     // Preserve the constructor's explicit open/closed state before any fixture await.
     let node_limit = node.as_ref().map(|node| node.snapshot().limit);
     let fixture = resume_admission_fixture(node.clone()).await;
+    // The endpoint monitor independently probes format.json after five seconds.
+    // Drain it before aging the cache so the path-only pause observes this scanner.
+    for pool in &fixture._store.ecstore.pools {
+        tokio::time::timeout(Duration::from_secs(30), pool.stop_endpoint_monitor_for_test())
+            .await
+            .expect("endpoint reconnection must stop before the scanner owns format probes");
+    }
     let disks = crate::heal::local_disk_map_read()
         .await
         .values()

crates/heal/src/heal/pacing.rs

@@ -7,8 +7,14 @@ use rustfs_concurrency::{
     workload::{ForegroundPressure, foreground_pressure},
 };
 use rustfs_heal_contracts::heal_channel::HealRequestSource;
-use std::{sync::Arc, time::Duration};
-use tokio::{sync::Mutex, time::Instant};
+use std::{
+    sync::{
+        Arc,
+        atomic::{AtomicU32, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{sync::Mutex as AsyncMutex, time::Instant};
 use tokio_util::sync::CancellationToken;
 
 #[derive(Default)]
@@ -22,17 +28,19 @@ pub(crate) enum PacingDecision {
     Wait(Option<ForegroundPressure>),
 }
 
-/// Cooperative pacing for one heal execution, not a storage admission permit.
-pub(crate) struct MainlinePacer {
+/// Foreground watermarks for one heal execution. Constructed once per request;
+/// disabled when watermarks are zero or no workload provider is configured.
+#[derive(Clone)]
+pub(crate) struct ForegroundPacing {
     provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
     read_high: usize,
     write_high: usize,
     pause: Duration,
     source: HealRequestSource,
-    state: Mutex<PacingState>,
+    state: Arc<AsyncMutex<PacingState>>,
 }
 
-impl MainlinePacer {
+impl ForegroundPacing {
     pub(crate) fn new(
         provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
         read_high: usize,
@@ -49,7 +57,7 @@ impl MainlinePacer {
             write_high: write_high.min(100),
             pause: pause.min(Duration::from_secs(1)),
             source,
-            state: Mutex::new(PacingState::default()),
+            state: Arc::new(AsyncMutex::new(PacingState::default())),
         })
     }
 
@@ -125,6 +133,142 @@ impl MainlinePacer {
     }
 }
 
+/// Single owner of "how hard may this heal pass push storage?".
+///
+/// One instance spans the whole erasure-set scan (all buckets and pages):
+/// - foreground admission pressure (`ForegroundPacing`)
+/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
+///
+/// Call `observe_object_outcome` exactly once per finished object so a single
+/// SlowDown cannot double-reduce concurrency.
+pub(crate) struct HealPageRate {
+    base_concurrency: usize,
+    /// 0 = full base concurrency; each SlowDown increases the reduction.
+    slow_down_step: AtomicU32,
+    slow_down_until: std::sync::Mutex<Option<Instant>>,
+    foreground: Option<ForegroundPacing>,
+}
+
+impl HealPageRate {
+    /// Always-on controller. Foreground yield is optional; SlowDown control is not.
+    pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
+        Self {
+            base_concurrency: base_concurrency.max(1),
+            slow_down_step: AtomicU32::new(0),
+            slow_down_until: std::sync::Mutex::new(None),
+            foreground,
+        }
+    }
+
+    /// Concurrency after applying the current SlowDown reduction. Always ≥ 1
+    /// so maintenance can still make progress under sustained rate pressure.
+    pub(crate) fn effective_concurrency(&self) -> usize {
+        let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
+        (self.base_concurrency >> step).max(1)
+    }
+
+    /// Fold one finished object into the rate state. Call once per outcome.
+    /// `saw_slow_down` is true when any attempt of this object hit SlowDown,
+    /// including attempts that later succeeded on retry.
+    pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
+        if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
+            self.observe_slow_down();
+            return;
+        }
+        if result.is_ok() {
+            self.observe_success();
+        }
+    }
+
+    /// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
+    fn observe_slow_down(&self) {
+        // fetch_add returns the previous step; +1 so the first SlowDown uses
+        // exponent 1 (2s), not exponent 0 (1s).
+        let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
+        let delay = Self::backoff(step);
+        let ready = Instant::now() + delay;
+        if let Ok(mut until) = self.slow_down_until.lock()
+            && until.is_none_or(|current| current < ready)
+        {
+            *until = Some(ready);
+        }
+    }
+
+    /// Restore one concurrency step only after the SlowDown cool-down expires.
+    /// A same-page success must not undo the reduction while the pass is still
+    /// in the rate-limited window (the page semaphore is fixed until the next page).
+    fn observe_success(&self) {
+        let cooling = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .is_some_and(|ready| ready > Instant::now());
+        if cooling {
+            return;
+        }
+        self.slow_down_step
+            .try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
+            .ok();
+    }
+
+    /// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
+    pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
+        self.wait_slow_down(cancel).await?;
+        if let Some(foreground) = &self.foreground {
+            foreground.wait(cancel).await?;
+        }
+        Ok(())
+    }
+
+    /// Foreground admission decision while the caller owns page capacity.
+    pub(crate) fn admission_decision(&self) -> PacingDecision {
+        match &self.foreground {
+            Some(foreground) => foreground.admission_decision(),
+            None => PacingDecision::Ready,
+        }
+    }
+
+    /// Foreground wait after a rejected admission. Returns whether a pause was paid.
+    pub(crate) async fn wait_after_admission(
+        &self,
+        cancel: &CancellationToken,
+        observed: Option<ForegroundPressure>,
+    ) -> Result<bool> {
+        match &self.foreground {
+            Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
+            None => Ok(false),
+        }
+    }
+
+    async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
+        let delay = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .map(|ready| ready.saturating_duration_since(Instant::now()));
+        let Some(delay) = delay else {
+            return Ok(());
+        };
+        if delay.is_zero() {
+            return Ok(());
+        }
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            _ = tokio::time::sleep(delay) => Ok(()),
+        }
+    }
+
+    fn backoff(step: u32) -> Duration {
+        // step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
+        // rate with exponential backoff, and heal must do the same to itself.
+        let exponent = step.clamp(1, 5);
+        Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
+    }
+}
+
 #[cfg(test)]
 pub(crate) struct TestPressure {
     pub(crate) active: std::sync::atomic::AtomicUsize,
@@ -164,12 +308,87 @@ mod tests {
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
 
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_first_slow_down_waits_two_seconds() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("cool-down wait");
+        assert_eq!(Instant::now() - before, Duration::from_secs(2));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_reduces_concurrency_and_backs_off() {
+        let rate = HealPageRate::new(8, None);
+        assert_eq!(rate.effective_concurrency(), 8);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        assert_eq!(rate.effective_concurrency(), 2);
+        // Floor is one so maintenance never stops entirely.
+        for _ in 0..8 {
+            rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        }
+        assert_eq!(rate.effective_concurrency(), 1);
+
+        let cancel = CancellationToken::new();
+        assert!(rate.wait_before_page(&cancel).await.is_ok());
+
+        // After cool-down, successes restore concurrency one step at a time.
+        for _ in 0..12 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert!(rate.effective_concurrency() > 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        // Same-page successes while cool-down is armed must not restore fan-out.
+        for _ in 0..4 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
+        let rate = HealPageRate::new(8, None);
+        // Replacement retry: SlowDown on attempt 1, success on attempt 2.
+        rate.observe_object_outcome(&Ok(true), true);
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_wait_is_cancellable() {
+        let rate = HealPageRate::new(4, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        let cancel = CancellationToken::new();
+        cancel.cancel();
+        assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
+        let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
+            .expect("foreground pacer");
+        let rate = HealPageRate::new(4, Some(foreground));
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
+        assert_eq!(Instant::now() - before, Duration::from_millis(250));
+    }
+
     #[tokio::test(start_paused = true)]
     async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
         for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
             let provider = Arc::new(TestPressure::new(class, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(
+            let pacer = ForegroundPacing::new(
                 provider.clone(),
                 if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
                 if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
@@ -197,9 +416,9 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
-        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
-        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
-        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
+        assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
+        assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
+        let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
         let before = Instant::now();
         pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
         assert_eq!(Instant::now() - before, Duration::from_secs(1));
@@ -209,7 +428,7 @@ mod tests {
     async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
         let paced = Arc::new(
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
         );
         let cancel_first = CancellationToken::new();
         let first = tokio::spawn({
@@ -228,7 +447,7 @@ mod tests {
         cancel_second.cancel();
         assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
         provider.active.store(0, Ordering::SeqCst);
-        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
+        let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
             .expect("independent task");
         let before = Instant::now();
         other_task

crates/heal/src/heal/resume.rs

@@ -27,6 +27,7 @@ use super::{
     BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskBytes,
 };
 
+mod bucket_pass;
 mod checkpoint;
 mod gc;
 mod handoff;
@@ -35,7 +36,9 @@ mod replacement;
 mod utils;
 
 pub(crate) use checkpoint::AdminErasureCheckpoint;
-pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
+pub use checkpoint::{
+    CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
+};
 pub(crate) use gc::ResumeGc;
 pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
 pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};
@@ -343,6 +346,16 @@ pub struct ResumeState {
     /// input after restart.
     #[serde(default)]
     pub replacement_buckets: Vec<String>,
+    /// Current namespace pass, separate from the immutable generation plan.
+    /// Legacy snapshots have no binding and must discard every old cursor.
+    #[serde(default)]
+    pub replacement_bucket_incarnations: Option<std::collections::BTreeMap<String, Uuid>>,
+    /// Last proven retired identity per name, retained for returning stale volumes.
+    #[serde(default)]
+    pub replacement_retired_bucket_incarnations: std::collections::BTreeMap<String, Uuid>,
+    /// Reapply the checkpoint rewind after any interruption between the two writes.
+    #[serde(default)]
+    pub replacement_bucket_pass_reset_pending: bool,
     /// A task-scoped generation assigned before formatting an automatic
     /// replacement. A new unformatted replacement receives a new generation
     /// and therefore cannot reuse an older disk's cursor or checkpoint.
@@ -452,6 +465,9 @@ impl ResumeState {
             replacement_targets: Vec::new(),
             replacement_target_identities: Vec::new(),
             replacement_buckets: Vec::new(),
+            replacement_bucket_incarnations: None,
+            replacement_retired_bucket_incarnations: std::collections::BTreeMap::new(),
+            replacement_bucket_pass_reset_pending: false,
             replacement_generation: None,
             replacement_phase: ReplacementPhase::None,
             replacement_execution_protocol: 0,
@@ -983,7 +999,16 @@ impl ResumeManager {
                     return Err(replacement_recovery_conflict("invalid retired legacy generation"));
                 }
             }
-            if state.replacement_execution_protocol != 1
+            let valid_execution_protocol = match state.replacement_execution_protocol {
+                1 => state.replacement_bucket_incarnations.is_none(),
+                2 => state
+                    .replacement_bucket_incarnations
+                    .as_ref()
+                    .is_some_and(|identities| identities.values().all(|incarnation| !incarnation.is_nil())),
+                _ => false,
+            };
+            if !valid_execution_protocol
+                || state.replacement_retired_bucket_incarnations.values().any(Uuid::is_nil)
                 || state.replacement_predecessor.as_deref()
                     != state.replacement_lineage.last().map(|link| link.predecessor.as_str())
             {

crates/heal/src/heal/resume/bucket_pass.rs

@@ -0,0 +1,411 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::heal::storage::HealStorageAPI;
+use crate::heal::storage_api::storage::DiskSetSelector;
+use std::collections::BTreeMap;
+
+async fn namespace_snapshot(storage: &dyn HealStorageAPI, state: &ResumeState) -> Result<BTreeMap<String, Uuid>> {
+    let (pool, set) = crate::heal::utils::parse_set_disk_id(&state.set_disk_id)?;
+    // This API rejects partial topology and propagates read-quorum failures.
+    let buckets = storage
+        .list_buckets_for_admin_heal(Some(DiskSetSelector::new(pool, set)))
+        .await?;
+    let mut identities = BTreeMap::new();
+    for bucket in buckets {
+        let Some(incarnation) = storage.replacement_bucket_incarnation(&bucket.name).await? else {
+            // A returning disk may still list a retired volume. Only the admitted
+            // old UUID plus its durable receipt can discharge that obligation.
+            if let Some(expected) = state
+                .replacement_bucket_incarnations
+                .as_ref()
+                .and_then(|map| map.get(&bucket.name))
+                .or_else(|| state.replacement_retired_bucket_incarnations.get(&bucket.name))
+                && storage.mrf_bucket_incarnation_retired(&bucket.name, *expected).await?
+            {
+                continue;
+            }
+            return Err(Error::ReplacementNamespaceChanged);
+        };
+        if incarnation.is_nil() {
+            return Err(Error::InvalidBucketIncarnation);
+        }
+        identities.insert(bucket.name, incarnation);
+    }
+    Ok(identities)
+}
+
+impl ResumeManager {
+    pub(crate) async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Uuid> {
+        self.state
+            .read()
+            .await
+            .replacement_bucket_incarnations
+            .as_ref()
+            .and_then(|identities| identities.get(bucket).copied())
+            .ok_or(Error::ReplacementNamespaceChanged)
+    }
+
+    pub(crate) async fn prepare_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<Vec<String>> {
+        let previous = self.get_state().await;
+        let current = namespace_snapshot(storage, &previous).await?;
+        if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
+            for (bucket, expected) in identities {
+                if current.get(bucket) != Some(expected) && !storage.mrf_bucket_incarnation_retired(bucket, *expected).await? {
+                    // Absence or a different current identity alone is not a deletion receipt.
+                    return Err(Error::ReplacementNamespaceChanged);
+                }
+            }
+        }
+        if previous.replacement_bucket_incarnations.as_ref() != Some(&current) {
+            let mut state = self.state.write().await;
+            if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
+                for (bucket, expected) in identities {
+                    if current.get(bucket) != Some(expected) {
+                        state
+                            .replacement_retired_bucket_incarnations
+                            .insert(bucket.clone(), *expected);
+                    }
+                }
+            }
+            state.replacement_bucket_incarnations = Some(current.clone());
+            if state.replacement_generation.is_some() {
+                // Protocol-1 readers reject this before interpreting a new pass cursor.
+                state.replacement_execution_protocol = 2;
+            }
+            state.replacement_bucket_pass_reset_pending = true;
+            state.reset_for_retry();
+            state.pending_buckets = current.keys().cloned().collect();
+            state.total_objects = 0;
+            state.total_bytes = 0;
+            state.baseline_generation = None;
+            state.baseline_known = false;
+            drop(state);
+            self.save_state_strict().await?;
+        }
+        if self.get_state().await.replacement_bucket_pass_reset_pending {
+            let checkpoint = if CheckpointManager::has_checkpoint(&self.disk, &previous.task_id).await {
+                CheckpointManager::load_from_disk(self.disk.clone(), &previous.task_id).await?
+            } else {
+                CheckpointManager::new(self.disk.clone(), previous.task_id.clone()).await?
+            };
+            // Unlike the throttled retry writer this is durable before clearing the intent.
+            checkpoint.reset_after_delete_recovery().await?;
+            self.state.write().await.replacement_bucket_pass_reset_pending = false;
+            self.save_state_strict().await?;
+        }
+        Ok(current.into_keys().collect())
+    }
+
+    pub(crate) async fn verify_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<()> {
+        let state = self.get_state().await;
+        let current = namespace_snapshot(storage, &state).await?;
+        if state.replacement_bucket_incarnations.as_ref() != Some(&current) {
+            return Err(Error::ReplacementNamespaceChanged);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
+    use crate::heal::storage_api::{
+        storage::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _},
+        test_support::{
+            ObjectOptions, PutObjReader, delete_config, init_bucket_metadata_sys, isolated_store_over_temp_disks, read_config,
+            save_config,
+        },
+    };
+    use rustfs_heal_contracts::heal_channel::HealOpts;
+    use tokio_util::sync::CancellationToken;
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn replacement_bucket_pass_retires_deleted_identity_and_restarts_recreated_bucket() {
+        // Keep the concrete storage futures off the test executor stack.
+        let (directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "replacement-pass-bucket";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create original generation");
+        let old = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("original identity");
+        let storage = ECStoreHealStorage::new(store.clone());
+        let (_anchor_dir, disk) = super::super::tests::schema_test_disk().await;
+        let task_id = ResumeUtils::generate_task_id();
+        let manager = ResumeManager::new_replacement_intent(
+            disk.clone(),
+            task_id.clone(),
+            "pool_0_set_0".to_string(),
+            vec![bucket.to_owned()],
+            vec!["replacement-a".to_string()],
+            vec![ReplacementTargetIdentity {
+                endpoint: "replacement-a".to_string(),
+                canonical_path: "/mnt/replacement-a".to_string(),
+                physical_device_ids: vec!["device-a".to_string()],
+                filesystem_identity: "1:2:3".to_string(),
+            }],
+        )
+        .await
+        .expect("persist generation");
+        // A legacy name-only cursor cannot be attributed to the currently live bucket.
+        manager
+            .set_resume_cursor(Some("unbound-legacy-cursor".to_string()))
+            .await
+            .expect("legacy cursor");
+        let legacy_checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
+            .await
+            .expect("legacy checkpoint");
+        legacy_checkpoint
+            .add_processed_object("legacy-key".to_string())
+            .await
+            .expect("legacy dedup");
+        legacy_checkpoint.complete_page(0, 99).await.expect("persist legacy cursor");
+        assert_eq!(
+            manager
+                .prepare_replacement_bucket_pass(&storage)
+                .await
+                .expect("admit original pass"),
+            [bucket]
+        );
+        assert_eq!(
+            manager
+                .get_state()
+                .await
+                .replacement_bucket_incarnations
+                .as_ref()
+                .and_then(|m| m.get(bucket))
+                .copied(),
+            Some(old)
+        );
+        assert!(
+            manager.get_state().await.resume_cursor.is_none(),
+            "legacy cursor must not enter the fresh pass"
+        );
+        let fresh_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+            .await
+            .expect("fresh checkpoint")
+            .get_checkpoint()
+            .await;
+        assert_eq!(fresh_checkpoint.current_object_index, 0);
+        assert!(fresh_checkpoint.processed_objects.is_empty());
+        manager
+            .set_resume_cursor(Some("old-generation-cursor".to_string()))
+            .await
+            .expect("persist old cursor");
+        let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+            .await
+            .expect("checkpoint");
+        checkpoint
+            .add_processed_object("same-key".to_string())
+            .await
+            .expect("old dedup entry");
+        checkpoint.complete_page(0, 7).await.expect("persist old position and dedup");
+
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete original generation");
+        assert!(matches!(
+            manager.verify_replacement_bucket_pass(&storage).await,
+            Err(Error::ReplacementNamespaceChanged)
+        ));
+        manager
+            .record_replacement_failure(&Error::ReplacementNamespaceChanged, 99)
+            .await
+            .expect("defer without exhausting replacement");
+        assert_eq!(manager.get_state().await.retry_count, 0);
+        let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
+            .await
+            .expect("restart replacement manager");
+        let retirement_path = format!("bucket-retirements/{bucket}/{old}.json");
+        let retirement = Box::pin(read_config(store.clone(), &retirement_path))
+            .await
+            .expect("authoritative retirement");
+        Box::pin(delete_config(store.clone(), &retirement_path))
+            .await
+            .expect("simulate missing retirement evidence");
+        assert!(
+            matches!(
+                resumed.prepare_replacement_bucket_pass(&storage).await,
+                Err(Error::ReplacementNamespaceChanged)
+            ),
+            "physical absence alone cannot retire an admitted identity"
+        );
+        Box::pin(save_config(store.clone(), &retirement_path, b"corrupt retirement".to_vec()))
+            .await
+            .expect("inject unreadable proof");
+        assert!(
+            matches!(resumed.prepare_replacement_bucket_pass(&storage).await, Err(Error::Storage(_))),
+            "a corrupt proof cannot be treated as an absent bucket"
+        );
+        assert_eq!(resumed.get_state().await.resume_cursor.as_deref(), Some("old-generation-cursor"));
+        Box::pin(save_config(store.clone(), &retirement_path, retirement))
+            .await
+            .expect("restore proof");
+        assert!(
+            resumed
+                .prepare_replacement_bucket_pass(&storage)
+                .await
+                .expect("retirement proves deleted plan item")
+                .is_empty()
+        );
+
+        // A disk returning after DELETE may expose the retired physical volume
+        // again; the persisted receipt identity still discharges that old name.
+        tokio::fs::create_dir(directories[0].path().join(bucket))
+            .await
+            .expect("return stale retired volume");
+        let returned = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
+            .await
+            .expect("restart after retirement");
+        assert!(
+            returned
+                .prepare_replacement_bucket_pass(&storage)
+                .await
+                .expect("retired stale volume is not a new bucket")
+                .is_empty()
+        );
+        tokio::fs::remove_dir(directories[0].path().join(bucket))
+            .await
+            .expect("remove stale test volume");
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate same name");
+        let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
+        assert_ne!(new, old);
+        Box::pin(store.put_object(
+            bucket,
+            "same-key",
+            &mut PutObjReader::from_vec(b"new payload".to_vec()),
+            &ObjectOptions::default(),
+        ))
+        .await
+        .expect("new generation payload");
+        assert_eq!(
+            resumed
+                .prepare_replacement_bucket_pass(&storage)
+                .await
+                .expect("new generation gets independent pass"),
+            [bucket]
+        );
+        let state = resumed.get_state().await;
+        assert_eq!(
+            state.replacement_execution_protocol, 2,
+            "older binaries must reject identity-bound cursors"
+        );
+        assert_eq!(
+            state
+                .replacement_bucket_incarnations
+                .as_ref()
+                .and_then(|m| m.get(bucket))
+                .copied(),
+            Some(new)
+        );
+        assert!(state.resume_cursor.is_none());
+        let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+            .await
+            .expect("restarted checkpoint")
+            .get_checkpoint()
+            .await;
+        assert_eq!((checkpoint.current_bucket_index, checkpoint.current_object_index), (0, 0));
+        assert!(checkpoint.processed_objects.is_empty());
+
+        let opts = HealOpts {
+            pool: Some(0),
+            set: Some(0),
+            ..Default::default()
+        };
+        assert!(
+            storage
+                .heal_replacement_object_at_incarnation(bucket, "same-key", None, old, &opts, &CancellationToken::new())
+                .await
+                .is_err(),
+            "a stale worker cannot mutate a recreated bucket"
+        );
+        let targets = directories
+            .iter()
+            .map(|dir| dir.path().to_string_lossy().into_owned())
+            .collect::<Vec<_>>();
+        assert!(
+            storage
+                .heal_replacement_bucket_metadata_at_incarnation(bucket, old, &opts, &targets)
+                .await
+                .is_err(),
+            "a stale metadata pass cannot certify the new incarnation"
+        );
+        storage
+            .heal_replacement_bucket_metadata_at_incarnation(bucket, new, &opts, &targets)
+            .await
+            .expect("current incarnation can repair metadata");
+        let mut reader =
+            Box::pin(store.get_object_reader(bucket, "same-key", None, Default::default(), &ObjectOptions::default()))
+                .await
+                .expect("open new generation payload");
+        assert_eq!(reader.read_all().await.expect("read new payload"), b"new payload");
+        drop(reader);
+        assert_eq!(
+            store
+                .bucket_incarnation_id_from_disk(bucket)
+                .await
+                .expect("new identity remains"),
+            new
+        );
+        // Simulate interruption after publishing the reset obligation, before
+        // replacing the old checkpoint.
+        let stale_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+            .await
+            .expect("checkpoint before interruption");
+        stale_checkpoint
+            .add_processed_object("stale-before-reset".to_string())
+            .await
+            .expect("stale dedup");
+        stale_checkpoint
+            .complete_page(0, 42)
+            .await
+            .expect("persist interrupted checkpoint");
+        resumed.state.write().await.replacement_bucket_pass_reset_pending = true;
+        resumed.save_state_strict().await.expect("persist reset obligation");
+        let interrupted = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
+            .await
+            .expect("restart interrupted pass");
+        interrupted
+            .prepare_replacement_bucket_pass(&storage)
+            .await
+            .expect("finish interrupted checkpoint rewind");
+        let repaired_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
+            .await
+            .expect("repaired checkpoint")
+            .get_checkpoint()
+            .await;
+        assert_eq!(repaired_checkpoint.current_object_index, 0);
+        assert!(repaired_checkpoint.processed_objects.is_empty());
+        assert!(!interrupted.get_state().await.replacement_bucket_pass_reset_pending);
+        let loaded = ResumeManager::load_replacement_intent(disk, &task_id)
+            .await
+            .expect("new pass survives restart");
+        loaded
+            .verify_replacement_bucket_pass(&storage)
+            .await
+            .expect("new pass matches current namespace");
+    }
+}

crates/heal/src/heal/resume/checkpoint.rs

@@ -152,6 +152,42 @@ impl AdminErasureCheckpoint {
     }
 }
 
+/// Lightweight progress view for resume decisions. Omits the per-page object
+/// ledgers so callers do not clone a million-object checkpoint just to read
+/// counters and positions.
+#[derive(Debug, Clone, Copy)]
+pub struct CheckpointProgressSnapshot {
+    pub current_bucket_index: usize,
+    pub current_object_index: usize,
+    pub successful_objects: u64,
+    pub failed_object_count: u64,
+    pub skipped_object_count: u64,
+    pub skipped_new_versions: u64,
+    pub skipped_ilm_expired: u64,
+    pub processed_bytes: u64,
+    pub total_objects: u64,
+    pub total_bytes: u64,
+    pub baseline_generation: Option<u64>,
+    pub baseline_known: bool,
+    pub counter_unknown: bool,
+}
+
+impl CheckpointProgressSnapshot {
+    pub fn has_progress(&self) -> bool {
+        self.baseline_known
+            || self.successful_objects > 0
+            || self.failed_object_count > 0
+            || self.skipped_object_count > 0
+            || self.skipped_new_versions > 0
+            || self.skipped_ilm_expired > 0
+            || self.processed_bytes > 0
+            || self.total_objects > 0
+            || self.total_bytes > 0
+            || self.baseline_generation.is_some()
+            || self.counter_unknown
+    }
+}
+
 /// resume checkpoint
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ResumeCheckpoint {
@@ -715,6 +751,75 @@ impl CheckpointManager {
         self.checkpoint.read().await.clone()
     }
 
+    /// True when a page object is already settled in the current ledger.
+    /// Membership is checked under the lock without cloning the checkpoint,
+    /// which used to copy every page's dedup sets on the small-object path.
+    pub async fn is_object_settled(&self, key: &str) -> bool {
+        let checkpoint = self.checkpoint.read().await;
+        checkpoint.processed_objects.contains(key)
+            || checkpoint.failed_objects.contains(key)
+            || checkpoint.skipped_objects.contains(key)
+    }
+
+    /// True when the administrator checkpoint already finished.
+    pub async fn admin_completed(&self) -> bool {
+        self.checkpoint
+            .read()
+            .await
+            .admin
+            .as_ref()
+            .is_some_and(|admin| admin.completed)
+    }
+
+    /// Current resume position without cloning the object ledgers.
+    pub async fn resume_position(&self) -> (usize, usize) {
+        let checkpoint = self.checkpoint.read().await;
+        (checkpoint.current_bucket_index, checkpoint.current_object_index)
+    }
+
+    /// Dangling-delete grace summary without cloning the object ledgers.
+    pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
+        let checkpoint = self.checkpoint.read().await;
+        (
+            checkpoint.dangling_delete_grace_objects,
+            checkpoint.dangling_delete_retry_not_before,
+            checkpoint.counter_unknown,
+        )
+    }
+
+    /// Aggregate progress counters without cloning the page dedup sets.
+    pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
+        let checkpoint = self.checkpoint.read().await;
+        CheckpointProgressSnapshot {
+            current_bucket_index: checkpoint.current_bucket_index,
+            current_object_index: checkpoint.current_object_index,
+            successful_objects: checkpoint.successful_objects,
+            failed_object_count: checkpoint.failed_object_count,
+            skipped_object_count: checkpoint.skipped_object_count,
+            skipped_new_versions: checkpoint.skipped_new_versions,
+            skipped_ilm_expired: checkpoint.skipped_ilm_expired,
+            processed_bytes: checkpoint.processed_bytes,
+            total_objects: checkpoint.total_objects,
+            total_bytes: checkpoint.total_bytes,
+            baseline_generation: checkpoint.baseline_generation,
+            baseline_known: checkpoint.baseline_known,
+            counter_unknown: checkpoint.counter_unknown,
+        }
+    }
+
+    /// Finish the administrator outcome in place and persist it.
+    pub async fn complete_admin(&self) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        let admin = checkpoint
+            .admin
+            .as_mut()
+            .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
+        admin.completed = true;
+        admin.outcome.finish(None);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
     /// update position
     pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;

crates/heal/src/heal/resume/handoff.rs

@@ -277,7 +277,7 @@ impl ResumeManager {
             }
             return self.ensure_handoff_successor().await;
         }
-        if state.replacement_execution_protocol != 1
+        if !matches!(state.replacement_execution_protocol, 1 | 2)
             || !matches!(
                 state.replacement_phase,
                 ReplacementPhase::Intent
@@ -458,6 +458,12 @@ impl ResumeManager {
             predecessor.replacement_targets.clone(),
             link.targets.clone(),
         );
+        if predecessor.replacement_execution_protocol == 2 {
+            state.replacement_execution_protocol = 2;
+            state.replacement_bucket_incarnations = predecessor.replacement_bucket_incarnations.clone();
+            state.replacement_retired_bucket_incarnations = predecessor.replacement_retired_bucket_incarnations.clone();
+            state.replacement_bucket_pass_reset_pending = true;
+        }
         state.replacement_phase = ReplacementPhase::OwnershipPending;
         // The constructor marks this compatibility grant consumed for every
         // successor, including descendants of a recovered older generation.
@@ -580,7 +586,7 @@ fn preformat_handoff_retry_candidate(state: &ResumeState) -> bool {
         && state.replacement_generation.as_deref() == Some(state.task_id.as_str())
         && !state.replacement_preformat_handoff_retry_done
         && state.replacement_phase == ReplacementPhase::Intent
-        && state.replacement_execution_protocol == 1
+        && matches!(state.replacement_execution_protocol, 1 | 2)
         && state.max_retries >= 2
         && state.replacement_predecessor.is_none()
         && state.replacement_handoff.is_none()

crates/heal/src/heal/resume/handoff/tests.rs

@@ -918,3 +918,33 @@ async fn committed_predecessor_is_collectible_once_its_successor_is_verified() {
         "the verified successor keeps its own record until cleanup"
     );
 }
+
+#[tokio::test]
+async fn handoff_preserves_bucket_pass_protocol_and_rewinds_successor_progress() {
+    let (_dirs, parent, execution) = fixture(1).await;
+    let identities = [("bucket-a".to_string(), Uuid::new_v4())].into_iter().collect();
+    {
+        let mut state = parent.state.write().await;
+        state.replacement_execution_protocol = 2;
+        state.replacement_bucket_incarnations = Some(identities);
+        state.resume_cursor = Some("old-pass-cursor".to_string());
+    }
+    parent.save_state_strict().await.expect("persist identity-bound predecessor");
+    let child = parent
+        .prepare_replacement_handoff(&execution, Vec::new())
+        .await
+        .expect("handoff protocol two");
+    let state = child.get_state().await;
+    assert_eq!(state.replacement_execution_protocol, 2);
+    assert_eq!(
+        state.replacement_bucket_incarnations,
+        parent.get_state().await.replacement_bucket_incarnations
+    );
+    assert!(state.replacement_bucket_pass_reset_pending);
+    assert!(state.resume_cursor.is_none());
+    assert!(state.completed_buckets.is_empty());
+    let recovered = ResumeManager::load_replacement_intent(child.disk.clone(), &state.task_id)
+        .await
+        .expect("reload protocol-two successor");
+    assert_eq!(recovered.get_state().await, state);
+}

crates/heal/src/heal/resume/replacement.rs

@@ -345,6 +345,16 @@ pub(crate) fn merge_replacement_recovery_candidate(
 
     let candidate_revision = candidate.state.replacement_revision;
     let current_revision = current.state.replacement_revision;
+    let candidate_protocol = candidate.state.replacement_execution_protocol;
+    let current_protocol = current.state.replacement_execution_protocol;
+    if (candidate_protocol > current_protocol && candidate_revision <= current_revision)
+        || (candidate_protocol < current_protocol && candidate_revision >= current_revision)
+    {
+        return Err(Error::ReplacementGenerationConflict {
+            task_id: candidate.state.task_id,
+            reason: "replacement execution protocol regresses with durable revision".to_string(),
+        });
+    }
     if candidate_revision == current_revision && candidate.state != current.state {
         return Err(Error::ReplacementGenerationConflict {
             task_id: candidate.state.task_id,
@@ -374,7 +384,10 @@ fn same_replacement_generation_binding(left: &ResumeState, right: &ResumeState)
         && left.replacement_target_identities == right.replacement_target_identities
         && left.replacement_buckets == right.replacement_buckets
         && left.replacement_generation == right.replacement_generation
-        && left.replacement_execution_protocol == right.replacement_execution_protocol
+        // A revision-ordered protocol-1 -> protocol-2 pass admission preserves
+        // the generation's target ownership and immutable lineage.
+        && matches!(left.replacement_execution_protocol, 1 | 2)
+        && matches!(right.replacement_execution_protocol, 1 | 2)
         && left.replacement_predecessor == right.replacement_predecessor
         && left.replacement_lineage == right.replacement_lineage
         && left.replacement_legacy_import == right.replacement_legacy_import
@@ -592,6 +605,12 @@ impl ResumeManager {
         if state.replacement_generation.as_deref() != Some(state.task_id.as_str()) {
             return Err(replacement_recovery_conflict("replacement failure has no matching generation"));
         }
+        if matches!(error, Error::ReplacementNamespaceChanged | Error::BucketDiscoveryIncomplete) {
+            // Namespace churn is not evidence that the replacement is irreparable.
+            state.error_message = Some(error.to_string());
+            drop(state);
+            return self.save_state_strict().await;
+        }
         if let Error::DanglingDeleteDeferred { retry_not_before } = error {
             if !super::dangling_delete_deadline_is_representable(*retry_not_before) {
                 return Err(Error::TaskExecutionFailed {

crates/heal/src/heal/resume/tests.rs

@@ -3629,3 +3629,129 @@ async fn blocked_checkpoint_is_not_revalidated_before_its_schedule() {
     assert_eq!(serde_json::from_slice::<serde_json::Value>(&retained).expect("marker JSON"), marker);
     temp_dir.close().expect("remove scheduled revalidation fixture");
 }
+
+#[test]
+fn replacement_candidate_merge_rejects_protocol_downgrade_in_either_read_order() {
+    for downgrade in [false, true] {
+        for reverse in [false, true] {
+            let task_id = ResumeUtils::generate_task_id();
+            let mut protocol_one = replacement_candidate_state(&task_id);
+            protocol_one.replacement_revision = if downgrade { 11 } else { 9 };
+            let mut protocol_two = replacement_candidate_state(&task_id);
+            protocol_two.replacement_execution_protocol = 2;
+            protocol_two.replacement_bucket_incarnations = Some(Default::default());
+            protocol_two.replacement_revision = 10;
+            let (first, second) = if reverse {
+                (protocol_two, protocol_one)
+            } else {
+                (protocol_one, protocol_two)
+            };
+            let mut selected = None;
+            merge_replacement_recovery_candidate(
+                &mut selected,
+                ReplacementRecoveryCandidate::new(first, "anchor-a", false).expect("first durable copy"),
+            )
+            .expect("initial observation");
+            let result = merge_replacement_recovery_candidate(
+                &mut selected,
+                ReplacementRecoveryCandidate::new(second, "anchor-b", true).expect("second durable copy"),
+            );
+            if downgrade {
+                assert!(
+                    matches!(result, Err(Error::ReplacementGenerationConflict { .. })),
+                    "protocol 1 at a newer revision must not erase protocol 2, reverse={reverse}"
+                );
+            } else {
+                result.expect("protocol 1 at an older revision may advance to protocol 2");
+                let state = selected.expect("selected upgraded generation").state;
+                assert_eq!(state.replacement_execution_protocol, 2);
+                assert_eq!(state.replacement_revision, 10);
+            }
+        }
+    }
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn replacement_recovery_survives_same_named_bucket_deletion_and_recreation() {
+    use crate::heal::storage_api::{
+        storage::{BucketOperations as _, DeleteBucketOptions, DiskSetSelector, MakeBucketOptions, StorageAdminApi as _},
+        test_support::{init_bucket_metadata_sys, isolated_store_over_temp_disks},
+    };
+
+    let (_directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
+    init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+    let bucket = "ahm-replacement";
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("create legal same-name bucket");
+    let disk = store
+        .disk_set_inventory(DiskSetSelector::new(0, 0))
+        .await
+        .expect("survivor inventory")
+        .into_iter()
+        .flatten()
+        .next()
+        .expect("survivor disk");
+    let task_id = ResumeUtils::generate_task_id();
+    let manager = ResumeManager::new_replacement_intent(
+        disk.clone(),
+        task_id.clone(),
+        "pool_0_set_0".to_string(),
+        vec!["other-bucket".to_string()],
+        vec!["replacement-a".to_string()],
+        vec![ReplacementTargetIdentity {
+            endpoint: "replacement-a".to_string(),
+            canonical_path: "/mnt/replacement-a".to_string(),
+            physical_device_ids: vec!["device-a".to_string()],
+            filesystem_identity: "1:2:3".to_string(),
+        }],
+    )
+    .await
+    .expect("persist sealed replacement intent");
+    let expected = manager.get_state().await;
+    store
+        .delete_bucket(bucket, &DeleteBucketOptions::default())
+        .await
+        .expect("delete same-name user bucket");
+    let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
+        .await
+        .expect("sealed intent remains recoverable");
+    assert_eq!(resumed.get_state().await, expected);
+    assert_eq!(
+        ResumeUtils::get_replacement_intent_tasks(&disk)
+            .await
+            .expect("scanner recovery enumeration"),
+        std::slice::from_ref(&task_id)
+    );
+    resumed
+        .mark_replacement_completed_and_verified()
+        .await
+        .expect("replacement can finish after deletion");
+    resumed
+        .ensure_replacement_completion_proof()
+        .await
+        .expect("persist completion proof");
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("recreate same-name user bucket");
+    store
+        .delete_bucket(bucket, &DeleteBucketOptions::default())
+        .await
+        .expect("delete recreated bucket");
+    let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id)
+        .await
+        .expect("completion proof survives recreation");
+    assert_eq!(proof.task_id, task_id);
+    assert_eq!(
+        ResumeManager::load_replacement_intent(disk, &task_id)
+            .await
+            .expect("verified intent survives")
+            .get_state()
+            .await
+            .replacement_phase,
+        ReplacementPhase::Verified
+    );
+}

crates/heal/src/heal/storage.rs

@@ -476,6 +476,15 @@ pub trait HealStorageAPI: Send + Sync {
         Ok(None)
     }
 
+    /// Published identity for a replacement pass; None proves missing metadata only.
+    async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
+        match self.admit_bucket_incarnation(bucket).await {
+            Ok(id) => Ok(Some(id)),
+            Err(Error::StaleBucketIncarnation { .. }) => Ok(None),
+            Err(error) => Err(error),
+        }
+    }
+
     /// Whether the exact old bucket generation has a durable deletion record.
     /// Alternate backends must fail closed unless they can validate the same
     /// deployment, bucket and incarnation identity.
@@ -598,6 +607,23 @@ pub trait HealStorageAPI: Send + Sync {
         self.heal_object(bucket, object, version_id, opts).await
     }
 
+    /// Replacement writes retain both cancellation ownership and the admitted bucket fence.
+    async fn heal_replacement_object_at_incarnation(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: Option<&str>,
+        expected: Uuid,
+        opts: &HealOpts,
+        task_cancel: &CancellationToken,
+    ) -> Result<HealStorageObjectResult> {
+        if task_cancel.is_cancelled() {
+            return Err(Error::TaskCancelled);
+        }
+        self.heal_object_at_incarnation(bucket, object, version_id, expected, opts)
+            .await
+    }
+
     async fn heal_object_with_receipt(
         &self,
         bucket: &str,
@@ -663,6 +689,17 @@ pub trait HealStorageAPI: Send + Sync {
         Err(Error::Storage(StorageError::PreconditionFailed))
     }
 
+    async fn heal_replacement_bucket_metadata_at_incarnation(
+        &self,
+        _bucket: &str,
+        _expected: Uuid,
+        _opts: &HealOpts,
+        _targets: &[String],
+    ) -> Result<()> {
+        // An unfenced fallback could certify a different same-name bucket.
+        Err(Error::Storage(StorageError::PreconditionFailed))
+    }
+
     /// Read target-specific physical evidence for one replacement version.
     ///
     /// This is only used by automatic replacement healing after the normal
@@ -1316,6 +1353,35 @@ impl HealStorageAPI for ECStoreHealStorage {
         .await
     }
 
+    async fn heal_replacement_object_at_incarnation(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: Option<&str>,
+        expected: Uuid,
+        opts: &HealOpts,
+        task_cancel: &CancellationToken,
+    ) -> Result<HealStorageObjectResult> {
+        run_background_object_inner(
+            self.admission.as_ref(),
+            || self.ecstore.physical_heal_plan(object, opts).map_err(Error::Storage),
+            opts,
+            Some(task_cancel),
+            |result: &HealStorageObjectResult| result.error.is_none(),
+            boxed_object_operation(|| async {
+                let result = self
+                    .ecstore
+                    .heal_object_at_incarnation(bucket, object, version_id.unwrap_or_default(), expected, opts)
+                    .await
+                    .map_err(|error| incarnation_storage_error(bucket, expected, error))?;
+                Ok(self
+                    .object_result_with_receipt(bucket, object, version_id, opts, result, Some(expected))
+                    .await)
+            }),
+        )
+        .await
+    }
+
     async fn heal_mrf_object_at_incarnation(
         &self,
         bucket: &str,
@@ -1520,6 +1586,13 @@ impl HealStorageAPI for ECStoreHealStorage {
         }
     }
 
+    async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
+        self.ecstore
+            .replacement_bucket_incarnation_from_disk(bucket)
+            .await
+            .map_err(Error::Storage)
+    }
+
     async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
         self.ecstore
             .is_bucket_incarnation_retired(bucket, expected)
@@ -1959,6 +2032,19 @@ impl HealStorageAPI for ECStoreHealStorage {
             .map_err(Error::Storage)
     }
 
+    async fn heal_replacement_bucket_metadata_at_incarnation(
+        &self,
+        bucket: &str,
+        expected: Uuid,
+        opts: &HealOpts,
+        targets: &[String],
+    ) -> Result<()> {
+        self.ecstore
+            .heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
+            .await
+            .map_err(|error| incarnation_storage_error(bucket, expected, error))
+    }
+
     async fn replacement_targets_have_version(
         &self,
         bucket: &str,

crates/heal/src/heal/storage_api.rs

@@ -98,10 +98,19 @@ pub(crate) mod storage {
     };
 
     #[cfg(test)]
-    pub(crate) use super::storage_contracts::DeleteBucketOptions;
+    pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions};
 }
 
 #[cfg(test)]
 pub(crate) mod status {
     pub(crate) use super::storage_contracts::BucketInfo;
 }
+
+#[cfg(test)]
+pub(crate) mod test_support {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
+        init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
+    };
+    pub(crate) use rustfs_ecstore::api::config::com::{delete_config, read_config, save_config};
+    pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
+}

crates/heal/src/heal/task.rs

@@ -485,7 +485,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 impl HealTask {
@@ -586,7 +586,7 @@ impl HealTask {
         task
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }
@@ -1310,7 +1310,22 @@ impl HealTask {
                 .load(Ordering::Acquire)
                 .max(self.retry_attempts)
                 .saturating_add(1);
-            manager.record_replacement_failure(&failure, attempt).await
+            // A deletion can race any stage, including hidden-transaction recovery.
+            // Only a successful complete inventory may reclassify that failure;
+            // lookup/quorum failures remain the original storage failure.
+            if manager.get_state().await.replacement_bucket_incarnations.is_some()
+                && !matches!(failure, Error::TaskCancelled | Error::TaskTimeout)
+                && matches!(
+                    manager.verify_replacement_bucket_pass(self.storage.as_ref()).await,
+                    Err(Error::ReplacementNamespaceChanged)
+                )
+            {
+                manager
+                    .record_replacement_failure(&Error::ReplacementNamespaceChanged, attempt)
+                    .await
+            } else {
+                manager.record_replacement_failure(&failure, attempt).await
+            }
         }
         .await;
         match persisted {