Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
-sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
+sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Nightly and manual-dispatch lane owned by
+# Storage-sensitive PR / nightly / dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -970,16 +970,10 @@ threads-required = "num-test-threads"
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
 
-# Keep the storage-heavy smoke cases isolated in the full suite too. They
-# start multiple server fixtures and issue many sequential S3 operations.
+# Keep storage-heavy cases isolated in the full suite too. The large pagination
+# fixture alone seeds 2,000 objects and can otherwise starve other servers.
 [[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
-threads-required = "num-test-threads"
-
-# The mc fixture uploads hundreds of objects concurrently before measuring listing.
-# Reserve the runner so other server fixtures do not starve its disk operations.
-[[profile.e2e-full.overrides]]
-filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
 # Fixed, small contract for every executable server change. These existing

.github/dependabot.yml

@@ -32,8 +32,6 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
-      - dependency-name: "pulsar"
-        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,32 +296,6 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
-    {
-      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
-      "name": "RustFS Validation Sweep",
-      "roles": [
-        "periodic",
-        "manual"
-      ],
-      "events": [
-        "schedule",
-        "workflow_dispatch"
-      ],
-      "cadence": "nightly",
-      "timezone": "UTC",
-      "schedule": [
-        "30 19 * * *"
-      ],
-      "owner": "@overtrue",
-      "policy": "active",
-      "monitor": "workflow",
-      "max_age_hours": 60,
-      "result_owner": "rustfs-validation-sweep.yml",
-      "disposition": "canonical validation cases executed in rotating batches",
-      "failure_owner": "workflow",
-      "origin": "baseline",
-      "alert_contract": "scheduled-validation"
-    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -769,10 +743,12 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
+        "pr",
         "periodic",
         "manual"
       ],
       "events": [
+        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -847,11 +823,12 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
+        "pr",
         "periodic",
-        "manual",
-        "release"
+        "manual"
       ],
       "events": [
+        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -497,63 +497,6 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
-  fips-provider:
-    name: FIPS provider boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
-    needs: [ quick-checks, classify-changes ]
-    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      matrix:
-        mode: [default, fips]
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      CARGO_BUILD_JOBS: "2"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-fips-provider
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-
-      - name: Install FIPS build tools
-        if: matrix.mode == 'fips'
-        run: sudo apt-get install -y cmake golang clang libclang-dev
-
-      - name: Verify provider installation
-        shell: bash
-        run: |
-          args=()
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            args+=(--features fips)
-          fi
-          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
-          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
-          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
-          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
-            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
-            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
-          fi
-          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
-          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
-            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
-          fi
-          for test_name in "${tests[@]}"; do
-            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
-          done
-          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
-
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     # Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1454,7 +1397,6 @@ jobs:
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
-      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1482,7 +1424,6 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
-      - fips-provider
       - required-checks
       - typos
       - quick-checks

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
-# and manual dispatches execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
 name: e2e-distributed
 
 on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".config/e2e-distributed-selection.txt"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,19 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Single-node upgrade and rollback lane.
-#
-# The weekly Monday schedule, version tags, and workflow_dispatch run this
-# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
-
 name: Upgrade Compatibility
 
 on:
+  pull_request:
+    paths:
+      - ".github/workflows/e2e-upgrade.yml"
+      - "crates/e2e_test/src/common.rs"
+      - "crates/e2e_test/src/distributed/**"
+      - "crates/e2e_test/src/fake_s3_target/**"
+      - "crates/e2e_test/src/lib.rs"
+      - "crates/e2e_test/src/replication_extension_test.rs"
+      - "crates/e2e_test/src/rpc_fault_proxy.rs"
+      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/kms/**"
+      - "crates/storage-api/**"
+      - "rustfs/**"
+      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
-    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -33,7 +44,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/nightly-gnu.yml

@@ -346,12 +346,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
+      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
+      # engineers can download and install the nightly directly. The branch is
+      # a single-commit orphan rewritten on every build, which keeps the repo
+      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
-      # The dispatch below runs only after this upload and the immutable R2
-      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
-      # which still starts from this workflow's workflow_run completion.
-      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,90 +360,6 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
-
-      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
-      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
-      # or fine-grained Contents write (the permission GitHub checks for
-      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
-      # step, and therefore this job, so the scheduled failure alert fires.
-      # GitHub allows at most 10 top-level client_payload properties, so the
-      # schema 2 candidate object is nested under `candidate` and the release
-      # fields sit beside it.
-      - name: Dispatch nightly candidate to auto-testing
-        if: ${{ steps.publish.outputs.candidate_file != '' }}
-        env:
-          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
-          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
-          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
-          SOURCE_REPO: ${{ github.repository }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
-          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
-          : "${DEB_DATE:?DEB_DATE is required}"
-          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
-          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
-
-          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
-            --arg event_type "nightly-candidate-ready" \
-            --arg source_repo "${SOURCE_REPO}" \
-            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
-            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
-            --arg channel "nightly" \
-            --arg deb_date "${DEB_DATE}" \
-            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
-            '{
-              event_type: $event_type,
-              source_repo: $source_repo,
-              latest_alias_url: $latest_alias_url,
-              auto_testing_release: $auto_testing_release,
-              channel: $channel,
-              deb_date: $deb_date,
-              nightly_workflow: $nightly_workflow,
-              candidate: $candidate[0]
-            }' > "${payload}"
-          if ! jq -e '
-            .event_type == "nightly-candidate-ready"
-            and .source_repo == "rustfs/rustfs"
-            and .channel == "nightly"
-            and .deb_date == env.DEB_DATE
-            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
-            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
-            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
-            and .candidate.schema == 2
-            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
-            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
-          ' "${payload}" >/dev/null; then
-            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
-            exit 1
-          fi
-
-          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --slurpfile client_payload "${payload}" \
-            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
-          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
-          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-            -o "${response}" -w '%{http_code}' \
-            -X POST \
-            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            -H "Content-Type: application/json" \
-            --data-binary @"${body}" \
-            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
-            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
-            exit 1
-          }
-          if [[ "${status}" != "204" ]]; then
-            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
-            if [[ -s "${response}" ]]; then
-              cat "${response}" >&2
-            fi
-            exit 1
-          fi
-          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-validation-sweep.yml

@@ -1,457 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: RustFS Validation Sweep
-
-# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
-# on the shared functional fleet. Everything with a lane executor is swept in
-# rotating nightly batches so the whole set is covered roughly weekly; a
-# single case or one module can be dispatched manually. That is the
-# implemented catalog (66 today) plus the eight planned cases whose dedicated
-# executors already exist and have live PASS evidence - fresh nightly
-# evidence feeds their acceptance review. heal.* cases stay on the
-# interactive validation flow: they require reviewed native/tool collectors
-# built by separate validation-tools runs before fleet execution.
-
-on:
-  schedule:
-    # After the nightly functional chain settles (chain starts ~16:48 UTC and
-    # runs for about two hours).
-    - cron: "30 19 * * *"
-  workflow_dispatch:
-    inputs:
-      case:
-        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
-        type: string
-        required: false
-        default: ''
-      module:
-        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
-        type: string
-        required: false
-        default: ''
-      limit:
-        description: 'Maximum cases per sweep (0 = run everything selected)'
-        type: number
-        required: false
-        default: 12
-      build_run_id:
-        description: 'Successful nightly package run; empty resolves the latest'
-        type: string
-        required: false
-        default: ''
-      build_run_attempt:
-        description: 'Attempt of that package run'
-        type: string
-        required: false
-        default: '1'
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-validation-sweep
-  cancel-in-progress: false
-
-env:
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  sweep:
-    if: github.repository == 'rustfs/rustfs'
-    runs-on: smoke-testing
-    timeout-minutes: 150
-    steps:
-      - name: Checkout chain tooling
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Resolve nightly candidate
-        id: candidate
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          BUILD_RUN_ID: ${{ inputs.build_run_id }}
-          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
-          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
-        run: |
-          set -euo pipefail
-          # Scheduled runs carry no inputs and the resolver only derives a
-          # build from a workflow_run payload (which a schedule cannot
-          # provide), so resolve the newest successful nightly build
-          # explicitly before invoking it.
-          if [ -z "${BUILD_RUN_ID:-}" ]; then
-            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
-            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
-            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
-            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
-            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
-          fi
-          python3 scripts/resolve_functional_candidate.py
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
-          root.mkdir(parents=True, exist_ok=True)
-          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
-          manifest = chain["candidate"]["manifest"]
-          (root / "candidate.json").write_text(json.dumps({
-              "package_url": manifest["package_url"],
-              "package_sha256": manifest["package_sha256"],
-              "source_sha": manifest["source_sha"],
-          }, sort_keys=True) + "\n")
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("testing_sha=" + chain["testing_sha"] + "\n")
-              output.write("package_url=" + manifest["package_url"] + "\n")
-              output.write("source_sha=" + manifest["source_sha"] + "\n")
-          PY
-
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ steps.candidate.outputs.testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Prepare validation Python runtime
-        # The pool / ilm / replication validation runners require a managed
-        # Python, exactly like the chain's validation job provides.
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.12"
-
-      - name: Install validation Python requirements
-        # Case scripts call python3 and import boto3 at startup. Install the
-        # functional chain's client into a per-run env so that interpreter
-        # sees it without changing the runner's shared Python.
-        run: |
-          set -euo pipefail
-          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          test ! -e "${validation_python}"
-          python3 -m venv "${validation_python}"
-          "${validation_python}/bin/python" -m pip install \
-            -r auto-testing/scripts/validation_ilm/requirements.txt
-          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
-
-      - name: Select sweep cases
-        id: select
-        env:
-          SELECTED_CASE: ${{ inputs.case }}
-          SELECTED_MODULE: ${{ inputs.module }}
-          CASE_LIMIT: ${{ inputs.limit }}
-          SWEEP_EVENT: ${{ github.event_name }}
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import datetime
-          import json
-          import os
-          from pathlib import Path
-          sweep_root = Path(os.environ["SWEEP_ROOT"])
-
-          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
-          def sweepable(case):
-              status = case.get("source_status")
-              if status == "implemented":
-                  return True
-              # The remaining planned non-heal cases route to the lane
-              # executors and already have live PASS evidence; sweeping them
-              # nightly produces fresh evidence for their acceptance review.
-              return status == "planned" and not case["id"].startswith("heal.")
-
-          selected = [(case["id"], module["id"]) for module in catalog["modules"]
-                      for case in module["cases"]
-                      if sweepable(case)]
-          if not selected:
-              raise SystemExit("catalog has no sweepable cases")
-
-          one_case = os.environ.get("SELECTED_CASE", "").strip()
-          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
-          known = {case_id for case_id, _ in selected}
-          modules = {module_id for _, module_id in selected}
-          if one_case:
-              if one_case not in known:
-                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
-              selected = [(one_case, dict(selected)[one_case])]
-          elif module_filter:
-              if module_filter not in modules:
-                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
-              selected = [pair for pair in selected if pair[1] == module_filter]
-          elif os.environ.get("SWEEP_EVENT") == "schedule":
-              # Stateless rotation: every night take one window of the catalog
-              # so a full sweep needs len(selected)/limit nights and no state
-              # has to be carried between runs.
-              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
-              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
-              start = (day * limit) % len(selected)
-              selected = (selected[start:] + selected[:start])[:limit]
-          limit = int(os.environ.get("CASE_LIMIT") or 0)
-          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
-              if limit > 0:
-                  selected = selected[:limit]
-
-          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write(f"count={len(selected)}\n")
-          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
-          PY
-
-      - name: Admit clean shared environment
-        id: environment
-        timeout-minutes: 5
-        env:
-          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
-        run: python3 scripts/functional_environment.py enter --suite heal
-
-      - name: Run validation cases
-        id: run_cases
-        timeout-minutes: 120
-        env:
-          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -uo pipefail
-          mkdir -p "${SWEEP_ROOT}/results"
-          : > "${SWEEP_ROOT}/results.txt"
-          while read -r CASE_ID; do
-            [ -n "${CASE_ID}" ] || continue
-            echo "::group::case ${CASE_ID}"
-            rm -rf "${SWEEP_ROOT}/run"
-            if ./auto-testing/scripts/run_validation_case.sh \
-                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
-                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
-                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
-              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
-              echo "case ${CASE_ID}: PASS"
-            else
-              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
-              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
-            fi
-            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
-              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
-            fi
-            echo "::endgroup::"
-            # Fleet hygiene between cases: the case runners set up their own
-            # topology, this purge guarantees the next case starts clean even
-            # when the previous one died mid-run.
-            for node in ${RUSTFS_NODES}; do
-              # -n keeps ssh from consuming the case list this loop reads from stdin
-              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
-                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-                ${SUDO} systemctl stop rustfs 2>/dev/null || true
-                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                  ${SUDO} dpkg -P rustfs
-                fi
-                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-                ${SUDO} rm -f /etc/default/rustfs
-              ' || echo "inter-case cleanup failed on ${node} (continuing)"
-            done
-          done < "${SWEEP_ROOT}/sweep-cases.txt"
-          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
-
-      - name: Generate sweep report
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        env:
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
-          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
-          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-
-          root = Path(os.environ["SWEEP_ROOT"])
-          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
-          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
-          rows = []
-          for line in (root / "results.txt").read_text().splitlines():
-            if not line:
-                continue
-            case_id, status = line.split("\t")
-            rows.append((case_id, module_of.get(case_id, "?"), status))
-          passed = sum(1 for _, _, s in rows if s == "PASS")
-          failed = len(rows) - passed
-          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
-                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
-                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
-
-          report = [
-              "# RustFS validation sweep report", "",
-              f"- Run: {run_url}",
-              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
-              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
-              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
-              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
-              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
-              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
-          ]
-          # Group the executed cases by catalog module (the major category:
-          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
-          # data services, identity, s3 compatibility). Per-module subtotals
-          # deliberately use "- Subtotal:" so the dashboard overview regexes
-          # (which anchor on "- Total:") keep reading the grand totals below.
-          by_module = {}
-          module_order = []
-          for case_id, module, status in rows:
-              if module not in by_module:
-                  by_module[module] = []
-                  module_order.append(module)
-              by_module[module].append((case_id, status))
-          report += ["## Case Summary", ""]
-          for module in module_order:
-              cases = by_module[module]
-              mod_pass = sum(1 for _, st in cases if st == "PASS")
-              mod_fail = len(cases) - mod_pass
-              report += [f"### {module} ({len(cases)})", "",
-                         "| Case | Status |", "| --- | --- |"]
-              report += [f"| {case_id} | {status} |" for case_id, status in cases]
-              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
-          report += [
-              "## Totals", "",
-              f"- Total: {len(rows)}",
-              f"- PASS: {passed}",
-              f"- FAIL: {failed}",
-              "- UNSUPPORTED: 0", "",
-              "Failing cases keep this workflow red only as a report; the sweep artifact",
-              "retains every case log and per-case report for diagnosis.", "",
-          ]
-          (root / "report.md").write_text("\n".join(report))
-          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
-          PY
-          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        timeout-minutes: 2
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
-          SUITE: validation
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-          echo "dashboard updated: ${REPORT_PATH}"
-
-      - name: Retain sweep evidence
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            ${{ runner.temp }}/validation-sweep/report.md
-            ${{ runner.temp }}/validation-sweep/results.txt
-            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
-            ${{ runner.temp }}/validation-sweep/candidate.json
-            ${{ runner.temp }}/validation-sweep/results/
-            ${{ runner.temp }}/validation-sweep/run/
-          # warn (not error): an early infra failure legitimately leaves no
-          # per-case evidence, and the failing step is already red.
-          if-no-files-found: warn
-          retention-days: 30
-
-      - name: Cleanup environment (after)
-        id: environment_cleanup
-        timeout-minutes: 5
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        run: |
-          set -uo pipefail
-          cleanup_status=0
-          for node in ${RUSTFS_NODES}; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -f /etc/default/rustfs
-            ' || cleanup_status=$?
-          done
-          exit "${cleanup_status}"
-
-      - name: Release environment only after successful cleanup
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        timeout-minutes: 5
-        env:
-          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
-          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
-        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
-
-      - name: Notify on failure
-        if: failure()
-        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
-
-      - name: Enforce sweep result
-        if: ${{ always() && steps.environment.outcome == 'success' }}
-        env:
-          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
-        run: |
-          set -euo pipefail
-          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
-            echo "validation sweep has failing cases:"
-            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
-            exit 1
-          fi
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    # A scheduled sweep failure usually means a canonical validation case
-    # regressed or the fleet environment broke; surface it as a tracking
-    # issue instead of leaving it in the Actions tab.
-    needs: [sweep]
-    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
-- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.52" }
+async-compression = { version = "0.4.51" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -212,13 +212,12 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
-aws-lc-rs = { version = "1.18.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.1.0" }
+openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -235,7 +234,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.39" }
+jiff = { version = "0.2.38" }
 time = { version = "0.3.55" }
 
 # Database
@@ -254,9 +253,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.129.0" }
-aws-sdk-s3 = { default-features = false, version = "1.157.0" }
-aws-sdk-sts = { default-features = false, version = "1.124.0" }
+aws-sdk-kms = { default-features = false, version = "1.128.0" }
+aws-sdk-s3 = { default-features = false, version = "1.156.0" }
+aws-sdk-sts = { default-features = false, version = "1.123.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -338,7 +337,7 @@ strum = { version = "0.28.0" }
 sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.1"
+test-case = "3.4.0"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -357,13 +356,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.3.0"
+zip = "9.1.3"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.7", default-features = false }
+dial9 = { version = "0.5.6", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -385,9 +384,9 @@ russh-sftp = "3.0.1"
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.3" }
+rustfs-mimalloc = { version = "0.7.2" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.3", default-features = false }
+hotpath = { version = "0.30.2", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -428,3 +427,7 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
+# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
+# Upstream: openidconnect-rs#244 and oauth2-rs#352.
+openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
+oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,249 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
 use crate::common::init_logging;
-use aws_sdk_s3::{
-    Client,
-    error::{ProvideErrorMetadata, SdkError},
-    primitives::ByteStream,
-};
-use std::any::TypeId;
 use std::collections::BTreeSet;
-use std::fmt::Debug;
-use std::future::Future;
 use std::sync::Arc;
-use std::time::Duration;
 use tokio::sync::Barrier;
-use tokio::time::Instant;
-
-const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
-
-async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
-where
-    E: Debug + ProvideErrorMetadata + 'static,
-    F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T, SdkError<E>>>,
-{
-    let started = Instant::now();
-    let mut attempts = 0;
-    // The workload uses unique unversioned keys. Repeat only explicitly
-    // advertised lock backpressure; every step still needs its real result.
-    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
-        loop {
-            attempts += 1;
-            let result = call().await;
-            let retry = result.as_ref().err().is_some_and(|error| {
-                let Some(response) = error.raw_response() else {
-                    return false;
-                };
-                let Some(service_error) = error.as_service_error() else {
-                    return false;
-                };
-                // HEAD has no XML error body, so its SDK service error can
-                // lack a code even when the server supplies retry headers.
-                let namespace_code = service_error.code() == Some("ServiceUnavailable")
-                    || (service_error.code().is_none()
-                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
-                response.status().as_u16() == 503
-                    && namespace_code
-                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
-                    && response.headers().get("retry-after") == Some("1")
-            });
-            if !retry {
-                return result;
-            }
-            tokio::time::sleep(Duration::from_secs(1)).await;
-        }
-    })
-    .await;
-    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
-        eprintln!(
-            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
-            started.elapsed()
-        );
-    }
-    outcome.map_err(|_| {
-        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
-    })
-}
-
-async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
-    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
-    let actual = output.body.collect().await?.into_bytes();
-    if actual.as_ref() != expected {
-        return Err(format!(
-            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
-            expected.len(),
-            sha256_hex(expected),
-            actual.len(),
-            sha256_hex(&actual)
-        )
-        .into());
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use aws_smithy_http_client::test_util::infallible_client_fn;
-    use aws_smithy_types::body::SdkBody;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-
-    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
-
-    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
-        let requests = Arc::new(AtomicUsize::new(0));
-        let seen = Arc::clone(&requests);
-        let transport = infallible_client_fn(move |_| {
-            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
-            let body = if code.is_empty() {
-                body.to_string()
-            } else {
-                format!("<Error><Code>{code}</Code></Error>")
-            };
-            let mut response = http::Response::builder()
-                .status(status)
-                .header("content-type", "application/xml")
-                .header("content-length", body.len());
-            if let Some(cause) = cause {
-                response = response.header("x-rustfs-error-cause", cause);
-            }
-            if let Some(retry_after) = retry_after {
-                response = response.header("retry-after", retry_after);
-            }
-            response.body(SdkBody::from(body)).expect("scripted S3 response")
-        });
-        let config = crate::common::build_test_s3_config(
-            "http://contention.example.invalid",
-            "synthetic-access",
-            "synthetic-secret",
-            None,
-            "contention-test",
-        );
-        let client = Client::from_conf(config.to_builder().http_client(transport).build());
-        (s3_probe_client(&client), requests)
-    }
-
-    fn contention() -> Reply {
-        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
-        let (client, requests) = client(|attempt| {
-            if attempt < 3 {
-                contention()
-            } else {
-                (200, "", None, None, "")
-            }
-        });
-        let standard = Client::from_conf(
-            client
-                .config()
-                .to_builder()
-                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
-                .build(),
-        );
-        standard
-            .put_object()
-            .bucket("bucket")
-            .key("key")
-            .body(ByteStream::from_static(b"body"))
-            .send()
-            .await
-            .expect_err("three advertised lock responses exhaust the standard SDK policy");
-        assert_eq!(requests.load(Ordering::SeqCst), 3);
-        requests.store(0, Ordering::SeqCst);
-        let started = Instant::now();
-        after_lock_contention("PUT recovery", || {
-            client
-                .put_object()
-                .bucket("bucket")
-                .key("key")
-                .body(ByteStream::from_static(b"body"))
-                .send()
-        })
-        .await??;
-        assert_eq!(requests.load(Ordering::SeqCst), 4);
-        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
-        let (client, requests) = client(|attempt| {
-            if attempt == 0 {
-                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
-            } else {
-                (200, "", None, None, "")
-            }
-        });
-        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
-        assert_eq!(requests.load(Ordering::SeqCst), 2);
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
-        for reply in [
-            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
-            (403, "AccessDenied", None, None, ""),
-            (404, "NoSuchKey", None, None, ""),
-            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
-            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
-            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
-            (503, "ServiceUnavailable", None, Some("1"), ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
-            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
-        ] {
-            let (client, requests) = client(move |_| reply);
-            let result = after_lock_contention("PUT fatal response", || {
-                client
-                    .put_object()
-                    .bucket("bucket")
-                    .key("key")
-                    .body(ByteStream::from_static(b"body"))
-                    .send()
-            })
-            .await?;
-            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
-            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
-        }
-        Ok(())
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn persistent_lock_contention_exhausts_the_request_budget() {
-        let (client, requests) = client(|_| contention());
-        let started = Instant::now();
-        let result = after_lock_contention("PUT unavailable", || {
-            client
-                .put_object()
-                .bucket("bucket")
-                .key("key")
-                .body(ByteStream::from_static(b"body"))
-                .send()
-        })
-        .await;
-        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
-        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
-        assert!(requests.load(Ordering::SeqCst) > 1);
-        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn acknowledged_wrong_bytes_fail_without_retry() {
-        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
-        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
-            .await
-            .expect_err("a successful but incorrect read is a consistency failure");
-        assert!(error.to_string().contains("bytes mismatch"));
-        assert_eq!(requests.load(Ordering::SeqCst), 1);
-    }
-}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    // This consistency probe owns its admission policy; unrelated responses
-    // must not be hidden by the SDK's general-purpose retries.
-    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
+    let clients = Arc::new(dist.clients()?);
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                let context = format!("worker={idx} round={round}");
-                after_lock_contention(&format!("{context} PUT source"), || {
-                    writer
-                        .put_object()
-                        .bucket(&bucket)
-                        .key(&key)
-                        .body(ByteStream::from(body.clone()))
-                        .send()
-                })
-                .await??;
+                put_object(writer, &bucket, &key, body.clone()).await?;
 
-                let head = after_lock_contention(&format!("{context} HEAD source"), || {
-                    reader.head_object().bucket(&bucket).key(&key).send()
-                })
-                .await??;
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_workload_bytes(reader, &bucket, &key, &body).await?;
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
 
-                after_lock_contention(&format!("{context} COPY retained"), || {
-                    copier
-                        .copy_object()
-                        .bucket(&bucket)
-                        .key(&copy_key)
-                        .copy_source(format!("{bucket}/{key}"))
-                        .send()
-                })
-                .await??;
-                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                after_lock_contention(&format!("{context} DELETE source"), || {
-                    writer.delete_object().bucket(&bucket).key(&key).send()
-                })
-                .await??;
-                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
-                    reader.head_object().bucket(&bucket).key(&key).send()
-                })
-                .await?
-                .expect_err("deleted source key must not remain visible");
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
-    assert_eq!(
-        inventory.len(),
-        worker_count * rounds,
-        "every worker round must retain its acknowledged copy"
-    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
-            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
-        })
-        .await??;
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_workload_bytes(client, &bucket, key, body)
+            assert_object_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -576,6 +576,34 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn reserve_object_metadata(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
+    ) -> Result<Response<SnapshotLeaseResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn activate_object_metadata_reservation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
+    ) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn cancel_object_metadata_reservation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
+    ) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn rename_reserved_data(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn rename_data_with_pool_meta_cas(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
-    ServerSideEncryption, VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
+    VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
-    let source_log = format!("{}/source.log", source_env.temp_dir);
-    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    let wait_for_source_writeback = || async {
-        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
-        timeout(Duration::from_secs(30), async {
-            loop {
-                let response = signed_request(
-                    http::Method::GET,
-                    &metrics_url,
-                    &source_env.access_key,
-                    &source_env.secret_key,
-                    None,
-                    None,
-                )
-                .await?;
-                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
-                let queued = metrics["queued"]["curr"]["count"]
-                    .as_f64()
-                    .ok_or("missing replication queue count")?;
-                if metrics["provider_available"] == true && queued == 0.0 {
-                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
-                }
-                sleep(Duration::from_millis(50)).await;
-            }
-        })
-        .await?
-    };
-    wait_for_source_writeback().await?;
-
-    let purged = source_client
-        .delete_objects()
+    source_client
+        .delete_object()
         .bucket(source_bucket)
-        .delete(
-            Delete::builder()
-                .objects(
-                    ObjectIdentifier::builder()
-                        .key(object_key)
-                        .version_id(&purged_version_id)
-                        .build()?,
-                )
-                .objects(
-                    ObjectIdentifier::builder()
-                        .key(object_key)
-                        .version_id(delete_marker_version_id)
-                        .build()?,
-                )
-                .build()?,
-        )
+        .key(object_key)
+        .version_id(&purged_version_id)
         .send()
         .await?;
-    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
-    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
-    assert!(
-        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
-        "target retained the explicitly purged delete marker"
-    );
     assert!(
         target_state
             .iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
-    // Wait for the source workers to finish their final writeback, which runs
-    // after the remote purge has already become visible (rustfs#9121).
-    wait_for_source_writeback().await?;
-    let logs = fs::read_to_string(source_log).await?;
-    let writeback_failures: Vec<_> = logs
-        .lines()
-        .filter(|line| line.contains("apply_replication_delete_state"))
-        .collect();
-    assert!(
-        writeback_failures.is_empty(),
-        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
-    );
-
     Ok(())
 }
 

crates/ecstore/Cargo.toml

@@ -31,7 +31,6 @@ workspace = true
 
 [features]
 default = []
-fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/bucket/metadata_sys.rs

@@ -683,7 +683,6 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
-    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -691,7 +690,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
+        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -5210,61 +5209,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
-        let (_dirs, store) = isolated_store_over_temp_disks().await;
-        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-        let bucket = "catalog-cleanup-incarnation";
-        store
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create bucket");
-        let incarnation = store
-            .bucket_incarnation_id_from_disk(bucket)
-            .await
-            .expect("read bucket incarnation");
-        let called = std::sync::atomic::AtomicBool::new(false);
-        let result = store
-            .update_bucket_metadata_config_for_incarnation_validated(
-                bucket,
-                "quota.json",
-                br#"{"quota":1}"#.to_vec(),
-                Uuid::new_v4(),
-                || {
-                    called.store(true, std::sync::atomic::Ordering::SeqCst);
-                    Ok(())
-                },
-            )
-            .await;
-        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
-        assert!(
-            !called.load(std::sync::atomic::Ordering::SeqCst),
-            "incarnation must be checked before the external fence"
-        );
-        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
-            .await
-            .expect("read unchanged bucket metadata");
-        assert!(
-            metadata.quota_config_json.is_empty(),
-            "a stale cleanup must not modify the replacement bucket"
-        );
-        store
-            .update_bucket_metadata_config_for_incarnation_validated(
-                bucket,
-                "quota.json",
-                br#"{"quota":1}"#.to_vec(),
-                incarnation,
-                || Ok(()),
-            )
-            .await
-            .expect("the current incarnation permits the validated update");
-        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
-            .await
-            .expect("read updated bucket metadata");
-        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
-    }
-
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
-    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -117,10 +116,6 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
-        assert_eq!(
-            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
-            "config/replication/mrf.bin.replay"
-        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
-/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
-/// re-attempts every retained entry, so a target that stays down costs one
-/// attempt per entry per interval.
-const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next pass.
+/// after that generation are retained as a suffix and replayed on the next startup.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for one journal replay pass.
+/// Acquires the MRF recovery leader lock for the startup replay.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next replay pass and
+/// On acknowledgement failure the backlog is preserved for the next startup and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
-fn unix_millis_now() -> i64 {
-    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
-}
-
-/// Whether a periodic pass may run, given the start time of the last pass on
-/// any node. A stamp more than one interval ahead of this node's clock is skew
-/// or a bad value, so it does not suppress replay.
-fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
-    let Some(last_started_ms) = last_started_ms else {
-        return true;
-    };
-    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
-    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
-}
-
-async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
-    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
-        .await
-        .ok()?;
-    std::str::from_utf8(&data).ok()?.trim().parse().ok()
-}
-
-async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
-    if let Err(error) = ReplicationConfigStore::save(
-        storage.clone(),
-        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
-        started_ms.to_string().into_bytes(),
-    )
-    .await
-    {
-        warn!(
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION,
-            error = %error,
-            "Failed to record the MRF replay start time"
-        );
-    }
-}
-
-/// One MRF journal replay pass. The recovery leader reads the journal,
-/// re-drives every entry, and acknowledges the replayed prefix; entries that
-/// could not be admitted, and entries appended meanwhile, stay for the next pass.
-///
-/// Every node runs the periodic loop, and the leader lock only serializes
-/// passes. With `min_interval`, a pass is skipped when a pass on any node
-/// started within that interval, read from the shared replay stamp, so a
-/// cluster replays once per interval rather than once per node. The startup
-/// pass passes `None` and always runs. A pass that finds entries records its
-/// start time. Returns whether the journal was read.
-/// Lock order: recovery leader lock -> replay stamp object lock.
-async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
-    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
-        return false;
-    };
-    let started_ms = unix_millis_now();
-    if let Some(interval) = min_interval
-        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
-    {
-        return false;
-    }
-
-    let Some(entries) = load_mrf_recovery_entries(storage).await else {
-        return true;
-    };
-    if !entries.is_empty() {
-        write_mrf_replay_stamp(storage, started_ms).await;
-    }
-    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-    let total = entries.len();
-    let mut queued_count = 0usize;
-    let mut retry_entries = Vec::new();
-
-    for entry in entries.iter() {
-        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
-            continue;
-        };
-
-        if admission == ReplicationQueueAdmission::Missed {
-            retry_entries.push(entry.clone());
-        } else if admission == ReplicationQueueAdmission::Queued {
-            queued_count += 1;
-        }
-    }
-
-    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
-    let retained_count = retained.len();
-    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-    if queued_count > 0 {
-        info!(
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION,
-            recovered = queued_count,
-            total,
-            retained = retained_count,
-            "Replayed MRF entries admitted for retry"
-        );
-    }
-    true
-}
-
-/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
-/// one pass at a time across the cluster, including the startup pass, and the
-/// shared replay stamp spaces passes from different nodes by `period`.
-fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
-    tokio::spawn(async move {
-        loop {
-            tokio::time::sleep(period).await;
-            replay_mrf_journal(&storage, Some(period)).await;
-        }
-    })
-}
-
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
-        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
-    /// the pass while the server runs.
+    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
+    /// rewrites any entries that could not be admitted for a later startup retry.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-        let handle = tokio::spawn(async move {
-            replay_mrf_journal(&storage, None).await;
-        });
-        self.task_handles.lock().await.push(handle);
-    }
 
-    /// Starts the runtime MRF replayer — ongoing background task.
-    ///
-    /// Entries reach the journal while the server runs (worker-queue overflow,
-    /// delete-marker purge intents whose watch window drained with targets still
-    /// dirty). Replaying only at startup left them waiting for a restart.
-    async fn start_mrf_replayer(&self) {
-        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        let handle = tokio::spawn(async move {
+            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
+                return;
+            };
+
+            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
+                return;
+            };
+            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+            let total = entries.len();
+            let mut queued_count = 0usize;
+            let mut retry_entries = Vec::new();
+
+            for entry in entries.iter() {
+                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
+                    continue;
+                };
+
+                if admission == ReplicationQueueAdmission::Missed {
+                    retry_entries.push(entry.clone());
+                } else if admission == ReplicationQueueAdmission::Queued {
+                    queued_count += 1;
+                }
+            }
+
+            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
+            let retained_count = retained.len();
+            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+            if queued_count > 0 {
+                info!(
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION,
+                    recovered = queued_count,
+                    total,
+                    retained = retained_count,
+                    "Replayed MRF entries admitted for retry"
+                );
+            }
+        });
         self.task_handles.lock().await.push(handle);
     }
 
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// recovery leader of a replay pass may remove an acknowledged prefix.
+    /// startup recovery leader may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -3511,8 +3421,6 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
-        delete_error: StdMutex<EcstoreError>,
-        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3580,24 +3488,6 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
-            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
-                let stamp = self
-                    .shared
-                    .replay_stamp
-                    .lock()
-                    .expect("test replay stamp lock should not be poisoned")
-                    .clone()
-                    .ok_or(EcstoreError::FileNotFound)?;
-                return Ok(Self::GetObjectReader {
-                    object_info: ObjectInfo {
-                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
-                        ..Default::default()
-                    },
-                    stream: Box::new(Cursor::new(stamp)),
-                    buffered_body: None,
-                    body_source: Default::default(),
-                });
-            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3661,16 +3551,6 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
-            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
-                let mut stamp = Vec::new();
-                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
-                *self
-                    .shared
-                    .replay_stamp
-                    .lock()
-                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
-                return Ok(ObjectInfo::default());
-            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3792,12 +3672,7 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(self
-                .shared
-                .delete_error
-                .lock()
-                .expect("delete error lock should not be poisoned")
-                .clone())
+            Err(EcstoreError::NotImplemented)
         }
 
         async fn delete_objects(
@@ -4296,8 +4171,6 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
-            delete_error: StdMutex::new(EcstoreError::NotImplemented),
-            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4348,103 +4221,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn delete_worker_already_purged_source_completes_without_retry() {
-        let arn = "arn:test";
-        for delete_marker_version in [false, true] {
-            for version_id in [Uuid::new_v4(), Uuid::nil()] {
-                for error in [
-                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
-                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-                    EcstoreError::FileVersionNotFound,
-                    EcstoreError::FileNotFound,
-                ] {
-                    let shared = empty_resync_shared_state();
-                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
-                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
-                    let delete = DeletedObjectReplicationInfo {
-                        bucket: "source".to_string(),
-                        delete_object: ReplicationDeletedObject {
-                            object_name: "purged".to_string(),
-                            version_id: (!delete_marker_version).then_some(version_id),
-                            delete_marker_version_id: delete_marker_version.then_some(version_id),
-                            replication_state: Some(ReplicationState {
-                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
-                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
-                                ..Default::default()
-                            }),
-                            ..Default::default()
-                        },
-                        op_type: ReplicationType::Delete,
-                        ..Default::default()
-                    };
-
-                    assert!(
-                        replicate_delete_with_outcome(delete, storage).await,
-                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
-                    );
-                    assert_eq!(
-                        shared.delete_storage_calls.load(Ordering::SeqCst),
-                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
-                        "the worker must lock and attempt writeback, reading a non-null data version first"
-                    );
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
-        let arn = "arn:test";
-        for (version_purge, target_status, error) in [
-            (
-                true,
-                VersionPurgeStatusType::Pending,
-                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-            ),
-            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
-            (
-                false,
-                VersionPurgeStatusType::Empty,
-                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
-            ),
-        ] {
-            let shared = empty_resync_shared_state();
-            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
-            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
-            let delete = DeletedObjectReplicationInfo {
-                bucket: "source".to_string(),
-                delete_object: ReplicationDeletedObject {
-                    object_name: "purged".to_string(),
-                    version_id: version_purge.then(Uuid::new_v4),
-                    delete_marker: !version_purge,
-                    replication_state: Some(ReplicationState {
-                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
-                        targets: if version_purge {
-                            HashMap::new()
-                        } else {
-                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
-                        },
-                        purge_targets: if version_purge {
-                            HashMap::from([(arn.to_string(), target_status)])
-                        } else {
-                            HashMap::new()
-                        },
-                        ..Default::default()
-                    }),
-                    ..Default::default()
-                },
-                op_type: ReplicationType::Delete,
-                ..Default::default()
-            };
-
-            assert!(
-                !replicate_delete_with_outcome(delete, storage).await,
-                "unfinished target work, source I/O failures, and marker creation must remain retryable"
-            );
-        }
-    }
-
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5528,8 +5304,6 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
-                delete_error: StdMutex::new(EcstoreError::NotImplemented),
-                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6494,108 +6268,6 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
-    #[tokio::test]
-    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
-        let shared = empty_resync_shared_state();
-        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
-        // A force-delete intent from a superseded generation replays as
-        // Skipped, so a pass that reaches it acknowledges it off the journal.
-        let stale_intent = |object: &str| MrfReplicateEntry {
-            bucket: "mrf-runtime-replay".to_string(),
-            object: object.to_string(),
-            op: MrfOpKind::Delete,
-            force_delete_id: Some(uuid::Uuid::new_v4()),
-            force_delete_local_commit: false,
-            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
-            ..Default::default()
-        };
-        let journal_drained = || async {
-            tokio::time::timeout(Duration::from_secs(5), async {
-                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
-                    tokio::time::sleep(Duration::from_millis(10)).await;
-                }
-            })
-            .await
-            .is_ok()
-        };
-
-        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
-        for object in ["first", "second"] {
-            *shared.data.lock().expect("test data lock should not be poisoned") =
-                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
-            assert!(
-                journal_drained().await,
-                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
-            );
-        }
-        handle.abort();
-    }
-
-    #[tokio::test]
-    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
-        let shared = empty_resync_shared_state();
-        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
-        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
-        let interval = Duration::from_secs(1);
-        let persist_backlog = |object: &str| {
-            let entry = MrfReplicateEntry {
-                bucket: "mrf-node-replay".to_string(),
-                object: object.to_string(),
-                op: MrfOpKind::Delete,
-                force_delete_id: Some(uuid::Uuid::new_v4()),
-                force_delete_local_commit: false,
-                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
-                ..Default::default()
-            };
-            *shared.data.lock().expect("test data lock should not be poisoned") =
-                encode_mrf_file(&[entry]).expect("MRF entry should encode");
-        };
-        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
-
-        persist_backlog("first");
-        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
-        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
-
-        // Node B's timer fires within the same interval with a backlog present.
-        persist_backlog("second");
-        assert!(
-            !replay_mrf_journal(&node_b, Some(interval)).await,
-            "a node must not replay within one interval of another node's pass"
-        );
-        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
-
-        assert!(
-            replay_mrf_journal(&node_b, None).await,
-            "the startup pass should replay regardless of the shared stamp"
-        );
-        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
-
-        persist_backlog("third");
-        tokio::time::sleep(interval).await;
-        assert!(
-            replay_mrf_journal(&node_a, Some(interval)).await,
-            "a periodic pass should replay once the interval has elapsed"
-        );
-        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
-    }
-
-    #[test]
-    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
-        let interval = Duration::from_secs(300);
-        let now = 1_000_000_000;
-        assert!(mrf_replay_due(None, now, interval));
-        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
-        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
-        assert!(
-            !mrf_replay_due(Some(now + 1_000), now, interval),
-            "small clock skew should still space passes"
-        );
-        assert!(
-            mrf_replay_due(Some(now + 300_000), now, interval),
-            "a stamp an interval ahead must not suppress replay"
-        );
-    }
-
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            // A batch purge or another worker may have removed this version
-            // already. Remote failures still control the final outcome below.
-            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
-            if !source_already_purged {
-                error!(
-                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                    bucket = %bucket,
-                    arn = %dobj.target_arn,
-                    object = %dobj.delete_object.object_name,
-                    operation = "apply_replication_delete_state",
-                    error = %e,
-                    "Replication target operation failed"
-                );
-            }
+            error!(
+                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                arn = %dobj.target_arn,
+                object = %dobj.delete_object.object_name,
+                operation = "apply_replication_delete_state",
+                error = %e,
+                "Replication target operation failed"
+            );
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            source_already_purged
+            false
         }
     };
 
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
+/// purge intent as a durable MRF entry so the next startup replays it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; replay passes are serial background work, so
+/// round resolves the client; the replay loop is serial and startup-only, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
+use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
-use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
+type HmacSha256 = Hmac<Sha256>;
+
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
     BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
     CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
     DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
-    ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
-    PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
-    ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
-    RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
+    ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
+    MetadataReservationCancelRequest, MetadataReservationRequest,
+    PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
+    ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
+    RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
     RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
-    SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
-    SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
-    node_service_client::NodeServiceClient,
+    ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
+    SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
+    WriteMetadataRequest, node_service_client::NodeServiceClient,
 };
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
@@ -1637,7 +1638,6 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
-            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1709,7 +1709,6 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
-                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1780,7 +1779,6 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
-                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2012,7 +2010,6 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
-                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2027,7 +2024,6 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
-                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2457,6 +2453,194 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
 }
 
 impl RemoteDisk {
+    /// Prepare a remote token without retrying the RPC. It owns the bucket
+    /// publication fence but does not enter the object metadata queue yet.
+    pub async fn reserve_object_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        incarnation: Uuid,
+        timeout: Duration,
+    ) -> Result<Uuid> {
+        if timeout.is_zero() {
+            return Err(DiskError::Timeout);
+        }
+        self.execute_with_timeout_for_op_and_health_action(
+            "reserve_object_metadata",
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(MetadataReservationRequest {
+                    disk: self.endpoint.to_string(),
+                    bucket: bucket.to_owned(),
+                    object: object.to_owned(),
+                    timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
+                    bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
+                });
+                let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
+                    .map_err(|_| Error::other("reservation request length overflow"))?;
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
+                let response = client
+                    .reserve_object_metadata(request)
+                    .await
+                    .map_err(|status| {
+                        if status.code() == Code::Unimplemented {
+                            DiskError::UnsupportedDisk
+                        } else {
+                            status.into()
+                        }
+                    })?
+                    .into_inner();
+                if !response.success {
+                    return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
+                }
+                if response.protocol_version != 2 {
+                    return Err(DiskError::UnsupportedDisk);
+                }
+                Uuid::from_slice(&response.token)
+                    .ok()
+                    .filter(|token| !token.is_nil())
+                    .ok_or(DiskError::Unexpected)
+            },
+            timeout.min(Duration::from_secs(5)),
+            FailureHealthAction::IgnoreFailure,
+        )
+        .await
+    }
+
+    /// Enqueue a prepared token. The caller must cancel it on an indeterminate
+    /// result before allowing any subsequent unprotected mutation.
+    pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
+        if timeout.is_zero() {
+            return Err(DiskError::Timeout);
+        }
+        self.execute_with_timeout_for_op_and_health_action(
+            "activate_object_metadata_reservation",
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(MetadataReservationActivateRequest {
+                    disk: self.endpoint.to_string(),
+                    bucket: bucket.to_owned(),
+                    object: object.to_owned(),
+                    token: token.as_bytes().to_vec().into(),
+                });
+                let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
+                    .map_err(|_| Error::other("reservation activation request length overflow"))?;
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
+                let response = client
+                    .activate_object_metadata_reservation(request)
+                    .await
+                    .map_err(|status| {
+                        if status.code() == Code::Unimplemented {
+                            DiskError::UnsupportedDisk
+                        } else {
+                            status.into()
+                        }
+                    })?
+                    .into_inner();
+                if !response.success {
+                    return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
+                }
+                Ok(())
+            },
+            timeout.min(Duration::from_secs(5)),
+            FailureHealthAction::IgnoreFailure,
+        )
+        .await
+    }
+
+    /// Cancel an unclaimed remote reservation. False cannot prove that an
+    /// earlier claim did not start a physical mutation.
+    pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
+        self.execute_with_timeout_for_op(
+            "cancel_object_metadata_reservation",
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(MetadataReservationCancelRequest {
+                    disk: self.endpoint.to_string(),
+                    token: token.as_bytes().to_vec().into(),
+                });
+                let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
+                    .map_err(|_| Error::other("reservation cancellation request length overflow"))?;
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
+                let response = client
+                    .cancel_object_metadata_reservation(request)
+                    .await
+                    .map_err(|status| {
+                        if status.code() == Code::Unimplemented {
+                            DiskError::UnsupportedDisk
+                        } else {
+                            status.into()
+                        }
+                    })?
+                    .into_inner();
+                if let Some(error) = response.error {
+                    return Err(error.into());
+                }
+                Ok(response.success)
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    /// Publish a staged user-object write using a remote one-shot reservation.
+    /// The caller must retain the bucket lifecycle fence until the tail drains.
+    pub async fn rename_data_with_metadata_reservation(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        token: Uuid,
+    ) -> Result<RenameDataResp> {
+        self.execute_with_timeout_for_op(
+            "rename_reserved_data",
+            || async {
+                let mut client = self.get_client().await?;
+                let rename = RenameDataRequest {
+                    disk: self.endpoint.to_string(),
+                    src_volume: src_volume.to_owned(),
+                    src_path: src_path.to_owned(),
+                    file_info: compat_json(fi)?,
+                    dst_volume: dst_volume.to_owned(),
+                    dst_path: dst_path.to_owned(),
+                    file_info_bin: encode_file_info_msgpack(fi)?.into(),
+                    scanner_publication_lease_token: Bytes::new(),
+                    bucket_incarnation_id: Bytes::new(),
+                };
+                let mut request = Request::new(ReservedRenameDataRequest {
+                    rename: Some(rename),
+                    token: token.as_bytes().to_vec().into(),
+                });
+                let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
+                    .map_err(|_| Error::other("reserved rename request length overflow"))?;
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
+                let response = client
+                    .rename_reserved_data(request)
+                    .await
+                    .map_err(|status| {
+                        if status.code() == Code::Unimplemented {
+                            DiskError::UnsupportedDisk
+                        } else {
+                            status.into()
+                        }
+                    })?
+                    .into_inner();
+                if !response.success {
+                    return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
+                }
+                decode_msgpack_or_json::<RenameDataResp>(
+                    &response.rename_data_resp_bin,
+                    &response.rename_data_resp,
+                    "RenameDataResp",
+                )
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     #[tracing::instrument(level = "trace", skip_all)]
     pub(crate) async fn rename_data_borrowed(
         &self,
@@ -5114,6 +5298,56 @@ mod tests {
         peer.stop().await;
     }
 
+    #[tokio::test]
+    async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let token = Uuid::new_v4();
+        assert!(matches!(
+            remote
+                .reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
+                .await,
+            Err(DiskError::UnsupportedDisk)
+        ));
+        assert!(matches!(
+            remote
+                .activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
+                .await,
+            Err(DiskError::UnsupportedDisk)
+        ));
+        assert!(matches!(
+            remote.cancel_object_metadata_reservation(token).await,
+            Err(DiskError::UnsupportedDisk)
+        ));
+        assert!(matches!(
+            remote
+                .rename_data_with_metadata_reservation(
+                    crate::disk::RUSTFS_META_TMP_BUCKET,
+                    "staged",
+                    &sample_rename_data_file_info(),
+                    "bucket",
+                    "object",
+                    token,
+                )
+                .await,
+            Err(DiskError::UnsupportedDisk)
+        ));
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods lock"),
+            vec![
+                "/node_service.NodeService/ReserveObjectMetadata",
+                "/node_service.NodeService/ActivateObjectMetadataReservation",
+                "/node_service.NodeService/CancelObjectMetadataReservation",
+                "/node_service.NodeService/RenameReservedData",
+            ],
+            "reservation operations must never downgrade to legacy methods",
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
         runtime_sources::ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,10 +445,6 @@ impl RemoteClient {
                 }
             }
         });
-        // A successful lock RPC over a cached or fresh channel proves the peer is
-        // reachable; reconcile the internode peer-health gauge so it does not stay
-        // stale when no fresh dial fires (issue #9119).
-        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -676,7 +672,6 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
-                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;
@@ -1320,6 +1315,122 @@ impl LockClient for RemoteClient {
         Ok(resp.success)
     }
 
+    async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
+        let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
+        let mut request = Self::create_unlock_request(lock_id);
+        request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
+        let mut client = self.get_client().await?;
+        let resource_summary = request.resource.to_string();
+        let mut req = Request::new(GenerallyLockRequest {
+            args: serde_json::to_string(&request)
+                .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
+        });
+        attach_lock_mutation_body_digest(&mut req)?;
+        let response = self
+            .execute_rpc(
+                "refresh",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.refresh(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner();
+        if let Some(error) = response.error_info {
+            return Err(LockError::internal(error));
+        }
+        // Older nodes refresh normally but return no conversion marker.
+        Ok(response.success && response.lock_info.as_deref() == Some(marker))
+    }
+
+    async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
+        let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
+        let mut request = Self::create_unlock_request(lock_id);
+        request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
+        let mut client = self.get_client().await?;
+        let resource_summary = request.resource.to_string();
+        let mut req = Request::new(GenerallyLockRequest {
+            args: serde_json::to_string(&request)
+                .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
+        });
+        attach_lock_mutation_body_digest(&mut req)?;
+        let response = self
+            .execute_rpc(
+                "refresh",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.refresh(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner();
+        if let Some(error) = response.error_info {
+            return Err(LockError::internal(error));
+        }
+        Ok(response.success && response.lock_info.as_deref() == Some(marker))
+    }
+
+    async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
+        let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
+        let mut request = Self::create_unlock_request(lock_id);
+        request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
+        let mut client = self.get_client().await?;
+        let resource_summary = request.resource.to_string();
+        let mut req = Request::new(GenerallyLockRequest {
+            args: serde_json::to_string(&request)
+                .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
+        });
+        attach_lock_mutation_body_digest(&mut req)?;
+        let response = self
+            .execute_rpc(
+                "refresh",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.refresh(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner();
+        if let Some(error) = response.error_info {
+            return Err(LockError::internal(error));
+        }
+        Ok(response.success && response.lock_info.as_deref() == Some(marker))
+    }
+
+    async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
+        let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
+        let mut request = Self::create_unlock_request(lock_id);
+        request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
+        let mut client = self.get_client().await?;
+        let resource_summary = request.resource.to_string();
+        let mut req = Request::new(GenerallyLockRequest {
+            args: serde_json::to_string(&request)
+                .map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
+        });
+        attach_lock_mutation_body_digest(&mut req)?;
+        let response = self
+            .execute_rpc(
+                "refresh",
+                &resource_summary,
+                Self::rpc_timeout(),
+                async move { client.refresh(req).await },
+                RpcCleanup::new(None, None, None),
+            )
+            .await?
+            .into_inner();
+        if let Some(error) = response.error_info {
+            return Err(LockError::internal(error));
+        }
+        if !response.success {
+            return Ok(None);
+        }
+        Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
+            Some(":0") => Some(false),
+            Some(":1") => Some(true),
+            _ => None,
+        })
+    }
+
     async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
         info!("remote force_release for {}", lock_id);
         let force_request = Self::create_unlock_request(lock_id);

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
-use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
-
-async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
-    let fixture = fixture(0).await;
-    for pool in disks(&fixture.store).await {
-        for disk in pool {
-            let object = object_path(&disk);
-            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
-            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
-                if entry.file_type().await.expect("fixture entry type").is_dir() {
-                    fs::remove_dir_all(entry.path())
-                        .await
-                        .expect("remove task-owned fixture rollback directory");
-                }
-            }
-        }
-    }
-    fixture
-}
-
-async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
-    let mut all = Vec::new();
-    for pool in &store.pools {
-        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
-        let members = set
-            .disks
-            .read()
-            .await
-            .iter()
-            .cloned()
-            .map(|disk| disk.expect("identity fixture disk must be online"))
-            .collect::<Vec<_>>();
-        let mut heads = Vec::new();
-        for disk in members {
-            match disk
-                .pool_meta_native_recovery(Request::InspectIdentity {})
-                .await
-                .expect("physical identity inspection")
-            {
-                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
-                _ => panic!("identity inspection must return its physical head"),
-            }
-        }
-        all.push(heads);
-    }
-    all
-}
-
-async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
-    let fixture = owner_repair_no_backups_fixture().await;
-    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
-        PoolMetaReplica::Valid {
-            meta, committed: true, ..
-        } => meta,
-        _ => panic!("fixture must provide valid committed pool metadata"),
-    };
-    meta.version = version;
-    let raw = meta
-        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
-        .expect("legacy fixture should encode in its original version");
-    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
-    let all = disks(&fixture.store).await;
-    let now = OffsetDateTime::now_utc();
-    for pool in &all {
-        let heads = replicas(&raw, now).await;
-        for (index, disk) in pool.iter().enumerate() {
-            fs::write(object_path(disk).join("xl.meta"), &heads[index])
-                .await
-                .expect("install exact legacy head");
-        }
-    }
-    if stale {
-        let old = replicas(&raw, now - Duration::seconds(10)).await;
-        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
-            .await
-            .expect("install stale same-raw legacy head");
-    }
-    (fixture, raw)
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
-            )
-            .await
-            .expect("legacy native heal must terminate");
-            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
-            drop(state);
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            for read in &reads {
-                assert!(
-                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
-                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
-                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
-                );
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
-                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
-                let mut source_raw = raw.clone();
-                // A second MessagePack value is readable by legacy decoding but
-                // must not be copied as part of an unverified repair payload.
-                source_raw.push(0xc0);
-                assert!(
-                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
-                    "ordinary legacy reads must remain compatible with trailing bytes"
-                );
-                let all_disks = disks(&fixture.store).await;
-                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-                for (index, disk) in all_disks[0].iter().enumerate() {
-                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
-                        .await
-                        .expect("install readable legacy source with trailing bytes");
-                }
-                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
-                    .await
-                    .expect("legacy semantic selection must remain readable");
-                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
-                let before = snapshots(&fixture.store).await;
-                let identities = owner_repair_identity_heads(&fixture.store).await;
-                let guard = owner(&fixture.store).await;
-                let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                let result = tokio::time::timeout(
-                    std::time::Duration::from_secs(30),
-                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-                )
-                .await
-                .expect("legacy native heal must terminate");
-                drop(state);
-                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
-                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
-                assert_eq!(
-                    owner_repair_identity_heads(&fixture.store).await,
-                    identities,
-                    "version {version}: no identity shard may change"
-                );
-                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
-                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
-                for target_pool in [0, 1] {
-                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
-                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
-                        _ => panic!("initial V1 metadata must decode"),
-                    };
-                    meta.version = POOL_META_VERSION;
-                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
-                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
-                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
-                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
-                        PoolMetaReplica::Valid { canonical, .. } => canonical,
-                        _ => panic!("V1 payload must remain valid"),
-                    };
-                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
-                        PoolMetaReplica::Valid { canonical, .. } => canonical,
-                        _ => panic!("V2 payload must remain valid"),
-                    };
-                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
-                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
-                    // Pool two follows pool zero; pools zero and one cover both
-                    // orderings and both target formats in separate runs.
-                    let versions = [first_version, other_version, first_version];
-                    let physical = disks(&fixture.store).await;
-                    let timestamp = OffsetDateTime::now_utc();
-                    for (pool_index, pool) in physical.iter().enumerate() {
-                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
-                        let heads = replicas(raw, timestamp).await;
-                        for (index, disk) in pool.iter().enumerate() {
-                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
-                        }
-                    }
-                    let before = snapshots(&fixture.store).await;
-                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
-                        .await.expect("equivalent legacy pools should be selectable");
-                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
-                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
-                    let guard = owner(&fixture.store).await;
-                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                    tokio::time::timeout(std::time::Duration::from_secs(30),
-                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
-                        .await.expect("mixed legacy heal must terminate")
-                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
-                    drop(state);
-                    let after = snapshots(&fixture.store).await;
-                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                    for (pool_index, read) in reads.iter().enumerate() {
-                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
-                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
-                        assert!(matches!(&read.replica,
-                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
-                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
-                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
-                        for (index, snapshot) in after[pool_index].iter().enumerate() {
-                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
-                                .await.expect("every physical shard must match its selected legacy payload");
-                        }
-                        if pool_index != target_pool {
-                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
-                        }
-                    }
-                }
-            }
-        },
-    ).await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            for prepare_pool in [0, 1] {
-                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
-                    PoolMetaReplica::Valid {
-                        meta,
-                        canonical,
-                        revision,
-                        committed: true,
-                        ..
-                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
-                    _ => panic!("baseline V1 metadata must be committed and readable"),
-                };
-                let previous = PoolMetaCommittedCandidate {
-                    canonical: legacy_canonical,
-                    meta: legacy_meta.clone(),
-                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
-                };
-                let mut next = legacy_meta;
-                next.version = POOL_META_GENERATION_VERSION;
-                next.pools[0].last_update += Duration::seconds(1);
-                let revision = PoolMetaRevision {
-                    version: POOL_META_GENERATION_VERSION,
-                    cluster_id: fixture.revision.cluster_id,
-                    epoch: fixture.revision.epoch,
-                    generation: 1,
-                    transaction_id: Some(Uuid::new_v4()),
-                };
-                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
-                    .expect("V3 PREPARE with V1 predecessor must encode");
-                assert!(matches!(decode_pool_meta_replica(pending.clone()),
-                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
-                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-                let physical = disks(&fixture.store).await;
-                for (index, disk) in physical[prepare_pool].iter().enumerate() {
-                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
-                        .await
-                        .expect("install task-owned V3 PREPARE evidence");
-                }
-                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-                assert!(matches!(&reads[prepare_pool].replica,
-                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
-                        if raw == &pending && *actual == revision));
-                let other_pool = 1 - prepare_pool;
-                assert!(matches!(&reads[other_pool].replica,
-                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
-                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
-                let before = snapshots(&fixture.store).await;
-                let identities = owner_repair_identity_heads(&fixture.store).await;
-                let guard = owner(&fixture.store).await;
-                let mut state = fixture.store.pool_meta_save_gate.lock().await;
-                let result = tokio::time::timeout(
-                    std::time::Duration::from_secs(30),
-                    fixture
-                        .store
-                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
-                )
-                .await
-                .expect("mixed V1/V3 PREPARE heal must terminate");
-                drop(state);
-                assert!(
-                    result.is_err(),
-                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
-                );
-                assert_eq!(
-                    snapshots(&fixture.store).await,
-                    before,
-                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
-                );
-                assert_eq!(
-                    owner_repair_identity_heads(&fixture.store).await,
-                    identities,
-                    "pool {prepare_pool}: rejection must not mutate identity evidence"
-                );
-            }
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    meta, committed: true, ..
-                } => meta,
-                _ => panic!("baseline V1 metadata must decode"),
-            };
-            let statuses = meta
-                .pools
-                .iter()
-                .map(|pool| {
-                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
-                    LegacyPoolStatus {
-                        id: pool.id,
-                        cmd_line: pool.cmd_line.clone(),
-                        last_update: pool.last_update,
-                        decommission: None,
-                    }
-                })
-                .collect();
-            let legacy = LegacyPoolMeta {
-                version: POOL_META_V1_VERSION,
-                pools: statuses,
-                dont_save: true,
-            };
-            let mut dangerous_raw = Vec::new();
-            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
-            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
-            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
-            let old_reader: LegacyPoolMeta =
-                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
-            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
-            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
-            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
-                PoolMetaReplica::Valid {
-                    canonical,
-                    revision,
-                    committed: true,
-                    ..
-                } => {
-                    assert_eq!(revision.version, POOL_META_V1_VERSION);
-                    canonical
-                }
-                _ => panic!("current reader must recognize the legacy source"),
-            };
-            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
-                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
-                    if canonical == source && revision.version == POOL_META_V1_VERSION));
-            let physical = disks(&fixture.store).await;
-            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-            for (index, disk) in physical[0].iter().enumerate() {
-                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
-                    .await
-                    .expect("install legacy source on pool zero");
-            }
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
-            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
-            let before = snapshots(&fixture.store).await;
-            let identities = owner_repair_identity_heads(&fixture.store).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-            )
-            .await
-            .expect("unsafe legacy heal must terminate");
-            drop(state);
-            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
-            assert!(
-                error.to_string().contains("dont_save"),
-                "rejection must identify the unsafe legacy field: {error}"
-            );
-            assert_eq!(
-                snapshots(&fixture.store).await,
-                before,
-                "rejection must preserve every original head and backup"
-            );
-            assert_eq!(
-                owner_repair_identity_heads(&fixture.store).await,
-                identities,
-                "rejection must not write identity shards"
-            );
-            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
-            assert!(
-                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
-                "the dangerous source bytes must not be copied into the target pool"
-            );
-        },
-    )
-    .await;
-}
-
-#[tokio::test]
-#[serial_test::serial(pool_meta_version_env)]
-async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
-    temp_env::async_with_vars(
-        [
-            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
-            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
-        ],
-        async {
-            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
-            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    meta, committed: true, ..
-                } => meta,
-                _ => panic!("baseline V1 metadata must decode"),
-            };
-            let statuses = meta
-                .pools
-                .iter()
-                .map(|pool| {
-                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
-                    LegacyPoolStatus {
-                        id: pool.id,
-                        cmd_line: pool.cmd_line.clone(),
-                        last_update: pool.last_update,
-                        decommission: None,
-                    }
-                })
-                .collect();
-            let legacy = LegacyPoolMeta {
-                version: POOL_META_V1_VERSION,
-                pools: statuses,
-                dont_save: true,
-            };
-            let mut dangerous_raw = Vec::new();
-            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
-            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
-            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
-            let old_reader: LegacyPoolMeta =
-                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
-            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
-            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
-            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
-                PoolMetaReplica::Valid {
-                    canonical,
-                    revision,
-                    committed: true,
-                    ..
-                } => {
-                    assert_eq!(revision.version, POOL_META_V1_VERSION);
-                    canonical
-                }
-                _ => panic!("safe pool-zero source must decode"),
-            };
-            assert!(
-                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
-                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
-                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
-                "the dangerous target must remain semantically equivalent to the safe source"
-            );
-            let physical = disks(&fixture.store).await;
-            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
-            for (index, disk) in physical[1].iter().enumerate() {
-                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
-                    .await
-                    .expect("install legacy dangerous target head");
-            }
-            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
-            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
-            let before = snapshots(&fixture.store).await;
-            let identities = owner_repair_identity_heads(&fixture.store).await;
-            let guard = owner(&fixture.store).await;
-            let mut state = fixture.store.pool_meta_save_gate.lock().await;
-            let result = tokio::time::timeout(
-                std::time::Duration::from_secs(30),
-                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
-            )
-            .await
-            .expect("unsafe legacy target heal must terminate");
-            drop(state);
-            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
-            assert!(
-                error.to_string().contains("dont_save"),
-                "rejection must identify the unsafe legacy field: {error}"
-            );
-            assert_eq!(
-                snapshots(&fixture.store).await,
-                before,
-                "rejection must preserve all physical heads and backups"
-            );
-            assert_eq!(
-                owner_repair_identity_heads(&fixture.store).await,
-                identities,
-                "rejection must not write identity shards"
-            );
-            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
-            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
-            assert!(
-                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
-                "unsafe target evidence must not be overwritten by the safe source"
-            );
-        },
-    )
-    .await;
-}

crates/ecstore/src/core/pools.rs

@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
+#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -3719,45 +3720,33 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
     data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
 }
 
-/// A copy whose object namespace lock remained unavailable through its
-/// in-place retries, retained as a typed deferral.
+/// The storage error of a version copy that kept timing out on a namespace
+/// lock through its in-place retries, kept typed so the entry failure policy
+/// defers it.
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
-struct DecommissionCopyLockUnavailable(#[source] Error);
+struct DecommissionCopyLockTimeout(#[source] Error);
 
-fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
-    let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
-    match source {
-        Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
-        Error::NamespaceLockQuorumUnavailable {
-            mode: "read",
-            bucket: lock_bucket,
-            object: lock_object,
-            ..
-        } => {
-            data_movement::data_movement_error_is_stage(err, "prepare_put_object")
-                && lock_bucket == bucket
-                && lock_object == object
-        }
-        _ => false,
-    }
+fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
+    let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
+    matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
 }
 
-/// A version copy that could not acquire its namespace lock leaves the source
+/// A version copy that timed out acquiring a namespace lock leaves the source
 /// in place; the next attempt copies the version again or finds an equivalent
 /// committed target. The lock is held by another mutation of the key or, when
 /// this node restarted mid-copy, by its previous incarnation until that lock's
 /// lease expires on the peers. Either clears by waiting, so the entry is
 /// deferred instead of being counted as a permanent per-object failure.
-fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
+fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement::data_movement_context_error(
-        format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
-        DecommissionCopyLockUnavailable(err),
+        format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
+        DecommissionCopyLockTimeout(err),
     )
 }
 
-fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
-    data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
+fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
+    data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
 }
 
 /// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3767,8 +3756,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
         return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
     }
     error
-        .take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
-        .map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
+        .take_if(|err| is_decommission_copy_lock_timeout_error(err))
+        .map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
 }
 
 fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5833,13 +5822,13 @@ enum DecommissionEntryOutcome {
 /// entry is done and must not be counted as a failure or escalated. A quorum
 /// failure of the source cleanup is deferred too: the copied source is still
 /// in place and the next attempt cleans it up again. So is a version copy that
-/// lost drive quorum or could not acquire an object namespace lock through its in-place
+/// lost drive quorum or timed out on a namespace lock through its in-place
 /// retries: the source still holds the version and the next attempt copies it
 /// again. Every other failure stays terminal.
 fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
     if is_decommission_source_cleanup_retryable_error(&err)
         || is_decommission_copy_quorum_deferral(&err)
-        || is_decommission_copy_lock_unavailable_deferral(&err)
+        || is_decommission_copy_lock_timeout_deferral(&err)
     {
         return Ok(DecommissionEntryOutcome::Deferred(err));
     }
@@ -6838,47 +6827,11 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
-fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
-    if data.len() >= 4
-        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
-        // The selection already validates the payload. The runtime-only
-        // flag can only appear in the legacy three-field sequence or map.
-        && rmp::decode::read_array_len(&mut &data[4..])
-            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
-            .ok()
-            == Some(3)
-        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
-        && legacy.dont_save
-    {
-        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
-    }
-    Ok(())
-}
-
-// Ordinary legacy reads retain their compatibility behavior, but bytes that
-// will be copied to another pool must not contain an unparsed extension.
-fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
-    let payload = data
-        .get(4..)
-        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
-    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
-    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
-    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
-    if decoder.get_ref().position() != payload_len {
-        return Err(Error::other("pool metadata legacy source has trailing bytes"));
-    }
-    Ok(())
-}
-
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
-    // Legacy semantic comparison canonicalizes to V2; a heal must instead
-    // preserve verified raw bytes at the highest observed format floor.
-    legacy_raw: Option<Vec<u8>>,
-    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7269,8 +7222,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
-            legacy_raw: None,
-            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7325,8 +7276,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
-            legacy_raw: None,
-            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7340,10 +7289,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
-    let legacy_has_unsafe_flag = committed
-        .iter()
-        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
-    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7357,36 +7302,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
-                if floor_raw.is_none()
-                    && candidate.revision.version == observed_version
-                    && raw
-                        .get(2..4)
-                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
-                {
-                    floor_raw = Some(raw);
-                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, selected_raw, mut selected) =
+    let (_, _, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
-    if selected.revision.version == observed_version
-        && selected_raw
-            .get(2..4)
-            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
-    {
-        floor_raw = Some(selected_raw);
-    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
-        legacy_raw: floor_raw,
-        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -14606,23 +14534,6 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
-        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
-            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
-        }
-        if selection.legacy_has_unsafe_flag {
-            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
-        }
-        let repair_data = if selection.revision.is_generation_protocol() {
-            canonical.clone()
-        } else {
-            let raw = selection
-                .legacy_raw
-                .clone()
-                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
-            ensure_legacy_pool_meta_payload_complete(&raw)?;
-            raw
-        };
-        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14633,10 +14544,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != repair_data {
+        if confirmed != *canonical {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -28442,7 +28353,7 @@ mod tests {
             "one version must commit before the next version times out"
         );
         assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
             "{outcome:?}"
         );
         assert_eq!(
@@ -28476,88 +28387,6 @@ mod tests {
         assert_eq!(actual, b"written after versioning was enabled");
     }
 
-    /// A target lookup that loses its read-lock quorum must retain the source
-    /// and resume an entry after an earlier version has already committed.
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
-        use tokio::io::AsyncReadExt as _;
-
-        let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
-        let object = fixture.object;
-        let attempted = Arc::new(AtomicUsize::new(0));
-        let fault = DecommissionTestFaultGuard::install_with_error(
-            Arc::new({
-                let attempted = Arc::clone(&attempted);
-                move |stage, _, faulted, _, success| {
-                    stage == "before_target_commit"
-                        && faulted == object
-                        && success
-                        && attempted.fetch_add(1, Ordering::SeqCst) > 0
-                }
-            }),
-            Some(Arc::new(|_, bucket, object| {
-                data_movement::data_movement_stage_error_for_test(
-                    "decommission_object",
-                    "prepare_put_object",
-                    bucket,
-                    object,
-                    Error::NamespaceLockQuorumUnavailable {
-                        mode: "read",
-                        bucket: bucket.to_owned(),
-                        object: object.to_owned(),
-                        required: 1,
-                        achieved: 0,
-                    },
-                )
-            })),
-        );
-        let (outcome, _) = fixture.run_entry().await;
-        assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
-            "{outcome:?}"
-        );
-        fixture.assert_source_retained().await;
-        assert_eq!(
-            fixture
-                .target_versions()
-                .await
-                .expect("first version committed")
-                .versions
-                .len(),
-            1
-        );
-        assert_eq!(
-            fixture.store.pool_meta.read().await.pools[0]
-                .decommission
-                .as_ref()
-                .expect("active decommission")
-                .items_decommission_failed,
-            0
-        );
-
-        drop(fault);
-        let (outcome, _) = fixture.run_entry().await;
-        assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
-        fixture.assert_moved().await;
-        let mut reader = fixture.store.pools[1]
-            .get_object_reader(
-                fixture.bucket,
-                fixture.object,
-                None,
-                HeaderMap::new(),
-                &ObjectOptions {
-                    version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("the newer version must remain readable");
-        let mut actual = Vec::new();
-        reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
-        assert_eq!(actual, b"written after versioning was enabled");
-    }
-
     /// A namespace lock still held on the target key, as by a copy whose node
     /// was killed mid-write, makes target selection fail with a typed
     /// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28600,7 +28429,7 @@ mod tests {
 
             let (outcome, _) = fixture.run_entry().await;
             assert!(
-                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
                 "{outcome:?}"
             );
             assert_eq!(
@@ -29009,17 +28838,17 @@ mod tests {
             timeout(),
         );
         for err in [timeout(), prepare] {
-            assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
+            assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
             assert!(!is_decommission_copy_quorum_error(&err), "{err}");
             let mut error = Some(err);
             let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
                 .expect("a lock timeout copy must be deferred");
             assert!(error.is_none());
-            assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
+            assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
             assert!(!is_decommission_copy_quorum_deferral(&deferred));
-            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
+            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
                 .expect("the deferred error must keep its lock-timeout origin");
-            assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
+            assert!(is_decommission_copy_lock_timeout_error(&failure.0));
             assert!(matches!(
                 resolve_decommission_entry_failure(deferred),
                 Ok(DecommissionEntryOutcome::Deferred(_))
@@ -29044,43 +28873,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
-        let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
-            mode,
-            bucket: bucket.to_owned(),
-            object: object.to_owned(),
-            required: 1,
-            achieved: 0,
-        };
-        let wrap = |stage, err| {
-            data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
-        };
-        let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
-        let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
-            .expect("matching read-lock quorum must defer");
-        assert!(error.is_none());
-        assert!(matches!(
-            resolve_decommission_entry_failure(deferred),
-            Ok(DecommissionEntryOutcome::Deferred(_))
-        ));
-        for err in [
-            quorum("read", "bucket-a", "object-a"),
-            wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
-            wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
-            wrap(
-                "prepare_put_object",
-                quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
-            ),
-            wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
-            wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
-        ] {
-            let mut error = Some(err);
-            assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
-            assert!(error.is_some(), "unrelated lock failures must stay terminal");
-        }
-    }
-
     #[test]
     fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
         let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29559,53 +29351,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
-        let frame = |payload: Vec<u8>| {
-            let mut data = Vec::new();
-            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
-            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
-            data.extend_from_slice(&payload);
-            data
-        };
-        let two_fields = PersistedPoolMetaV1 {
-            version: POOL_META_V1_VERSION,
-            pools: Vec::new(),
-        };
-        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
-            let data = frame(payload.expect("two-field V1 payload must serialize"));
-            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
-            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
-            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
-            let mut trailing = data;
-            trailing.push(0xc0);
-            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
-        }
-        for dont_save in [false, true] {
-            let legacy = LegacyPoolMeta {
-                version: POOL_META_V1_VERSION,
-                pools: Vec::new(),
-                dont_save,
-            };
-            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
-                let data = frame(payload.expect("three-field V1 payload must serialize"));
-                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
-                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
-                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
-                let mut trailing = data;
-                trailing.push(0xc0);
-                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
-            }
-        }
-        // This guard is only called for already-decoded replicas; malformed
-        // containers must fail selection rather than becoming repair sources.
-        let truncated = frame(vec![0x93, 0x01]);
-        assert!(matches!(
-            decode_pool_meta_replica(truncated),
-            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
-        ));
-    }
-
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();

crates/ecstore/src/core/sets.rs

@@ -655,6 +655,19 @@ impl Sets {
             .await
     }
 
+    pub(crate) async fn put_object_with_prepared_reservations(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut PutObjReader,
+        opts: &ObjectOptions,
+        reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
+    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
+        self.get_disks_by_key(object)
+            .put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
+            .await
+    }
+
     pub(crate) async fn put_object_with_old_current_size_for_data_movement(
         &self,
         bucket: &str,

crates/ecstore/src/data_movement/mod.rs

@@ -605,7 +605,6 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
-    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -627,7 +626,6 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
-        stage: None,
         source: Box::new(err),
     })
 }
@@ -636,21 +634,7 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    Error::other(DataMovementStageError {
-        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
-        stage: Some(stage.to_owned()),
-        source: Box::new(err),
-    })
-}
-
-pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
-    let Error::Io(io_err) = err else {
-        return false;
-    };
-    io_err
-        .get_ref()
-        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
-        .is_some_and(|source| source.stage.as_deref() == Some(stage))
+    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/disk_store.rs

@@ -604,8 +604,69 @@ impl LocalDiskWrapper {
         let super::RenameDataGuards {
             external_guard,
             namespace_owner,
+            metadata_reservation,
+            metadata_reservation_token,
             ..
         } = guards;
+        let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
+            (Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
+            (Some(reservation), None) => Some(reservation),
+            (None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
+                Ok(reservation) => Some(reservation),
+                // A missing token may already have been claimed by an earlier
+                // request that is still publishing; do not claim it was safe.
+                Err(DiskError::DiskOngoingReq) => {
+                    return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
+                }
+                Err(error) => return super::RenameDataObservation::not_dispatched(error),
+            },
+            (None, None) => None,
+        };
+        // A queued reservation must acquire its metadata lease before disk
+        // admission. Otherwise a waiting background mutation could hold the
+        // disk permit needed by the earlier lease owner.
+        let metadata_reservation = match metadata_reservation {
+            Some(reservation) => Some(reservation.wait().await),
+            None => None,
+        };
+        // The claimed reservation stays in this cancellable future while disk
+        // admission can wait. Only physical work moves into an owned task.
+        let prechecked_permit = if metadata_reservation.is_some() {
+            // Background admission is exclusive. Waiting for it while holding
+            // this object's lease can deadlock an admitted mutation waiting for
+            // the same lease. An already owned permit does not need to wait.
+            if current_disk_io_context().is_some_and(|context| {
+                context.class == DiskIoClass::Background
+                    && !context
+                        .permit
+                        .as_ref()
+                        .is_some_and(|permit| permit.is_for(&self.io_admission))
+            }) {
+                return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
+            }
+            let preflight = self
+                .with_native_resources(async {
+                    if self.health.is_faulty() {
+                        return Err(DiskError::FaultyDisk);
+                    }
+                    self.check_disk_stale().await?;
+                    self.acquire_disk_io_permit().await
+                })
+                .await;
+            match preflight {
+                Ok(Ok(Some(permit))) => Some(permit),
+                Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
+                Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
+            }
+        } else {
+            None
+        };
+        // The reserved metadata lease must outlive a cancelled RPC waiter.
+        let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
+            Some(Arc::new(external_guard))
+        } else {
+            external_guard
+        };
         let operation = self.clone();
         let src_volume = src_volume.to_owned();
         let src_path = src_path.to_owned();
@@ -624,27 +685,54 @@ impl LocalDiskWrapper {
         };
         let observed = run_owned_mutation(external_guard, move || async move {
             let mut preflight_rejection = None;
-            let result = operation
-                .track_disk_health_mutation(
-                    "rename_data",
-                    DiskMetricMutation::Write,
-                    || async {
-                        // Preserve the former DiskAPI future's single boxing boundary.
-                        let observed = Box::pin(operation.disk.rename_data_observed(
-                            &src_volume,
-                            &src_path,
-                            &fi,
-                            &dst_volume,
-                            &dst_path,
-                            namespace_owner,
+            let rename = || async {
+                // Preserve the former DiskAPI future's single boxing boundary.
+                let observed = if let Some(reservation) = metadata_reservation {
+                    Box::pin(operation.disk.rename_data_observed_with_reservation(
+                        &src_volume,
+                        &src_path,
+                        &fi,
+                        &dst_volume,
+                        &dst_path,
+                        (namespace_owner, Some(reservation)),
+                    ))
+                    .await
+                } else {
+                    Box::pin(operation.disk.rename_data_observed(
+                        &src_volume,
+                        &src_path,
+                        &fi,
+                        &dst_volume,
+                        &dst_path,
+                        namespace_owner,
+                    ))
+                    .await
+                };
+                preflight_rejection = observed.preflight_rejection;
+                observed.result
+            };
+            let result = if let Some(permit) = prechecked_permit {
+                with_disk_io_permit(Some(permit), async {
+                    operation
+                        .with_native_resources(operation.track_admitted_disk_operation(
+                            "rename_data",
+                            move |io_permit| async move {
+                                let _io_permit = io_permit;
+                                rename().await
+                            },
+                            Duration::ZERO,
+                            TimeoutHealthAction::MarkFailure,
+                            DiskMetricMutation::Write,
+                            true,
                         ))
-                        .await;
-                        preflight_rejection = observed.preflight_rejection;
-                        observed.result
-                    },
-                    timeout_duration,
-                )
-                .await;
+                        .await?
+                })
+                .await
+            } else {
+                operation
+                    .track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
+                    .await
+            };
             // Health tracking must observe the real disk error, not an Ok tuple.
             Ok(super::RenameDataObservation {
                 result,
@@ -853,8 +941,6 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
-    #[cfg(test)]
-    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -882,8 +968,6 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
-            #[cfg(test)]
-            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1577,20 +1661,6 @@ impl Default for DiskHealthTracker {
     }
 }
 
-#[cfg(test)]
-#[derive(Debug)]
-pub(crate) struct ScopedDiskOperationTimeout {
-    metrics: Arc<DiskHealthMetricEpoch>,
-    previous: Option<Duration>,
-}
-
-#[cfg(test)]
-impl Drop for ScopedDiskOperationTimeout {
-    fn drop(&mut self) {
-        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
-    }
-}
-
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1616,6 +1686,58 @@ pub struct LocalDiskWrapper {
 }
 
 impl LocalDiskWrapper {
+    /// Reserve the metadata mutation lock for one object on this disk instance.
+    /// Unclaimed reservations expire after 60 seconds.
+    pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.reserve_object_metadata(bucket, object, timeout).await
+    }
+
+    /// Register an object metadata reservation in FIFO order on this disk.
+    /// The returned token may be claimed before its physical lease is available.
+    pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.enqueue_object_metadata(bucket, object).await
+    }
+
+    /// Prepare an inert reservation token bound to this disk and object.
+    pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
+        self.prepare_object_metadata_with_owner(bucket, object, None).await
+    }
+
+    pub(crate) async fn prepare_object_metadata_with_owner(
+        &self,
+        bucket: &str,
+        object: &str,
+        owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
+    ) -> Result<Uuid> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
+    }
+
+    /// Enqueue a prepared token; a cancelled token cannot be reactivated.
+    pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.activate_object_metadata(bucket, object, token).await
+    }
+
+    /// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
+    pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
+        self.disk.cancel_object_metadata_reservation(token)
+    }
+
     /// Create a new LocalDiskWrapper
     pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
         let admission = DiskIoAdmission::new(disk.endpoint().to_string());
@@ -1932,23 +2054,6 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
-    #[cfg(test)]
-    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
-        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
-        // The metrics epoch follows this drive across clones and reconnects.
-        // Its scoped override cannot shorten unrelated healthy drives' I/O.
-        let previous = self
-            .metrics
-            .operation_timeout_override
-            .lock()
-            .expect("test timeout lock")
-            .replace(timeout);
-        ScopedDiskOperationTimeout {
-            metrics: Arc::clone(&self.metrics),
-            previous,
-        }
-    }
-
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2433,10 +2538,6 @@ impl LocalDiskWrapper {
         .await
     }
 
-    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
-        self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
-    }
-
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
@@ -2616,6 +2717,7 @@ impl LocalDiskWrapper {
             timeout_duration,
             timeout_health_action,
             mutation,
+            false,
         ))
         .await?
     }
@@ -2627,6 +2729,7 @@ impl LocalDiskWrapper {
         timeout_duration: Duration,
         timeout_health_action: TimeoutHealthAction,
         mutation: DiskMetricMutation,
+        identity_prechecked: bool,
     ) -> Result<T>
     where
         F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
@@ -2648,7 +2751,7 @@ impl LocalDiskWrapper {
         }
 
         // Check if disk is stale
-        if let Err(err) = self.check_disk_stale().await {
+        if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
             self.metrics.record_availability_error();
             return Err(err);
         }
@@ -2673,13 +2776,6 @@ impl LocalDiskWrapper {
             }
             return result;
         }
-        #[cfg(test)]
-        let timeout_duration = self
-            .metrics
-            .operation_timeout_override
-            .lock()
-            .expect("test timeout lock")
-            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -5623,6 +5719,115 @@ mod tests {
         drop(metadata_lease);
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
+        use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
+
+        let dir = tempfile::tempdir().expect("disk directory");
+        let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(0);
+        let meta = dir.path().join(RUSTFS_META_BUCKET);
+        tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
+        let mut format = FormatV3::new(1, 1);
+        format.erasure.this = format.erasure.sets[0][0];
+        tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
+            .await
+            .expect("write format");
+        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
+        disk.make_volume("bucket").await.expect("bucket volume");
+        let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
+        let token = wrapper
+            .reserve_object_metadata("bucket", "object", Duration::from_secs(5))
+            .await
+            .expect("reserve key");
+        wrapper.set_disk_id_state(Some(format.erasure.this)).await;
+        disk.format_info.write().await.last_check = None;
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel();
+        disk.install_format_probe_hook(false, move || {
+            let _ = entered_tx.send(());
+            let _ = release_rx.recv_timeout(Duration::from_secs(30));
+        });
+        let publisher = Arc::clone(&wrapper);
+        let rename = tokio::spawn(async move {
+            publisher
+                .rename_data_observed_with_guards(
+                    RUSTFS_META_TMP_BUCKET,
+                    "staging",
+                    &FileInfo::default(),
+                    "bucket",
+                    "object",
+                    RenameDataGuards {
+                        metadata_reservation_token: Some(token),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(10), entered_rx)
+            .await
+            .expect("disk identity preflight starts")
+            .expect("preflight hook");
+        rename.abort();
+        assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
+        let key = disk.get_object_path("bucket", "object").expect("object path");
+        let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
+        let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
+        release_tx.send(()).expect("release disk identity preflight");
+        if !released_before_io {
+            drop(
+                tokio::time::timeout(Duration::from_secs(10), successor)
+                    .await
+                    .expect("preflight finishes"),
+            );
+        }
+        assert!(
+            released_before_io,
+            "a cancelled request must not retain the reservation before physical mutation starts"
+        );
+    }
+
+    #[tokio::test]
+    async fn reserved_background_rename_does_not_wait_for_disk_admission() {
+        use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
+
+        let dir = tempfile::tempdir().expect("disk directory");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
+        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
+        let token = wrapper
+            .reserve_object_metadata("bucket", "object", Duration::from_secs(5))
+            .await
+            .expect("reserve key");
+        let held = wrapper
+            .io_admission
+            .background(&CancellationToken::new())
+            .await
+            .expect("background permit");
+        let fi = FileInfo::default();
+        let rename = crate::disk::with_background_disk_io(
+            CancellationToken::new(),
+            wrapper.rename_data_observed_with_guards(
+                RUSTFS_META_TMP_BUCKET,
+                "staging",
+                &fi,
+                "bucket",
+                "object",
+                RenameDataGuards {
+                    metadata_reservation_token: Some(token),
+                    ..Default::default()
+                },
+            ),
+        );
+        let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
+        drop(held);
+        let observation = result.expect("reserved mutation must not wait while holding the object lease");
+        assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
+        assert!(observation.rejected_before_publication());
+    }
+
     #[tokio::test]
     async fn delete_versions_counts_faulty_drive_availability_rejection() {
         let dir = tempfile::tempdir().expect("temp dir should be created");
@@ -6086,67 +6291,6 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
-    #[tokio::test]
-    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
-        let directories = [
-            tempfile::tempdir().expect("faulted drive"),
-            tempfile::tempdir().expect("healthy drive"),
-        ];
-        let mut wrappers = Vec::new();
-        for directory in &directories {
-            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
-            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
-            wrappers.push(LocalDiskWrapper::new(disk, false));
-        }
-        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
-            Arc::clone(&wrappers[0].disk),
-            false,
-            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
-        );
-        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
-        tokio::time::pause();
-        let operation = || async {
-            tokio::time::sleep(Duration::from_secs(2)).await;
-            Ok(())
-        };
-        let (faulted, healthy) = tokio::join!(
-            reconnected.track_disk_health_with_op_and_timeout_action(
-                "read_metadata",
-                operation,
-                Duration::from_secs(3),
-                TimeoutHealthAction::IgnoreFailure,
-            ),
-            wrappers[1].track_disk_health_with_op_and_timeout_action(
-                "read_metadata",
-                operation,
-                Duration::from_secs(3),
-                TimeoutHealthAction::IgnoreFailure,
-            ),
-        );
-        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
-        healthy.expect("healthy drive keeps its caller's deadline");
-        let shorter_started = tokio::time::Instant::now();
-        let shorter = wrappers[0]
-            .track_disk_health_with_op_and_timeout_action(
-                "read_metadata",
-                operation,
-                Duration::from_millis(500),
-                TimeoutHealthAction::IgnoreFailure,
-            )
-            .await;
-        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
-        assert!(shorter_started.elapsed() < Duration::from_secs(1));
-        wrappers[0]
-            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
-            .await
-            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
-        drop(scope);
-        reconnected
-            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
-            .await
-            .expect("dropping the scope restores reconnected callers' deadlines");
-    }
-
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
-use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry {
-        file,
-        len: 3,
-        invalidated: AtomicBool::new(false),
-    });
+    let entry = Arc::new(FdCacheEntry { file, len: 3 });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume invalidation"
+            "new entries must survive the earlier volume predicate"
         );
     }
     cache.clear();
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
-        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);

crates/ecstore/src/disk/local/commit.rs

@@ -184,8 +184,6 @@ pub(super) fn create_local_inline_rollback_backup(
     remove_file_if_exists(&backup_path)?;
     if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
         let write_res = (|| -> std::io::Result<()> {
-            #[cfg(test)]
-            run_owned_file_before_open(&backup_path);
             let mut file = std::fs::File::create(&backup_path)?;
             std::io::Write::write_all(&mut file, old_metadata)?;
             if sync && !old_metadata.is_empty() {
@@ -369,12 +367,9 @@ fn prepare_rename_commit_and_destination_metadata_std(
 
 #[cfg(target_os = "linux")]
 enum FusedCommitAttempt {
-    Committed(Box<RenameDataResp>),
+    Committed(RenameDataResp),
     CommitErr(DiskError),
-    RestoreDataSource(os::RenameCommitGuard, DiskError),
-    Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
-    #[cfg(test)]
-    ResumePanic(Box<dyn std::any::Any + Send + 'static>),
+    Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
 }
 
 async fn prepare_rename_commit_and_destination_metadata(
@@ -444,12 +439,29 @@ async fn prepare_rename_commit_and_destination_metadata(
 #[derive(Debug)]
 pub(in crate::disk) struct LocalRenamePreflightRejection(());
 
+impl LocalRenamePreflightRejection {
+    pub(in crate::disk) fn not_dispatched() -> Self {
+        Self(())
+    }
+}
+
 #[derive(Default)]
 pub(super) struct RenameDataState {
     namespace_owner: Option<Arc<dyn Send + Sync>>,
+    metadata_reservation: Option<os::MetadataMutationReservation>,
     preflight_rejection: Option<LocalRenamePreflightRejection>,
 }
 
+#[cfg(test)]
+impl RenameDataState {
+    pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
+        Self {
+            metadata_reservation: Some(reservation),
+            ..Default::default()
+        }
+    }
+}
+
 // Keep optional native MPU state machines out of ordinary rename frames.
 // Construct the future here as well as pinning it, so its temporary stays off
 // the caller's poll stack. Call only from the existing native MPU branches.
@@ -491,6 +503,15 @@ impl LocalDisk {
             wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
         }
         let mut fi = fi;
+        // Pre-acquired metadata must never reverse quota or multipart-source
+        // lock order. These paths need their own reservation protocol.
+        if state.metadata_reservation.is_some()
+            && (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
+                || rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
+        {
+            state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
+            return Err(DiskError::DiskOngoingReq);
+        }
         let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
             Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
                 state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
@@ -556,9 +577,7 @@ impl LocalDisk {
             None
         };
         let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
-            Some(Arc::new(
-                pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
-            ))
+            Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
         } else {
             None
         };
@@ -586,7 +605,27 @@ impl LocalDisk {
             Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
             None => state.namespace_owner.take(),
         };
-        let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
+        // A reserved lease is single-use and bound to the exact disk path.
+        // Keep the independent namespace/source owner alongside it; a pre-held
+        // mutex cannot be acquired again without deadlocking.
+        let reservation_fence = state
+            .metadata_reservation
+            .as_ref()
+            .and_then(|reservation| reservation.fence());
+        let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
+            if multipart_staging.is_some() || quota_fence_claim.is_some() {
+                state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
+                return Err(DiskError::DiskOngoingReq);
+            }
+            let lease = reservation
+                .into_lease_for(&root_destination_object_path, self.reservation_instance)
+                .inspect_err(|_| {
+                    state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
+                })?;
+            Arc::new((lease, metadata_owner))
+        } else {
+            os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
+        };
         drop(metadata_wait);
         #[cfg(unix)]
         let parent_receipt = if native_timing {
@@ -600,17 +639,23 @@ impl LocalDisk {
         };
         #[cfg(unix)]
         let mut receipt_mutation = native_timing
-            .then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
+            .then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
         let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
         let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
             &self.root,
             dst_volume,
             &root_destination_object_path,
             staging_prefix.as_deref(),
-            Some(metadata_lease),
+            Some(metadata_owner),
         )
         .await;
         drop(namespace_wait);
+        // The volume read lease prevents deletion of this disk's bucket until
+        // publication finishes. Recheck after its wait, before any mutation.
+        if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
+            state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
+            return Err(DiskError::DiskOngoingReq);
+        }
         if let Some(claim) = quota_fence_claim {
             mutation_lease.attach_external_guard(claim);
         }
@@ -759,6 +804,11 @@ impl LocalDisk {
             && !fi_healing
             && !os::dst_dir_fsync_group_commit_enabled()
             && !os::file_fdatasync_group_commit_enabled()
+            && !super::has_rename_data_test_failpoints(dst_path)
+            && !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
+            && !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
+            && !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
+            && !super::has_delete_version_test_failpoints()
         {
             #[cfg(test)]
             if !no_inline {
@@ -802,7 +852,7 @@ impl LocalDisk {
                 // requires a rollback directory; appending a new version keeps
                 // an inline rollback backup for post-rename directory sync errors.
                 if is_cancelled() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
                 }
                 let mut xlmeta = FileMeta::new();
                 let mut dst_meta_unparsable = false;
@@ -825,19 +875,7 @@ impl LocalDisk {
                     }
                 });
                 if rollback_data_dir.is_some() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
-                }
-                #[cfg(test)]
-                let probe_active = os::file_sync_probe::is_active();
-                #[cfg(not(test))]
-                let probe_active = false;
-                if no_inline
-                    && sync_data_shards
-                    && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
-                    && (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
-                    && os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
-                {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
                 }
                 let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
                     has_dst_buf.is_some(),
@@ -856,31 +894,20 @@ impl LocalDisk {
                     skip_parent = dst_parent_owned.clone();
                 }
 
-                #[cfg(test)]
-                run_owned_file_before_open(&src_file_path_owned);
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
-                prepared_metadata_source.write_all(&new_dst_buf, false)?;
-                if !no_inline {
-                    run_inline_preparation_before_backup(&dst_path_str);
-                }
+                prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
 
                 let mut staged_rollback_path = None;
-                if let Some(ref old_metadata) = has_dst_buf
-                    && (sync_commit_metadata || cfg!(test))
-                {
-                    let backup_path = create_local_inline_rollback_backup(
-                        &dst_file_path_owned,
-                        &src_file_path_owned,
-                        old_metadata,
-                        sync_commit_metadata,
-                    )?;
+                if sync_commit_metadata && let Some(ref old_metadata) = has_dst_buf {
+                    let backup_path =
+                        create_local_inline_rollback_backup(&dst_file_path_owned, &src_file_path_owned, old_metadata, true)?;
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 
                 if no_inline {
                     if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
-                        let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
+                        let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
                             // The payload was synced, but its part.1 directory entry
                             // must also be persisted before moving the data directory.
                             os::fsync_dir_std(src_data_path)
@@ -896,124 +923,37 @@ impl LocalDisk {
                             return Err(err);
                         }
                     }
-                    if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
-                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                            let _ = remove_file_if_exists(backup_path);
-                        }
-                        return Err(err);
-                    }
-                } else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
-                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                        let _ = remove_file_if_exists(backup_path);
-                    }
-                    return Err(err);
-                }
-
-                if is_cancelled() {
-                    drop(prepared_metadata_source);
-                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                        let _ = remove_file_if_exists(backup_path);
-                    }
-                    return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
-                }
-
-                let guard = match super::remove_dst_base_before_commit_std(
-                    &dst_path_str,
-                    guard,
-                    &src_parent_owned,
-                    &dst_parent_owned,
-                    &dst_volume_dir_owned,
-                    &publication_root_owned,
-                ) {
-                    Ok(guard) => guard,
-                    Err(err) => {
-                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                            let _ = remove_file_if_exists(backup_path);
-                        }
-                        return Ok(FusedCommitAttempt::CommitErr(err));
-                    }
-                };
-                if should_remove_staged_meta_before_commit(&dst_path_str) {
-                    drop(prepared_metadata_source);
-                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                        let _ = remove_file_if_exists(backup_path);
-                    }
-                    std::fs::remove_file(&src_file_path_owned)?;
-                    return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
-                }
-
-                if no_inline {
                     if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
                         && let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
                     {
                         if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
                             let _ = remove_file_if_exists(backup_path);
                         }
-                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
+                        return Ok(FusedCommitAttempt::CommitErr(err));
                     }
                     #[cfg(test)]
-                    if has_data_dir_path_owned.is_some()
-                        && let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-                            run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
-                        }))
-                    {
-                        return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
-                    }
-                    if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
-                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
-                    }
-                    if should_fail_before_old_metadata_backup(&dst_path_str) {
-                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
-                            let _ = remove_file_if_exists(backup_path);
-                        }
-                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
-                    }
-                    if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
-                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                    if has_data_dir_path_owned.is_some() {
+                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
                     }
                 }
 
-                let rename_res = if should_fail_commit_rename(&dst_path_str) {
-                    drop(prepared_metadata_source);
-                    Err(DiskError::other("test fail during metadata commit rename"))
-                } else {
-                    os::rename_prepared_source_on_thread(
-                        prepared_metadata_source,
-                        &src_file_path_owned,
-                        &dst_file_path_owned,
-                        &skip_parent,
-                        &publication_root_owned,
-                        &guard,
-                    )
-                };
-                if let Err(err) = rename_res {
+                if let Err(err) = os::rename_prepared_source_on_thread(
+                    prepared_metadata_source,
+                    &src_file_path_owned,
+                    &dst_file_path_owned,
+                    &skip_parent,
+                    &publication_root_owned,
+                    &guard,
+                ) {
                     if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
                         let _ = remove_file_if_exists(backup_path);
                     }
-                    if no_inline {
-                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
+                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        let _ = std::fs::rename(dst_data_path, src_data_path);
                     }
                     return Ok(FusedCommitAttempt::CommitErr(err));
                 }
 
-                if should_fail_after_metadata_commit(&dst_path_str) {
-                    if no_inline {
-                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
-                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
-                            remove_dir_all_if_exists(new_data_path)?;
-                        }
-                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
-                    }
-                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
-                        .err()
-                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
-                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
-                }
-
-                if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
-                    return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
-                }
-
                 if sync_commit_metadata
                     && let Err(err) = fsync_inline_commit_std(
                         &dst_file_path_owned,
@@ -1023,11 +963,8 @@ impl LocalDisk {
                         &guard,
                     )
                 {
-                    if no_inline {
-                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
-                            remove_dir_all_if_exists(new_data_path)?;
-                        }
-                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
+                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        let _ = std::fs::rename(dst_data_path, src_data_path);
                     }
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
                 }
@@ -1040,7 +977,7 @@ impl LocalDisk {
                     let _ = std::fs::remove_dir(parent);
                 }
 
-                Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
+                Ok(FusedCommitAttempt::Committed(RenameDataResp {
                     multipart_source: None,
                     old_data_dir: None,
                     rollback_data_dir: None,
@@ -1049,7 +986,7 @@ impl LocalDisk {
                     old_current_size,
                     old_current_source_checked,
                     old_current_source,
-                })))
+                }))
             };
 
             let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
@@ -1069,23 +1006,7 @@ impl LocalDisk {
                     if let Some(receipt_mutation) = receipt_mutation.as_mut() {
                         receipt_mutation.complete();
                     }
-                    return Ok(*resp);
-                }
-                Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
-                    let commit = RenameCommitContext {
-                        directory_guard,
-                        publication_root: &self.publication_root,
-                        mutation_lease,
-                        source_metadata: &src_file_path,
-                        destination_metadata: &dst_file_path,
-                        source_volume_dir: &src_volume_dir,
-                        data_paths: has_data_dir_path.as_ref(),
-                    };
-                    commit.restore_published_data_source().await?;
-                    for part_path in &invalidate_part_paths {
-                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
-                    }
-                    return Err(err);
+                    return Ok(resp);
                 }
                 Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
                     for part_path in &invalidate_part_paths {
@@ -1093,269 +1014,11 @@ impl LocalDisk {
                     }
                     return Err(err);
                 }
-                #[cfg(test)]
-                Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
-                    std::panic::resume_unwind(panic_payload);
-                }
                 Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
-                    fi = *restored_fi;
+                    fi = restored_fi;
                     (guard, dst_buf)
                 }
             }
-        } else if let Some(publication) = multipart_publication.as_ref()
-            && no_inline
-            && !preserve_upload_metadata
-            && !publication.legacy
-            && !fi_healing
-            && !os::dst_dir_fsync_group_commit_enabled()
-        {
-            enum MultipartFusedCommitOutcome {
-                Committed(Box<RenameDataResp>),
-                AbortBeforeDataRename(DiskError),
-                RestoreDataSource(os::RenameCommitGuard, DiskError),
-                RestoreTargetAfterError(DiskError, bool),
-            }
-            let publication_owned = Arc::clone(publication);
-            let src_parent_owned = src_file_parent.to_path_buf();
-            let dst_parent_owned = dst_file_parent.to_path_buf();
-            let dst_file_path_owned = dst_file_path.clone();
-            let src_file_path_owned = src_file_path.clone();
-            let dst_volume_dir_owned = dst_volume_dir.clone();
-            let publication_root_owned = self.publication_root.clone();
-            let has_data_dir_path_owned = has_data_dir_path.clone();
-            let sync_data_shards = durability.syncs_data_shards();
-            let sync_commit_metadata = durability.syncs_commit_metadata();
-            let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
-            let dst_volume_str = dst_volume.to_string();
-            let dst_path_str = dst_path.to_string();
-            let fi_moved = fi;
-            #[cfg(test)]
-            let root_owned = self.root.clone();
-
-            let outcome = os::run_blocking_namespace_operation_with_sync(
-                mutation_lease.clone(),
-                true,
-                move || -> std::io::Result<MultipartFusedCommitOutcome> {
-                    let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
-                        false,
-                        &src_parent_owned,
-                        &dst_parent_owned,
-                        &dst_file_path_owned,
-                        &dst_volume_dir_owned,
-                        &publication_root_owned,
-                    ) {
-                        Ok(res) => res,
-                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
-                    };
-                    let mut xlmeta = FileMeta::new();
-                    let mut dst_meta_unparsable = false;
-                    if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
-                        if FileMeta::is_xl2_v1_format(dst_buf)
-                            && let Ok(nmeta) = FileMeta::load(dst_buf)
-                        {
-                            xlmeta = nmeta;
-                        } else {
-                            dst_meta_unparsable = true;
-                        }
-                    }
-                    let skip_parent_owned = if has_dst_buf_owned.is_some() {
-                        dst_parent_owned.clone()
-                    } else {
-                        dst_volume_dir_owned.clone()
-                    };
-                    let version_id = fi_moved.version_id.unwrap_or_default();
-                    let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
-                        has_dst_buf_owned.is_some(),
-                        dst_meta_unparsable,
-                        &xlmeta,
-                        &dst_volume_str,
-                        &dst_path_str,
-                        version_id,
-                    );
-                    let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
-                    if let Some(old_data_dir) = has_old_data_dir.as_ref() {
-                        let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
-                    }
-                    if let Err(err) = xlmeta.add_version(fi_moved) {
-                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
-                    }
-                    let version_signature = rename_data_versions_signature(&xlmeta);
-                    let new_dst_buf_owned = match xlmeta.marshal_msg() {
-                        Ok(buf) => buf,
-                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
-                    };
-
-                    let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
-                        &new_dst_buf_owned,
-                        has_dst_buf_owned.as_deref(),
-                        &dst_file_path_owned,
-                        &directory_guard_owned,
-                    ) {
-                        Ok(source) => source,
-                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
-                    };
-                    #[cfg(test)]
-                    let probe_active = os::file_sync_probe::is_active();
-                    #[cfg(not(test))]
-                    let probe_active = false;
-                    // When presynced, shard payloads were synced during UploadPart and
-                    // prepare_and_install_head_on_thread just fsynced src_data_path after
-                    // writing RETIREMENT_ANCHOR.
-                    if sync_data_shards
-                        && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
-                        && (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
-                        && let Err(err) = os::sync_dir_files_std(src_data_path)
-                        && err.kind() != ErrorKind::NotFound
-                    {
-                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
-                    }
-                    let directory_guard_owned = match super::remove_dst_base_before_commit_std(
-                        &dst_path_str,
-                        directory_guard_owned,
-                        &src_parent_owned,
-                        &dst_parent_owned,
-                        &dst_volume_dir_owned,
-                        &publication_root_owned,
-                    ) {
-                        Ok(guard) => guard,
-                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
-                    };
-                    if should_remove_staged_meta_before_commit(&dst_path_str) {
-                        drop(prepared_metadata_source);
-                        if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
-                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
-                        }
-                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
-                    }
-                    if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
-                        let Some(src_parent) = src_data_path.parent() else {
-                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
-                        };
-                        let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
-                            Ok(guard) => guard,
-                            Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
-                        };
-                        if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
-                        {
-                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
-                        }
-                        #[cfg(test)]
-                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
-                        if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
-                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
-                        }
-                        if should_fail_before_old_metadata_backup(&dst_path_str) {
-                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
-                                directory_guard_owned,
-                                DiskError::Unexpected,
-                            ));
-                        }
-                        if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
-                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
-                        }
-                    }
-                    let rename_res = if should_fail_commit_rename(&dst_path_str) {
-                        drop(prepared_metadata_source);
-                        Err(DiskError::other("test fail during metadata commit rename"))
-                    } else {
-                        os::rename_prepared_source_on_thread(
-                            prepared_metadata_source,
-                            &src_file_path_owned,
-                            &dst_file_path_owned,
-                            &skip_parent_owned,
-                            &publication_root_owned,
-                            &directory_guard_owned,
-                        )
-                    };
-                    if let Err(err) = rename_res {
-                        return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
-                    }
-                    if should_fail_after_metadata_commit(&dst_path_str) {
-                        return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
-                    }
-                    if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
-                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
-                    }
-                    if sync_commit_metadata {
-                        if let Some(parent) = dst_file_path_owned.parent()
-                            && let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
-                        {
-                            return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
-                        }
-                        if has_dst_buf_owned.is_none() {
-                            let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
-                            while let Some(dir) = ancestor {
-                                if !dir.starts_with(&dst_volume_dir_owned) {
-                                    break;
-                                }
-                                if let Err(err) = os::fsync_dir_limited_std(dir) {
-                                    return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
-                                        to_file_error(err).into(),
-                                        true,
-                                    ));
-                                }
-                                if dir == dst_volume_dir_owned.as_path() {
-                                    break;
-                                }
-                                ancestor = dir.parent();
-                            }
-                        }
-                    }
-                    match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
-                        Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
-                            multipart_source: Some(Box::new(result)),
-                            old_data_dir: has_old_data_dir,
-                            rollback_data_dir: None,
-                            cleanup_data_dir: has_old_data_dir,
-                            sign: version_signature,
-                            old_current_size,
-                            old_current_source_checked,
-                            old_current_source,
-                        }))),
-                        Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
-                    }
-                },
-            )
-            .await
-            .map_err(to_file_error)
-            .map_err(DiskError::from)?;
-
-            match outcome {
-                MultipartFusedCommitOutcome::Committed(resp) => {
-                    for part_path in &invalidate_part_paths {
-                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
-                    }
-                    return Ok(*resp);
-                }
-                MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
-                MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
-                    let commit = RenameCommitContext {
-                        directory_guard,
-                        publication_root: &self.publication_root,
-                        mutation_lease,
-                        source_metadata: &src_file_path,
-                        destination_metadata: &dst_file_path,
-                        source_volume_dir: &src_volume_dir,
-                        data_paths: has_data_dir_path.as_ref(),
-                    };
-                    commit.restore_published_data_source().await?;
-                    return Err(err);
-                }
-                MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
-                    if !invalidate_before_return {
-                        for part_path in &invalidate_part_paths {
-                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
-                        }
-                    }
-                    pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
-                    if invalidate_before_return {
-                        for part_path in &invalidate_part_paths {
-                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
-                        }
-                    }
-                    return Err(err);
-                }
-            }
         } else {
             prepare_rename_commit_and_destination_metadata(
                 !no_inline && !preserve_upload_metadata,
@@ -1479,29 +1142,13 @@ impl LocalDisk {
             // after the commit rename, and a crash before the rename means the
             // PUT was never acknowledged. A metadata commit: relaxed tiers
             // leave it to the page cache.
-            #[cfg(all(test, target_os = "linux"))]
-            let probe_active = os::file_sync_probe::is_active();
-            #[cfg(all(not(test), target_os = "linux"))]
-            let probe_active = false;
             let shard_sync = async {
                 if durability.syncs_data_shards()
                     && let Some((src_data_path, _)) = has_data_dir_path.as_ref()
+                    && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
+                    && err.kind() != ErrorKind::NotFound
                 {
-                    #[cfg(target_os = "linux")]
-                    if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
-                        if multipart_publication.is_none()
-                            && let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
-                            && err.kind() != ErrorKind::NotFound
-                        {
-                            return Err::<(), DiskError>(to_file_error(err).into());
-                        }
-                        return Ok(());
-                    }
-                    if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
-                        && err.kind() != ErrorKind::NotFound
-                    {
-                        return Err::<(), DiskError>(to_file_error(err).into());
-                    }
+                    return Err::<(), DiskError>(to_file_error(err).into());
                 }
                 Ok(())
             };
@@ -1893,22 +1540,13 @@ impl LocalDisk {
             }
 
             let multipart_source = match multipart_publication.as_ref() {
-                Some(publication) => {
-                    let target_already_synced = durability.syncs_commit_metadata();
-                    let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
-                    match pin_multipart_future(|| {
-                        publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
-                    })
-                    .await
-                    {
-                        Ok(result) => Some(Box::new(result)),
-                        Err(error) => {
-                            pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
-                                .await?;
-                            return Err(error);
-                        }
+                Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
+                    Ok(result) => Some(Box::new(result)),
+                    Err(error) => {
+                        pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
+                        return Err(error);
                     }
-                }
+                },
                 None => None,
             };
             Ok(RenameDataResp {
@@ -2549,8 +2187,23 @@ impl LocalDisk {
         dst_path: &str,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> super::super::RenameDataObservation {
+        self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
+            .await
+    }
+
+    pub(in crate::disk) async fn rename_data_observed_with_reservation(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
+    ) -> super::super::RenameDataObservation {
+        let (namespace_owner, metadata_reservation) = owners;
         let mut state = RenameDataState {
             namespace_owner,
+            metadata_reservation,
             ..Default::default()
         };
         let result = self

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -1000,21 +1000,6 @@ pub(super) fn inspection_with_markers(
     Ok((inspection, has_physical_data_dirs_or_markers))
 }
 
-type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
-
-const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
-
-static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
-    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
-
-fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
-    PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
-}
-
-fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
-    PREPARED_DELETE_RECEIPTS.remove(rollback)
-}
-
 impl LocalDisk {
     // Native MPU source closure has its own conditional protocol. Generic
     // object recovery must never restore its head or acquire SourceRW under
@@ -1538,9 +1523,6 @@ impl LocalDisk {
         owner: Option<Arc<dyn Send + Sync>>,
     ) -> Result<DeleteRecoveryInspection> {
         self.reject_multipart_delete_recovery_mutation(volume, object)?;
-        if let Ok(root) = self.io_get_object_path(volume, object) {
-            let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
-        }
 
         let (mut observed, mut has_data_dir_work) = self
             .delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
@@ -1929,14 +1911,8 @@ impl LocalDisk {
                     return Err(DiskError::DiskOngoingReq);
                 }
                 let bucket_path = self.io_get_bucket_path(volume)?;
-                let mut file_leases = Vec::with_capacity(5);
-                for name in [
-                    STORAGE_FORMAT_FILE_BACKUP,
-                    DELETE_MARKER_ROLLBACK_FILE,
-                    TARGET,
-                    PREPARED,
-                    FINISHED,
-                ] {
+                let mut file_leases = Vec::with_capacity(3);
+                for name in [TARGET, PREPARED, FINISHED] {
                     let path = rollback.join(name);
                     let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
                     file_leases.push((path, lease));
@@ -2156,128 +2132,114 @@ impl LocalDisk {
                     Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
                     Err(err) => return Err(err),
                 };
-                type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
-                let (old_dir, target, journal_raw, journal, sole_transaction) =
-                    match (|| -> Result<Option<PreparedDeleteTuple>> {
-                        let mut meta = FileMeta::load(&buf)?;
-                        let mut deleted_headers: Vec<_> = meta
-                            .versions
-                            .iter()
-                            .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
-                            .collect();
-                        let original_versions: Vec<_> = meta
-                            .versions
-                            .iter()
-                            .map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
-                            .collect::<Result<_>>()?;
-                        let old_dir = meta.delete_version(&fi_clone)?;
-                        if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
-                            return Err(DiskError::FileCorrupt);
+                type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>);
+                let (old_dir, target, journal_raw) = match (|| -> Result<Option<PreparedDeleteTuple>> {
+                    let mut meta = FileMeta::load(&buf)?;
+                    let mut deleted_headers: Vec<_> = meta
+                        .versions
+                        .iter()
+                        .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
+                        .collect();
+                    let original_versions: Vec<_> = meta
+                        .versions
+                        .iter()
+                        .map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
+                        .collect::<Result<_>>()?;
+                    let old_dir = meta.delete_version(&fi_clone)?;
+                    if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
+                        return Err(DiskError::FileCorrupt);
+                    }
+                    if let Some(uuid) = old_dir {
+                        meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
+                        let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
+                        check_path_length(old_path.to_string_lossy().as_ref())?;
+                        if old_path.join(STORAGE_FORMAT_FILE).exists() {
+                            return Ok(None);
                         }
-                        if let Some(uuid) = old_dir {
-                            meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
-                            let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
-                            check_path_length(old_path.to_string_lossy().as_ref())?;
-                            if old_path.join(STORAGE_FORMAT_FILE).exists() {
-                                return Ok(None);
+                    }
+                    let surviving: HashSet<_> = meta
+                        .versions
+                        .iter()
+                        .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
+                        .collect();
+                    deleted_headers.retain(|header| !surviving.contains(header));
+                    let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
+                    let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
+                    let mut predecessors = BTreeSet::new();
+                    if let Ok(entries) = std::fs::read_dir(&file_path) {
+                        for (visited, entry) in entries.enumerate() {
+                            if visited >= 65536 {
+                                return Err(DiskError::DiskOngoingReq);
+                            }
+                            let entry = entry.map_err(to_file_error)?;
+                            let Some(id) = entry
+                                .file_name()
+                                .to_str()
+                                .and_then(|name| Uuid::parse_str(name).ok())
+                                .filter(|id| !id.is_nil() && *id != transaction)
+                            else {
+                                continue;
+                            };
+                            if !entry.file_type().map_err(to_file_error)?.is_dir() {
+                                return Err(DiskError::InvalidPath);
+                            }
+                            let prepared = regular_bytes(&entry.path().join(PREPARED))?;
+                            let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
+                            let mut records = Vec::new();
+                            if let Some(bytes) = prepared {
+                                records.push(
+                                    serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
+                                        .map_err(|_| DiskError::FileCorrupt)?,
+                                );
+                            }
+                            if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
+                                records.push(
+                                    serde_json::from_slice::<CommitReceipt>(&bytes)
+                                        .map_err(|_| DiskError::FileCorrupt)?
+                                        .journal,
+                                );
+                            }
+                            for record in records {
+                                if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
+                                    || record.volume != volume_str
+                                    || record.object != object_str
+                                    || record.transaction != id
+                                    || incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
+                                    || record.predecessors.iter().any(|before| before.is_nil() || *before == id)
+                                {
+                                    return Err(DiskError::FileCorrupt);
+                                }
+                                predecessors.insert(id);
                             }
                         }
-                        let surviving: HashSet<_> = meta
-                            .versions
-                            .iter()
-                            .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
-                            .collect();
-                        deleted_headers.retain(|header| !surviving.contains(header));
-                        let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
-                        let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
-                        let mut predecessors = BTreeSet::new();
-                        let mut sole_transaction = true;
-                        if let Ok(entries) = std::fs::read_dir(&file_path) {
-                            for (visited, entry) in entries.enumerate() {
-                                if visited >= 65536 {
-                                    return Err(DiskError::DiskOngoingReq);
-                                }
-                                let entry = entry.map_err(to_file_error)?;
-                                let file_name = entry.file_name();
-                                let Some(name_str) = file_name.to_str() else {
-                                    sole_transaction = false;
-                                    continue;
-                                };
-                                let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
-                                else {
-                                    continue;
-                                };
-                                if !entry.file_type().map_err(to_file_error)?.is_dir() {
-                                    return Err(DiskError::InvalidPath);
-                                }
-                                let dir_path = entry.path();
-                                let prepared = regular_bytes(&dir_path.join(PREPARED))?;
-                                let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
-                                if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
-                                    let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
-                                        && prepared.is_none()
-                                        && receipt.is_none()
-                                        && !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
-                                    if !is_surviving_data_dir {
-                                        sole_transaction = false;
-                                    }
-                                }
-                                let mut records = Vec::new();
-                                if let Some(bytes) = prepared {
-                                    records.push(
-                                        serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
-                                            .map_err(|_| DiskError::FileCorrupt)?,
-                                    );
-                                }
-                                if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
-                                    records.push(
-                                        serde_json::from_slice::<CommitReceipt>(&bytes)
-                                            .map_err(|_| DiskError::FileCorrupt)?
-                                            .journal,
-                                    );
-                                }
-                                for record in records {
-                                    if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
-                                        || record.volume != volume_str
-                                        || record.object != object_str
-                                        || record.transaction != id
-                                        || incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
-                                        || record.predecessors.iter().any(|before| before.is_nil() || *before == id)
-                                    {
-                                        return Err(DiskError::FileCorrupt);
-                                    }
-                                    predecessors.insert(id);
-                                }
-                            }
-                        }
-                        sole_transaction &= predecessors.is_empty();
-                        let mut sparse = fi_clone;
-                        sparse.data = None;
-                        let journal = DeleteRecoveryJournal {
-                            protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
-                            volume: volume_str,
-                            object: object_str,
-                            transaction,
-                            bucket_incarnation: incarnation,
-                            original_digest: Some(digest(&buf)),
-                            target_digest: target.as_deref().map(digest),
-                            data_dirs: old_dir.into_iter().collect(),
-                            deleted_versions,
-                            requests: vec![sparse],
-                            original_versions,
-                            predecessors: predecessors.into_iter().collect(),
-                        };
-                        let raw = journal_bytes(&journal)?;
-                        Ok(Some((old_dir, target, raw, journal, sole_transaction)))
-                    })() {
-                        Ok(Some(tuple)) => tuple,
-                        Ok(None) => return Ok(Ok(None)),
-                        Err(err) => return Ok(Err(err)),
+                    }
+                    let mut sparse = fi_clone;
+                    sparse.data = None;
+                    let journal = DeleteRecoveryJournal {
+                        protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                        volume: volume_str,
+                        object: object_str,
+                        transaction,
+                        bucket_incarnation: incarnation,
+                        original_digest: Some(digest(&buf)),
+                        target_digest: target.as_deref().map(digest),
+                        data_dirs: old_dir.into_iter().collect(),
+                        deleted_versions,
+                        requests: vec![sparse],
+                        original_versions,
+                        predecessors: predecessors.into_iter().collect(),
                     };
+                    Ok(Some((old_dir, target, journal_bytes(&journal)?)))
+                })() {
+                    Ok(Some(tuple)) => tuple,
+                    Ok(None) => return Ok(Ok(None)),
+                    Err(err) => return Ok(Err(err)),
+                };
                 std::fs::create_dir_all(&staging_root)?;
-                let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
-                let staged_dir = staged.path().to_path_buf();
-                let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
+                let staged = tempfile::Builder::new()
+                    .prefix("delete-preparation-")
+                    .tempdir_in(&staging_root)?;
+                let backup_path = staged.path().join(STORAGE_FORMAT_FILE_BACKUP);
                 if std::fs::hard_link(&xl_path, &backup_path).is_err() {
                     let mut file = std::fs::File::create(&backup_path)?;
                     std::io::Write::write_all(&mut file, &buf)?;
@@ -2285,22 +2247,18 @@ impl LocalDisk {
                         file.sync_data()?;
                     }
                 }
-                let mut written_files = Vec::with_capacity(2);
                 for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
                     let Some(bytes) = bytes else {
                         continue;
                     };
-                    let mut file = std::fs::File::create(staged_dir.join(name))?;
+                    let mut file = std::fs::File::create(staged.path().join(name))?;
                     std::io::Write::write_all(&mut file, bytes)?;
                     if sync && !bytes.is_empty() {
-                        written_files.push(file);
+                        file.sync_data()?;
                     }
                 }
-                for file in written_files {
-                    file.sync_data()?;
-                }
                 if sync {
-                    os::fsync_dir_std(&staged_dir)?;
+                    os::fsync_dir_std(staged.path())?;
                 }
                 match std::fs::symlink_metadata(&destination) {
                     Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
@@ -2313,54 +2271,53 @@ impl LocalDisk {
                     Err(err) if err.kind() == ErrorKind::NotFound => {}
                     Err(err) => return Err(err),
                 }
-                let staged_dir = staged.keep();
-                if let Err(err) = std::fs::rename(&staged_dir, &destination) {
-                    let _ = std::fs::remove_dir_all(&staged_dir);
-                    return Err(err);
+                if let Some(parent) = destination.parent() {
+                    std::fs::create_dir_all(parent)?;
                 }
-                let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
+                std::fs::rename(staged.path(), &destination)?;
                 if sync && let Err(err) = os::fsync_dir_std(&file_path) {
                     let _ = std::fs::remove_dir_all(&destination);
                     return Err(err);
                 }
                 let mut reserved_data_path = None;
-                if let Some(old_path) = old_data_path {
-                    let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
-                    let reserve_res = (|| -> std::io::Result<()> {
-                        let marker = std::fs::File::create(&reserved_marker)?;
-                        if sync {
-                            marker.sync_all()?;
-                            os::fsync_dir_std(&old_path)?;
+                if let Some(uuid) = old_dir {
+                    let old_path = file_path.join(uuid.to_string());
+                    if old_path.is_dir() {
+                        let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
+                        let reserve_res = (|| -> std::io::Result<()> {
+                            let marker = std::fs::File::create(&reserved_marker)?;
+                            if sync {
+                                marker.sync_all()?;
+                                os::fsync_dir_std(&old_path)?;
+                            }
+                            Ok(())
+                        })();
+                        if let Err(err) = reserve_res {
+                            let _ = std::fs::remove_file(&reserved_marker);
+                            let _ = std::fs::remove_dir_all(&destination);
+                            return Err(err);
                         }
-                        Ok(())
-                    })();
-                    if let Err(err) = reserve_res {
-                        let _ = std::fs::remove_file(&reserved_marker);
-                        let _ = std::fs::remove_dir_all(&destination);
-                        return Err(err);
+                        reserved_data_path = Some((old_path, reserved_marker));
                     }
-                    reserved_data_path = Some((old_path, reserved_marker));
                 }
                 if let Some(ref target_bytes) = target {
-                    let target_in_rollback = destination.join(TARGET);
-                    let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
-                    let _ = std::fs::remove_file(&linked_xl);
-                    let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
-                        let rename_res = std::fs::rename(&linked_xl, &xl_path);
-                        if rename_res.is_err() {
-                            let _ = std::fs::remove_file(&linked_xl);
-                        }
-                        rename_res
-                    } else {
-                        (|| -> std::io::Result<()> {
-                            let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
-                            std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
-                            if sync && !target_bytes.is_empty() {
-                                staged_xl.as_file().sync_data()?;
+                    let mut staged_xl = match tempfile::NamedTempFile::new_in(&staging_root) {
+                        Ok(staged_xl) => staged_xl,
+                        Err(err) => {
+                            if let Some((_, reserved_marker)) = &reserved_data_path {
+                                let _ = std::fs::remove_file(reserved_marker);
                             }
-                            std::fs::rename(staged_xl.path(), &xl_path)
-                        })()
+                            let _ = std::fs::remove_dir_all(&destination);
+                            return Err(err);
+                        }
                     };
+                    let write_res = (|| -> std::io::Result<()> {
+                        std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
+                        if sync && !target_bytes.is_empty() {
+                            staged_xl.as_file().sync_data()?;
+                        }
+                        std::fs::rename(staged_xl.path(), &xl_path)
+                    })();
                     if let Err(err) = write_res {
                         if let Some((_, reserved_marker)) = &reserved_data_path {
                             let _ = std::fs::remove_file(reserved_marker);
@@ -2389,7 +2346,6 @@ impl LocalDisk {
                         Err(err) => return Err(err),
                     }
                 }
-                put_prepared_delete_receipt(destination, (journal, sole_transaction));
                 Ok(Ok(Some(old_dir)))
             },
         )
@@ -2435,26 +2391,6 @@ impl LocalDisk {
             lease,
             sync,
             move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
-                if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
-                    && rollback.join(PREPARED).is_file()
-                {
-                    let receipt = CommitReceipt {
-                        journal: journal.clone(),
-                        finished: false,
-                    };
-                    let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
-                    std::fs::create_dir_all(&staging_root)?;
-                    let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
-                    std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
-                    if sync && !receipt_bytes.is_empty() {
-                        staged.as_file().sync_data()?;
-                    }
-                    std::fs::rename(staged.path(), &receipt_path)?;
-                    if sync {
-                        os::fsync_dir_std(&rollback)?;
-                    }
-                    return Ok(Ok(Some((journal, sole_transaction, true))));
-                }
                 let prepared = match regular_bytes(&rollback.join(PREPARED)) {
                     Ok(Some(bytes)) => bytes,
                     Ok(None) => return Ok(Ok(None)),
@@ -2590,7 +2526,7 @@ impl LocalDisk {
         let sync = effective_durability(volume).syncs_commit_metadata();
         let data_dirs = journal.data_dirs.clone();
         let has_target = journal.target_digest.is_some();
-        let journal_for_receipt = journal.clone();
+        let journal_for_receipt = has_target.then(|| journal.clone());
         let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
         let prune_paths: Vec<_> = rollback
             .ancestors()
@@ -2602,13 +2538,11 @@ impl LocalDisk {
             let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
             let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
             let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
-            let mut had_physical_data_dirs = false;
             for data_dir in &data_dirs {
                 for base in [&root, &rollback] {
                     let dir_path = base.join(data_dir.to_string());
                     match std::fs::read_dir(&dir_path) {
                         Ok(entries) => {
-                            had_physical_data_dirs = true;
                             let mut removable = Vec::new();
                             let mut preserve_dir = false;
                             for entry in entries {
@@ -2649,17 +2583,14 @@ impl LocalDisk {
                     }
                 }
             }
-            if had_physical_data_dirs && sync && root.is_dir() {
+            if !data_dirs.is_empty() && sync && root.is_dir() {
                 os::fsync_dir_std(&root)?;
             }
             for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
                 ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
             }
-            if has_target || had_physical_data_dirs {
-                let finished_receipt = CommitReceipt {
-                    journal: journal_for_receipt,
-                    finished: true,
-                };
+            if let Some(journal) = journal_for_receipt {
+                let finished_receipt = CommitReceipt { journal, finished: true };
                 let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
                 std::fs::create_dir_all(&staging_root)?;
                 let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
@@ -2674,20 +2605,13 @@ impl LocalDisk {
                 }
             }
             for name in [TARGET, PREPARED, FINISHED] {
-                let path = rollback.join(name);
-                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-                os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
-                ignore_not_found(std::fs::remove_file(path))?;
+                ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
             }
             if sync {
                 os::fsync_dir_std(&rollback)?;
             }
-            #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-            os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
             ignore_not_found(std::fs::remove_file(&receipt_path))?;
             for dir_path in &prune_paths {
-                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-                os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
                 if let Err(err) = std::fs::remove_dir(dir_path) {
                     if is_dir_not_empty_error(&err) {
                         break;
@@ -6585,63 +6509,6 @@ mod tests {
             "surviving v1 data_dir must remain intact"
         );
     }
-
-    #[tokio::test]
-    async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
-        let (_temp, disk, mut info, _) = fixture().await;
-        let root = disk.io_get_object_path("recovery", "object").expect("object path");
-
-        info.data = None;
-        rustfs_utils::http::metadata_compat::remove_str(
-            &mut info.metadata,
-            rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
-        );
-        let data_dir = Uuid::new_v4();
-        info.data_dir = Some(data_dir);
-        fs::create_dir_all(root.join(data_dir.to_string()))
-            .await
-            .expect("create data_dir");
-        fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
-            .await
-            .expect("write part.1");
-        let mut meta = FileMeta::new();
-        meta.add_version(info.clone()).expect("add version");
-        fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
-            .await
-            .expect("write xl.meta");
-
-        let transaction = Uuid::new_v4();
-        let opts = DeleteOptions {
-            recoverable_delete: true,
-            ..Default::default()
-        };
-        let fused = disk
-            .try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
-            .await
-            .expect("fused unversioned physical delete must succeed");
-        assert_eq!(fused, Some(()));
-
-        let (journal, sole_transaction, target_satisfied) = disk
-            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
-            .await
-            .expect("local publish must succeed")
-            .expect("absent target must satisfy local publish");
-        assert!(sole_transaction);
-        assert!(target_satisfied);
-        assert!(journal.target_digest.is_none());
-
-        let rollback = root.join(transaction.to_string());
-        assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
-
-        let finalized = disk
-            .try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
-            .await
-            .expect("local finalize must succeed");
-        assert!(finalized);
-        assert!(!rollback.exists(), "rollback directory must be pruned");
-        assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
-        assert!(!root.exists(), "empty object directory must be pruned");
-    }
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/local/multipart_source_abort.rs

@@ -421,7 +421,6 @@ impl LocalDisk {
                     return Err(DiskError::ConditionalDeleteConflict);
                 }
                 let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
-                super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
                 let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
                 let source = store.source.clone();
                 let base = store.base.clone();
@@ -619,9 +618,6 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
     root: os::PublicationRoot,
 }
 impl OrdinaryMultipartDeleteCheck {
-    pub(super) fn source(&self) -> &Path {
-        &self.source
-    }
     pub(super) fn validate(&self) -> Result<()> {
         let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
             Ok(pinned) => pinned,

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -238,177 +238,9 @@ pub(super) fn read_optional_in_pinned(
         Err(error) => Err(error.into()),
     }
 }
-#[derive(Clone, Copy, PartialEq, Eq)]
-struct FileStamp {
-    len: u64,
-    modified: Option<std::time::SystemTime>,
-    #[cfg(unix)]
-    ino: u64,
-}
-
-impl FileStamp {
-    fn from_path(path: &Path) -> Option<Self> {
-        let meta = std::fs::symlink_metadata(path).ok()?;
-        Some(Self {
-            len: meta.len(),
-            modified: meta.modified().ok(),
-            #[cfg(unix)]
-            ino: {
-                use std::os::unix::fs::MetadataExt;
-                meta.ino()
-            },
-        })
-    }
-}
-
-#[derive(Clone)]
-struct ActiveMultipartSourceEntry {
-    raw: Bytes,
-    fi: FileInfo,
-    head_stamp: FileStamp,
-    unsettled_parts: std::collections::HashSet<usize>,
-    parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
-}
-
-const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
-
-static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
-    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
-
-pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
-    let source = source.as_ref();
-    let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
-        return;
-    };
-    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
-        source,
-        ActiveMultipartSourceEntry {
-            raw,
-            fi,
-            head_stamp,
-            unsettled_parts: std::collections::HashSet::new(),
-            parts: std::collections::HashMap::new(),
-        },
-    );
-}
-
-fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
-    FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
-        && !source.join(STATE).exists()
-        && !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
-        && !source.join(".mpu-completions").exists()
-}
-
-pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
-    let (raw, fi, head_stamp) =
-        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
-    if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-        return None;
-    }
-    Some((raw, fi))
-}
-
-pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
-    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
-        return false;
-    };
-    if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-        return false;
-    }
-    true
-}
-
-pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
-    let Some((head_stamp, clean)) =
-        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
-    else {
-        return false;
-    };
-    if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-        return false;
-    }
-    clean
-}
-
-pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
-        entry.unsettled_parts.insert(part_number);
-        entry.parts.remove(&part_number);
-    });
-}
-
-pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
-    let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .with_get(source, |entry| entry.fi.data_dir)
-        .flatten()
-        .filter(|dir| !dir.is_nil())
-    else {
-        return;
-    };
-    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
-    let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
-        return;
-    };
-    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
-        entry.parts.insert(part.number, (part_stamp, part));
-    });
-}
-
-pub(super) fn finish_active_multipart_part_transaction(
-    source: &Path,
-    part_number: Option<usize>,
-    committed: bool,
-    retired_existing_txn: bool,
-) {
-    let Some(part_number) = part_number.filter(|_| committed) else {
-        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-        return;
-    };
-    if !retired_existing_txn {
-        return;
-    }
-    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
-}
-
-pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
-    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .with_get(source, |entry| {
-            if !entry.unsettled_parts.is_empty() {
-                return None;
-            }
-            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
-            let (expected_stamp, part) = entry.parts.get(&part_number)?;
-            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
-        })
-        .flatten()?;
-    if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-        return None;
-    }
-    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
-    if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
-        return None;
-    }
-    Some(part)
-}
-
-pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
-}
-
-pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
-}
-
 /// Canonical source mutations are admitted under SourceRW at the syscall,
 /// not on the earlier async caller that may have timed out.
 pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
-    if !initial && has_active_multipart_source_head(source) {
-        return Ok(());
-    }
     let pinned = match os::pin_multipart_source_directory(source, base, root) {
         Ok(pinned) => pinned,
         Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
@@ -608,7 +440,6 @@ impl SourceStore {
         terminal: Option<&Path>,
         sync_source_before: bool,
     ) -> std::io::Result<()> {
-        invalidate_active_multipart_source_head(source);
         #[cfg(unix)]
         let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
         #[cfg(not(unix))]
@@ -639,7 +470,7 @@ impl SourceStore {
         let terminal = terminal_record_path(&source, state);
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
             Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
         })
         .await
@@ -708,7 +539,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
         })
         .await
         .map_err(DiskError::from)
@@ -720,7 +551,6 @@ impl SourceStore {
         owner: Uuid,
         target: bool,
         bytes: &[u8],
-        sync_source_root: bool,
     ) -> std::io::Result<()> {
         let path = if target {
             target_backup_path(source, owner)
@@ -752,10 +582,9 @@ impl SourceStore {
         if !existed {
             os::replace_multipart_source_metadata(&path, bytes, base, root)?;
         }
-        os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
+        os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
-        invalidate_active_multipart_source_head(&self.source);
         let path = self.source.join(STORAGE_FORMAT_FILE);
         let base = self.base.clone();
         let root = self.publication_root.clone();
@@ -1268,82 +1097,6 @@ pub(super) fn load_source(
     })
 }
 
-fn load_uninitialized_source(
-    raw: Bytes,
-    path: &str,
-    disk_id: Uuid,
-    physical_slot: usize,
-    disk_ids: &[Uuid],
-) -> Result<LoadedSource> {
-    let sha = raw_sha(&raw);
-    let length = raw.len() as u64;
-    let current = get_file_info(
-        &raw,
-        RUSTFS_META_MULTIPART_BUCKET,
-        path,
-        "",
-        FileInfoOpts {
-            data: false,
-            include_free_versions: false,
-            include_part_checksums: true,
-        },
-    )
-    .map_err(Error::other)?;
-    native_source_protocol(&current)?;
-    if source_status(&current)?.is_some()
-        || publication_identity(&current)?.is_some()
-        || SourceConditionV1::take(&mut current.clone())?.is_some()
-    {
-        return Err(DiskError::FileCorrupt);
-    }
-    let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-bucket")
-        .ok_or(DiskError::FileCorrupt)?;
-    let object = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-object")
-        .ok_or(DiskError::FileCorrupt)?;
-    let condition = capture_source_condition(bucket, object, path, &current, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
-    if condition.generation.epoch != 0 {
-        return Err(DiskError::FileCorrupt);
-    }
-    let slot = condition
-        .lineage
-        .geometry
-        .distribution
-        .iter()
-        .position(|index| *index == current.erasure.index)
-        .ok_or(DiskError::FileCorrupt)?;
-    if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
-        return Err(DiskError::ConditionalDeleteConflict);
-    }
-    let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
-    let snapshot = Some(MultipartSourceSnapshot {
-        disk_id,
-        physical_slot: slot,
-        lineage: condition.lineage,
-        generation: condition.generation,
-        revision: 0,
-        phase: MultipartSourcePhase::Active,
-        state_sha256: [0; 32],
-        head_sha256: Some(sha),
-        inventory_sha256: inventory,
-        capability_sha256: None,
-        publication: None,
-    });
-    Ok(LoadedSource {
-        root_missing: false,
-        state: None,
-        head: Some(SourceHead {
-            sha256: sha,
-            length,
-            raw: Some(raw.clone()),
-        }),
-        original: Some(raw),
-        info: Some(current),
-        snapshot,
-        head_matches_state: true,
-        head_matches_activation: false,
-    })
-}
-
 pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
 
 #[derive(Serialize, Deserialize)]
@@ -1668,8 +1421,6 @@ struct ValidatedPreparedPublication {
     length: u64,
 }
 
-type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
-
 pub(super) struct NativeMultipartPublication {
     pub store: Arc<SourceStore>,
     pub publication: PublicationIdentity,
@@ -1677,49 +1428,22 @@ pub(super) struct NativeMultipartPublication {
     state: parking_lot::Mutex<SourceState>,
     capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
     validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
-    deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
     pub legacy: bool,
 }
 
 impl LocalDisk {
-    #[cfg(test)]
     pub(super) async fn prepare_native_multipart_publication(
         &self,
         path: &str,
         incoming: &mut FileInfo,
         staging: &MultipartCommitStaging,
-    ) -> Result<NativeMultipartPublication> {
-        self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
-            .await
-    }
-
-    pub(super) async fn prepare_native_multipart_publication_deferred(
-        &self,
-        path: &str,
-        incoming: &mut FileInfo,
-        staging: &MultipartCommitStaging,
-    ) -> Result<NativeMultipartPublication> {
-        self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
-            .await
-    }
-
-    async fn prepare_native_multipart_publication_inner(
-        &self,
-        path: &str,
-        incoming: &mut FileInfo,
-        staging: &MultipartCommitStaging,
-        defer_persist: bool,
     ) -> Result<NativeMultipartPublication> {
         // Real Complete is parts-backed, including a zero-byte last part.
         if incoming.data.is_some() || incoming.inline_data() {
             return Err(DiskError::UnsupportedDisk);
         }
         let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
-        let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
-            load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
-        } else {
-            store.load().await?
-        };
+        let loaded = store.load().await?;
         let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
         let original = loaded
             .head
@@ -1821,33 +1545,20 @@ impl LocalDisk {
         };
         let anchor_bytes = encode(&anchor_record)?;
         state.validate(store.disk_id, &store.path)?;
+        let state_raw = encode(&state)?;
+        let source_dir = store.source.clone();
+        let base_dir = store.base.clone();
+        let pub_root = store.publication_root.clone();
+        let terminal = terminal_record_path(&source_dir, &state);
         let original_bytes = original.to_vec();
-        let deferred_initial = if defer_persist {
-            Some((original_bytes, directory, anchor_bytes))
-        } else {
-            let state_raw = encode(&state)?;
-            let source_dir = store.source.clone();
-            let base_dir = store.base.clone();
-            let pub_root = store.publication_root.clone();
-            let terminal = terminal_record_path(&source_dir, &state);
-            #[cfg(target_os = "linux")]
-            let can_mark_presynced = has_active_multipart_source_head(&source_dir)
-                && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
-            os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
-                SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
-                SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
-                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
-                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
-                #[cfg(target_os = "linux")]
-                if can_mark_presynced {
-                    super::mark_presynced_tmp_data_dir(directory);
-                }
-                Ok(())
-            })
-            .await
-            .map_err(DiskError::from)?;
-            None
-        };
+        os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
+            SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes)?;
+            SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
+            let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
+            os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)
+        })
+        .await
+        .map_err(DiskError::from)?;
         let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
         let context = NativeMultipartPublication {
             store,
@@ -1856,7 +1567,6 @@ impl LocalDisk {
             state: parking_lot::Mutex::new(state),
             capability: parking_lot::Mutex::new(Some(capability)),
             validated_prepared: parking_lot::Mutex::new(None),
-            deferred_initial: parking_lot::Mutex::new(deferred_initial),
             legacy,
         };
         Ok(context)
@@ -1911,7 +1621,7 @@ impl NativeMultipartPublication {
                 return Err(IoError::other(DiskError::ConditionalDeleteConflict));
             }
             if let Some(previous) = old_bytes.as_deref() {
-                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
             }
             SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
         })
@@ -1921,133 +1631,6 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
-    #[cfg(target_os = "linux")]
-    pub(super) fn prepare_and_install_head_on_thread(
-        &self,
-        new_metadata: &[u8],
-        old_metadata: Option<&[u8]>,
-        dst_file_path: &Path,
-        directory_guard: &os::RenameCommitGuard,
-    ) -> Result<os::PreparedRenameSource> {
-        let mut state = self.state.lock().clone();
-        if !self.store.target_matches(&self.publication, new_metadata)? {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
-        let deferred_initial = self.deferred_initial.lock().take();
-        let expected_state_raw = encode(&state)?;
-        let expected_lineage = state.lineage.clone();
-        let expected_generation = state.generation;
-        let expected_revision = state.revision;
-        let validated = ValidatedPreparedPublication {
-            publication: self.publication.clone(),
-            sha256: raw_sha(new_metadata),
-            length: new_metadata.len() as u64,
-        };
-        if let Some(previous) = old_metadata {
-            state.previous_target_sha256 = Some(raw_sha(previous));
-            state.previous_target_length = Some(previous.len() as u64);
-        }
-        state.prepared_sha256 = Some(validated.sha256);
-        state.prepared_length = Some(validated.length);
-        state.advance(MultipartSourcePhase::Prepared)?;
-        state.validate(self.store.disk_id, &self.store.path)?;
-        let next_state_raw = encode(&state)?;
-        let source = &self.store.source;
-        let base = &self.store.base;
-        let root = &self.store.publication_root;
-        let path = &self.store.path;
-        let disk_id = self.store.disk_id;
-        let physical_slot = self.store.physical_slot;
-        let disk_ids = &self.store.disk_ids;
-        let backup_owner = state.backup_owner;
-        #[cfg(target_os = "linux")]
-        let can_mark_presynced = has_active_multipart_source_head(source)
-            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
-        let terminal = terminal_record_path(source, &state);
-        let prepared_path = &self.prepared_path;
-        let canonical = self.canonical_head();
-        let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
-            if let Some((ref original_bytes, _, _)) = deferred_initial {
-                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
-                let snapshot = current
-                    .snapshot
-                    .as_ref()
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                let current_head = current
-                    .head
-                    .as_ref()
-                    .and_then(SourceHead::raw)
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                if !current.head_matches_state
-                    || snapshot.phase != MultipartSourcePhase::Active
-                    || snapshot.capability_sha256.is_some()
-                    || snapshot.lineage != expected_lineage
-                    || snapshot.generation != expected_generation
-                    || snapshot.revision.checked_add(1) != Some(expected_revision)
-                    || current_head != original_bytes.as_slice()
-                {
-                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
-                }
-            } else {
-                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
-                let current_state = current
-                    .state
-                    .as_ref()
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                let current_raw = encode(current_state).map_err(IoError::other)?;
-                if current_raw != expected_state_raw || !current.head_matches_state {
-                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
-                }
-            }
-            #[cfg(test)]
-            super::run_owned_file_before_open(prepared_path);
-            let mut prepared_metadata_source =
-                os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
-            if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
-                drop(prepared_metadata_source);
-                let _ = std::fs::remove_file(prepared_path);
-                return Err(err);
-            }
-            let setup_res: std::io::Result<()> = (|| {
-                if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
-                    SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
-                    if let Some(previous) = old_metadata {
-                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
-                    }
-                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
-                    let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
-                    os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
-                    #[cfg(target_os = "linux")]
-                    if can_mark_presynced {
-                        super::mark_presynced_tmp_data_dir(directory);
-                    }
-                } else {
-                    if let Some(previous) = old_metadata {
-                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
-                    }
-                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
-                }
-                Ok(())
-            })();
-            if let Err(err) = setup_res {
-                drop(prepared_metadata_source);
-                let _ = std::fs::remove_file(prepared_path);
-                return Err(err);
-            }
-            if let Err(err) = prepared_metadata_source.sync_data() {
-                drop(prepared_metadata_source);
-                let _ = std::fs::remove_file(prepared_path);
-                return Err(err);
-            }
-            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
-        })()
-        .map_err(DiskError::from)?;
-        *self.state.lock() = state;
-        *self.validated_prepared.lock() = Some(validated);
-        Ok(prepared)
-    }
-
-    #[cfg(not(target_os = "linux"))]
     pub async fn prepare_and_install_head(
         &self,
         new_metadata: &[u8],
@@ -2060,11 +1643,7 @@ impl NativeMultipartPublication {
         if !self.store.target_matches(&self.publication, new_metadata)? {
             return Err(DiskError::ConditionalDeleteConflict);
         }
-        let deferred_initial = self.deferred_initial.lock().take();
         let expected_state_raw = encode(&state)?;
-        let expected_lineage = state.lineage.clone();
-        let expected_generation = state.generation;
-        let expected_revision = state.revision;
         let validated = ValidatedPreparedPublication {
             publication: self.publication.clone(),
             sha256: raw_sha(new_metadata),
@@ -2087,9 +1666,6 @@ impl NativeMultipartPublication {
         let physical_slot = self.store.physical_slot;
         let disk_ids = self.store.disk_ids.clone();
         let backup_owner = state.backup_owner;
-        #[cfg(target_os = "linux")]
-        let can_mark_presynced = has_active_multipart_source_head(&source)
-            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
         let old_bytes = old_metadata.map(|b| b.to_vec());
         let new_bytes = new_metadata.to_vec();
         let terminal = terminal_record_path(&source, &state);
@@ -2099,51 +1675,18 @@ impl NativeMultipartPublication {
         let owned_directory_guard = directory_guard.clone();
         let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
             let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
-            if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
-                let snapshot = current
-                    .snapshot
-                    .as_ref()
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                let current_head = current
-                    .head
-                    .as_ref()
-                    .and_then(SourceHead::raw)
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                if !current.head_matches_state
-                    || snapshot.phase != MultipartSourcePhase::Active
-                    || snapshot.capability_sha256.is_some()
-                    || snapshot.lineage != expected_lineage
-                    || snapshot.generation != expected_generation
-                    || snapshot.revision.checked_add(1) != Some(expected_revision)
-                    || current_head != original_bytes.as_slice()
-                {
-                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
-                }
-                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
-                if let Some(previous) = old_bytes.as_deref() {
-                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
-                }
-                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
-                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
-                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
-                #[cfg(target_os = "linux")]
-                if can_mark_presynced {
-                    super::mark_presynced_tmp_data_dir(directory);
-                }
-            } else {
-                let current_state = current
-                    .state
-                    .as_ref()
-                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
-                let current_raw = encode(current_state).map_err(IoError::other)?;
-                if current_raw != expected_state_raw || !current.head_matches_state {
-                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
-                }
-                if let Some(previous) = old_bytes.as_deref() {
-                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
-                }
-                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+            let current_state = current
+                .state
+                .as_ref()
+                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+            let current_raw = encode(current_state).map_err(IoError::other)?;
+            if current_raw != expected_state_raw || !current.head_matches_state {
+                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
             }
+            if let Some(previous) = old_bytes.as_deref() {
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
+            }
+            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
             #[cfg(test)]
             super::run_owned_file_before_open(&prepared_path);
             let mut prepared_metadata_source =
@@ -2157,28 +1700,6 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(prepared)
     }
-
-    #[cfg(target_os = "linux")]
-    pub async fn prepare_and_install_head(
-        self: &Arc<Self>,
-        new_metadata: &[u8],
-        old_metadata: Option<&[u8]>,
-        dst_file_path: &Path,
-        directory_guard: &os::RenameCommitGuard,
-        mutation_lease: Arc<os::NamespaceMutationLease>,
-    ) -> Result<os::PreparedRenameSource> {
-        let this = Arc::clone(self);
-        let new_bytes = new_metadata.to_vec();
-        let old_bytes = old_metadata.map(|b| b.to_vec());
-        let dst_file_path = dst_file_path.to_path_buf();
-        let owned_directory_guard = directory_guard.clone();
-        os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
-            this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
-                .map_err(IoError::other)
-        })
-        .await
-        .map_err(DiskError::from)
-    }
     #[cfg(test)]
     pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
         {
@@ -2250,11 +1771,7 @@ impl NativeMultipartPublication {
         // return an error after consuming the last claimable source heads.
         self.store.restore_exact_target_with_lease(&state, lease).await
     }
-    pub(super) fn finish_on_thread(
-        &self,
-        target_already_synced: bool,
-        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
-    ) -> Result<MultipartPublicationResult> {
+    pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
         let mut state = self.state.lock().clone();
         let expected_state_raw = encode(&state)?;
         let (expected_sha256, expected_length) = {
@@ -2273,16 +1790,16 @@ impl NativeMultipartPublication {
         state.validate(self.store.disk_id, &self.store.path)?;
         let next_state_raw = encode(&state)?;
         let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
-        let source = &self.store.source;
-        let base = &self.store.base;
-        let root = &self.store.publication_root;
-        let path = &self.store.path;
+        let source = self.store.source.clone();
+        let base = self.store.base.clone();
+        let root = self.store.publication_root.clone();
+        let path = self.store.path.clone();
         let disk_id = self.store.disk_id;
         let physical_slot = self.store.physical_slot;
-        let disk_ids = &self.store.disk_ids;
-        let terminal = terminal_record_path(source, &state);
-        (|| -> std::io::Result<()> {
-            let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+        let disk_ids = self.store.disk_ids.clone();
+        let terminal = terminal_record_path(&source, &state);
+        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
+            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
             let current_state = current
                 .state
                 .as_ref()
@@ -2291,64 +1808,28 @@ impl NativeMultipartPublication {
             if current.head.is_some() || current_raw != expected_state_raw {
                 return Err(IoError::other(DiskError::ConditionalDeleteConflict));
             }
-            if target_already_synced {
-                let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
-                    if error.kind() == std::io::ErrorKind::NotFound {
-                        IoError::other(DiskError::ConditionalDeleteConflict)
-                    } else {
-                        error
+            os::read_and_sync_multipart_target_metadata(
+                &target_dir,
+                std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                &base,
+                &root,
+                |target| {
+                    if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
+                        return Err(IoError::other(DiskError::ConditionalDeleteConflict));
                     }
-                })?;
-                let target = os::read_multipart_source_metadata_in_pinned(
-                    &pinned,
-                    &target_dir,
-                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
-                    base,
-                    root,
-                )
-                .map_err(|error| {
-                    if error.kind() == std::io::ErrorKind::NotFound {
-                        IoError::other(DiskError::ConditionalDeleteConflict)
-                    } else {
-                        error
-                    }
-                })?;
-                if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
-                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                    Ok(())
+                },
+            )
+            .map_err(|error| {
+                if error.kind() == std::io::ErrorKind::NotFound {
+                    IoError::other(DiskError::ConditionalDeleteConflict)
+                } else {
+                    error
                 }
-            } else {
-                os::read_and_sync_multipart_target_metadata(
-                    &target_dir,
-                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
-                    base,
-                    root,
-                    |target| {
-                        if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
-                            return Err(IoError::other(DiskError::ConditionalDeleteConflict));
-                        }
-                        Ok(())
-                    },
-                )
-                .map_err(|error| {
-                    if error.kind() == std::io::ErrorKind::NotFound {
-                        IoError::other(DiskError::ConditionalDeleteConflict)
-                    } else {
-                        error
-                    }
-                })?;
-            }
-            SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
-            if let Some((io_path, cleaned)) = staging_cleanup {
-                match std::fs::remove_dir(&io_path) {
-                    Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
-                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
-                        cleaned.store(true, std::sync::atomic::Ordering::Release);
-                    }
-                    Err(_) => {}
-                }
-            }
-            Ok(())
-        })()
+            })?;
+            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), true)
+        })
+        .await
         .map_err(DiskError::from)?;
         *self.state.lock() = state;
         let proof = self
@@ -2361,21 +1842,6 @@ impl NativeMultipartPublication {
             undo_capability: capability,
         })
     }
-
-    pub async fn finish(
-        self: &Arc<Self>,
-        target_lease: Arc<os::NamespaceMutationLease>,
-        target_already_synced: bool,
-        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
-    ) -> Result<MultipartPublicationResult> {
-        let this = Arc::clone(self);
-        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
-            this.finish_on_thread(target_already_synced, staging_cleanup)
-                .map_err(IoError::other)
-        })
-        .await
-        .map_err(DiskError::from)
-    }
 }
 
 impl SourceStore {

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -207,12 +207,6 @@ pub(super) struct MultipartSourceAdmission {
     initial: bool,
 }
 impl MultipartSourceAdmission {
-    pub(super) fn source(&self) -> &Path {
-        &self.source
-    }
-    pub(super) fn is_initial(&self) -> bool {
-        self.initial
-    }
     pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
         let admission = self.clone();
         Arc::new(move || admission.validate())
@@ -256,8 +250,6 @@ pub(super) struct MultipartCommitStaging {
     logical_path: PathBuf,
     io_path: PathBuf,
     owner: Option<Arc<dyn Send + Sync>>,
-    cleaned: Arc<std::sync::atomic::AtomicBool>,
-    fresh_source_head: Option<bytes::Bytes>,
 }
 
 impl MultipartCommitStaging {
@@ -267,19 +259,10 @@ impl MultipartCommitStaging {
     pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
         self.io_path.join(directory.to_string())
     }
-    pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
-        (self.io_path.clone(), Arc::clone(&self.cleaned))
-    }
-    pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
-        self.fresh_source_head.clone()
-    }
 }
 
 impl Drop for MultipartCommitStaging {
     fn drop(&mut self) {
-        if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
-            return;
-        }
         let logical_path = self.logical_path.clone();
         let io_path = self.io_path.clone();
         let owner = self.owner.clone();
@@ -302,14 +285,6 @@ impl Drop for MultipartCommitStaging {
 }
 
 impl LocalDisk {
-    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
-        let root = upload_scope_root(upload_or_part_path, part_path);
-        let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
-            return false;
-        };
-        super::multipart_source_state::has_clean_active_multipart_source(&source)
-    }
-
     pub(super) fn multipart_source_admission(
         &self,
         volume: &str,
@@ -381,14 +356,12 @@ impl LocalDisk {
         let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
         // The attempt root is exactly the namespace used by the tmp sweeper.
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
-        let mut staging = MultipartCommitStaging {
+        let staging = MultipartCommitStaging {
             path: attempt,
             lease,
             logical_path,
             io_path,
             owner,
-            cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
-            fresh_source_head: None,
         };
         let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
         let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
@@ -398,37 +371,31 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }
-            let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
-            let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
-                (raw, source, true)
-            } else {
-                let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
-                    // A retired source has no authority for this delayed completion.
-                    // Preserve confinement and other I/O failures as their real errors.
-                    if error.kind() == std::io::ErrorKind::NotFound {
-                        DiskError::FileNotFound.into()
-                    } else {
-                        error
-                    }
-                })?;
-                let source = get_file_info(
-                    &raw,
-                    RUSTFS_META_MULTIPART_BUCKET,
-                    &source_path,
-                    "",
-                    FileInfoOpts {
-                        data: false,
-                        include_free_versions: false,
-                        include_part_checksums: true,
-                    },
-                )
-                .map_err(std::io::Error::other)?;
-                (bytes::Bytes::from(raw), source, false)
-            };
+            let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
+                // A retired source has no authority for this delayed completion.
+                // Preserve confinement and other I/O failures as their real errors.
+                if error.kind() == std::io::ErrorKind::NotFound {
+                    DiskError::FileNotFound.into()
+                } else {
+                    error
+                }
+            })?;
+            let source = get_file_info(
+                &raw,
+                RUSTFS_META_MULTIPART_BUCKET,
+                &source_path,
+                "",
+                FileInfoOpts {
+                    data: false,
+                    include_free_versions: false,
+                    include_part_checksums: true,
+                },
+            )
+            .map_err(std::io::Error::other)?;
             let source_dir = source
                 .data_dir
                 .filter(|dir| !dir.is_nil())
@@ -445,31 +412,23 @@ impl LocalDisk {
             let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
             if incoming.data.is_none() && !incoming.is_remote() {
                 let source_data_dir = source_object.join(source_dir.to_string());
-                let mut pinned_source_data = None;
+                let pinned_source_data = (!incoming.parts.is_empty())
+                    .then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
+                    .transpose()?;
                 let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
                 for part in &incoming.parts {
                     if cancelled() {
                         return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
                     }
                     let meta_name = format!("part.{}.meta", part.number);
-                    let actual = if let Some(cached_part) =
-                        super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
-                    {
-                        cached_part
-                    } else {
-                        if pinned_source_data.is_none() {
-                            pinned_source_data =
-                                Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
-                        }
-                        let bytes = os::read_multipart_source_metadata_in_pinned(
-                            pinned_source_data.as_ref().expect("pinned source data directory"),
-                            &source_data_dir,
-                            std::ffi::OsStr::new(&meta_name),
-                            &base,
-                            &publication_root,
-                        )?;
-                        rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
-                    };
+                    let bytes = os::read_multipart_source_metadata_in_pinned(
+                        pinned_source_data.as_ref().expect("pinned source data directory"),
+                        &source_data_dir,
+                        std::ffi::OsStr::new(&meta_name),
+                        &base,
+                        &publication_root,
+                    )?;
+                    let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
                     let normalized = i64::try_from(actual.size)
                         .map_err(std::io::Error::other)?
                         .max(actual.actual_size);
@@ -505,15 +464,10 @@ impl LocalDisk {
                     cancelled,
                 )?;
             }
-            let has_state = !from_active_cache
-                && (source_object.join(".mpu-source-state").exists()
-                    || source_object.join(".mpu-recovery").exists()
-                    || source_object.join(".mpu-completions").exists());
-            Ok((!has_state).then_some(raw))
+            Ok(())
         })
         .await
         .map_err(DiskError::from)?;
-        staging.fresh_source_head = fresh_head;
         Ok(staging)
     }
 }
@@ -1265,211 +1219,4 @@ mod tests {
             .expect("non-part empty hash cleanup must remain supported");
         assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
     }
-
-    #[tokio::test]
-    async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
-        use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
-        let (_root, disk, completed, _) = fixture().await;
-        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
-        let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
-        let data_dir = Uuid::new_v4();
-        let mut init_fi = FileInfo::new(&upload, 2, 2);
-        init_fi.erasure.index = 1;
-        init_fi.data_dir = Some(data_dir);
-        init_fi.fresh = true;
-        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
-
-        disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
-            .await
-            .expect("fresh multipart upload metadata");
-        assert!(disk.has_clean_active_multipart_source(&upload, false));
-
-        let read_fi = disk
-            .read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
-            .await
-            .expect("cached read_version");
-        assert_eq!(read_fi.data_dir, Some(data_dir));
-
-        let destination = format!("{upload}/{data_dir}/part.1");
-        let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
-        disk.write_all(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.1",
-            bytes::Bytes::from_static(b"immutable shard"),
-        )
-        .await
-        .expect("stage tmp part");
-        disk.prepare_part_transaction(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.1",
-            RUSTFS_META_MULTIPART_BUCKET,
-            &destination,
-            part_meta.clone(),
-        )
-        .await
-        .expect("prepare part transaction");
-        assert!(
-            !disk.has_clean_active_multipart_source(&upload, false),
-            "in-flight part transaction must mark active source as having unsettled part txn"
-        );
-        disk.rename_part(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.1",
-            RUSTFS_META_MULTIPART_BUCKET,
-            &destination,
-            part_meta,
-        )
-        .await
-        .expect("rename part");
-        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
-            .await
-            .expect("settle part transaction");
-        assert!(
-            disk.has_clean_active_multipart_source(&upload, false),
-            "settled commit must restore clean active multipart source status"
-        );
-        let read_parts = disk
-            .read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
-            .await
-            .expect("cached read_parts");
-        assert_eq!(read_parts.len(), 1);
-        assert_eq!(read_parts[0].number, 1);
-
-        // Prepare part 2, then replay Commit for already-settled part 1: part 2's
-        // pending transaction must keep the upload marked as having unsettled part txns.
-        let destination2 = format!("{upload}/{data_dir}/part.2");
-        let mut part2_info = completed.parts[0].clone();
-        part2_info.number = 2;
-        let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
-        disk.write_all(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.2",
-            bytes::Bytes::from_static(b"immutable shard"),
-        )
-        .await
-        .expect("stage tmp part 2");
-        disk.prepare_part_transaction(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.2",
-            RUSTFS_META_MULTIPART_BUCKET,
-            &destination2,
-            part2_meta.clone(),
-        )
-        .await
-        .expect("prepare part 2 transaction");
-        assert!(!disk.has_clean_active_multipart_source(&upload, false));
-        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
-            .await
-            .expect("replayed part 1 commit settlement");
-        assert!(
-            !disk.has_clean_active_multipart_source(&upload, false),
-            "replayed settlement for part 1 must not clear part 2's pending transaction"
-        );
-        disk.rename_part(
-            RUSTFS_META_TMP_BUCKET,
-            "cache-test-part/part.2",
-            RUSTFS_META_MULTIPART_BUCKET,
-            &destination2,
-            part2_meta,
-        )
-        .await
-        .expect("rename part 2");
-        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
-            .await
-            .expect("settle part 2 transaction");
-        assert!(disk.has_clean_active_multipart_source(&upload, false));
-
-        // Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
-        disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
-            .await
-            .expect("delete xl.meta via delete_paths");
-        assert!(
-            !disk.has_clean_active_multipart_source(&upload, false),
-            "per-file head deletion must invalidate the canonical upload cache key"
-        );
-        assert!(matches!(
-            disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
-                .await,
-            Err(DiskError::FileNotFound)
-        ));
-    }
-
-    #[cfg(not(windows))]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
-        use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
-
-        let (_root, disk, _, _) = fixture().await;
-        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
-        let disk = Arc::new(disk);
-        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
-        let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
-        let data_dir = Uuid::new_v4();
-        let mut init_fi = FileInfo::new(&upload, 2, 2);
-        init_fi.erasure.index = 1;
-        init_fi.data_dir = Some(data_dir);
-        init_fi.fresh = true;
-        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
-
-        let source_dir = disk
-            .io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
-            .expect("source dir");
-        let base_dir = disk
-            .io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
-            .expect("multipart bucket path");
-        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
-        let (release_tx, release_rx) = std::sync::mpsc::channel();
-        let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
-            let _ = entered_tx.send(());
-            let _ = release_rx.recv();
-        });
-
-        let init_wrapper = wrapper.clone();
-        let init_upload = upload.clone();
-        let init_task = tokio::spawn(async move {
-            init_wrapper
-                .write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
-                .await
-        });
-
-        tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
-            .await
-            .expect("initial write_metadata reaches parent dir fsync")
-            .expect("entered channel");
-
-        let retire_wrapper = wrapper.clone();
-        let retire_path = format!("{upload}/xl.meta");
-        let mut retire_fut = Box::pin(async move {
-            retire_wrapper
-                .delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
-                .await
-        });
-        assert!(
-            futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
-            "concurrent retirement must block on SourceRW while initial head publication is in flight"
-        );
-
-        let _ = release_tx.send(());
-        init_task.await.expect("init join").expect("initial write_metadata succeeds");
-        retire_fut
-            .await
-            .expect("concurrent retirement completes after SourceRW release");
-
-        assert!(
-            !super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
-            "retired source must not remain active in cache after interleaved initial publication"
-        );
-        assert!(
-            matches!(
-                super::super::multipart_source_state::validate_source_mutation(
-                    &source_dir,
-                    &base_dir,
-                    &disk.publication_root,
-                    false
-                ),
-                Err(DiskError::FileNotFound)
-            ),
-            "validate_source_mutation must reject non-initial mutation on retired source"
-        );
-    }
 }

crates/ecstore/src/disk/mod.rs

@@ -106,6 +106,8 @@ pub(crate) struct RenameDataGuards {
     pub(crate) scanner_publication_lease_token: Option<Uuid>,
     pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
     pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
+    pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
+    pub(crate) metadata_reservation_token: Option<Uuid>,
 }
 
 /// Local preflight evidence stays outside DiskAPI and the RPC response format.
@@ -122,6 +124,13 @@ impl RenameDataObservation {
         }
     }
 
+    fn not_dispatched(error: DiskError) -> Self {
+        Self {
+            result: Err(error),
+            preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
+        }
+    }
+
     pub(crate) fn rejected_before_publication(&self) -> bool {
         self.result.is_err() && self.preflight_rejection.is_some()
     }
@@ -311,6 +320,60 @@ pub enum Disk {
 }
 
 impl Disk {
+    /// Prepare an inert single-use reservation on this disk.
+    /// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
+    pub async fn prepare_object_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        incarnation: Uuid,
+        timeout: std::time::Duration,
+    ) -> Result<Uuid> {
+        if timeout.is_zero() {
+            return Err(DiskError::Timeout);
+        }
+        match self {
+            Disk::Local(disk) => tokio::time::timeout(
+                timeout.min(std::time::Duration::from_secs(5)),
+                disk.prepare_object_metadata(bucket, object),
+            )
+            .await
+            .map_err(|_| DiskError::Timeout)?,
+            Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
+        }
+    }
+
+    /// Enqueue a prepared token in object mutation order.
+    pub async fn activate_object_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        token: Uuid,
+        timeout: std::time::Duration,
+    ) -> Result<()> {
+        if timeout.is_zero() {
+            return Err(DiskError::Timeout);
+        }
+        match self {
+            Disk::Local(disk) => tokio::time::timeout(
+                timeout.min(std::time::Duration::from_secs(5)),
+                disk.activate_object_metadata(bucket, object, token),
+            )
+            .await
+            .map_err(|_| DiskError::Timeout)?,
+            Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
+        }
+    }
+
+    /// Cancel an unclaimed reservation. False does not prove that a claim has
+    /// not already started a physical mutation.
+    pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
+        match self {
+            Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
+            Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
+        }
+    }
+
     /// Physical ownership belongs to the node hosting the local attachment.
     pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
         match self {
@@ -401,13 +464,6 @@ impl Disk {
         }
     }
 
-    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
-        match self {
-            Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
-            Self::Remote(_) => false,
-        }
-    }
-
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
@@ -1251,18 +1307,33 @@ impl Disk {
                     .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
                     .await
             }
-            Disk::Remote(remote_disk) => RenameDataObservation::unknown(
-                remote_disk
-                    .rename_data_borrowed_with_fence(
-                        src_volume,
-                        src_path,
-                        fi,
-                        dst_volume,
-                        dst_path,
-                        guards.scanner_publication_lease_token,
-                    )
-                    .await,
-            ),
+            Disk::Remote(remote_disk) => {
+                if guards.metadata_reservation.is_some() {
+                    return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
+                }
+                if let Some(token) = guards.metadata_reservation_token {
+                    if guards.scanner_publication_lease_token.is_some() {
+                        return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
+                    }
+                    return RenameDataObservation::unknown(
+                        remote_disk
+                            .rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
+                            .await,
+                    );
+                }
+                RenameDataObservation::unknown(
+                    remote_disk
+                        .rename_data_borrowed_with_fence(
+                            src_volume,
+                            src_path,
+                            fi,
+                            dst_volume,
+                            dst_path,
+                            guards.scanner_publication_lease_token,
+                        )
+                        .await,
+                )
+            }
         }
     }
 

crates/ecstore/src/io_support/rio.rs

@@ -429,7 +429,6 @@ impl WritePlan {
         let checksum = reader.content_hash().clone();
         let trailer = reader.get_trailer().cloned();
 
-        let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
         let encrypted = self.encryption.is_some();
         if let Some(algorithm) = self.compression {
             reader = HashReader::from_reader(
@@ -449,8 +448,7 @@ impl WritePlan {
                 }
                 WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
             };
-            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
-                .with_inner_etag_delegation(delegate_inner_etag);
+            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
             reader = match encryption.mode {
                 WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]

crates/ecstore/src/io_support/shard_integrity.rs

@@ -333,34 +333,6 @@ struct ProofCache {
     preferred: usize,
 }
 
-type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
-
-static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
-    std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
-
-fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
-    VERIFIED_STRIPE_RECORDS
-        .lock()
-        .get(&(part.generation, part.root, stripe))
-        .cloned()
-}
-
-fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
-    let mut map = VERIFIED_STRIPE_RECORDS.lock();
-    if map.len() >= 4096
-        && let Some(evict_key) = map.keys().next().copied()
-    {
-        map.remove(&evict_key);
-    }
-    map.insert((part.generation, part.root, stripe), record);
-}
-
-pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
-    VERIFIED_STRIPE_RECORDS
-        .lock()
-        .retain(|(stored_gen, _, _), _| *stored_gen != generation);
-}
-
 impl PartProofReader {
     pub(crate) async fn reconstruction_proof(
         &self,
@@ -438,9 +410,6 @@ impl PartProofReader {
 
     pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
         let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
-        if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
-            return Ok(bytes);
-        }
         let mut cached = self.cached.lock().await;
         if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
             return Ok(bytes.clone());
@@ -496,9 +465,6 @@ impl PartProofReader {
             bytes
         };
         self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
-        if self.part.stripes <= 4 {
-            put_verified_stripe_record(&self.part, stripe, record.clone());
-        }
         if cached.records.len() == 2 {
             cached.records.pop_front();
         }
@@ -897,41 +863,4 @@ mod tests {
             prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
         }
     }
-
-    #[tokio::test]
-    async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
-        let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
-        let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
-        let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
-        builder.push(shards.into_iter()).await.expect("stripe");
-        let prepared = builder.finish(8).await.expect("index");
-        let inline_bytes = prepared.inline_bytes().expect("inline bytes");
-        let part = prepared.part;
-
-        let warm_reader = PartProofReader {
-            part: part.clone(),
-            sources: Vec::new(),
-            volume: "bucket".to_owned(),
-            inline: Some(inline_bytes),
-            cached: Mutex::new(ProofCache::default()),
-        };
-        warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
-        assert!(
-            get_verified_stripe_record(&part, 0).is_some(),
-            "small verified stripe record should populate the verified stripe cache"
-        );
-
-        let cached_reader = Arc::new(PartProofReader {
-            part,
-            sources: Vec::new(),
-            volume: "bucket".to_owned(),
-            inline: None,
-            cached: Mutex::new(ProofCache::default()),
-        });
-        cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
-        assert!(
-            cached_reader.verify(0, 0, shards[1]).await.is_err(),
-            "cached record must still reject corrupt or mismatched shard payloads"
-        );
-    }
 }

crates/ecstore/src/object_api/types.rs

@@ -160,6 +160,20 @@ pub struct ObjectLockConfigSnapshot {
 }
 
 impl ObjectLockConfigSnapshot {
+    pub(crate) fn owns_lifecycle_guard(&self) -> bool {
+        self._lifecycle_guard.is_some()
+    }
+
+    pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
+        self.bucket.as_deref() == Some(bucket)
+            && incarnation.is_some()
+            && self.bucket_incarnation_id == incarnation
+            && self
+                ._lifecycle_guard
+                .as_ref()
+                .is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
+    }
+
     pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
         Self {
             store_id: None,

crates/ecstore/src/services/rebalance/worker.rs

@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     }
 }
 
-fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
-    match rebalance_error_source(err) {
-        Error::Lock(err) => is_rebalance_transient_lock_error(err),
-        // No metadata was written when lock admission failed. Each attempt
-        // must acquire its quorum again before loading or saving this run.
-        Error::NamespaceLockQuorumUnavailable {
-            mode: "write",
-            bucket,
-            object,
-            ..
-        } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
-        _ => false,
-    }
-}
-
 fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return true;
@@ -400,7 +385,9 @@ where
         match result {
             Ok(value) => return Ok(value),
             Err(err) => {
-                if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
+                if attempt.saturating_add(1) >= max_attempts.max(1)
+                    || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
+                {
                     return Err(err);
                 }
                 #[cfg(test)]
@@ -732,82 +719,6 @@ mod error_source_tests {
         }
     }
 
-    #[tokio::test(start_paused = true)]
-    async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
-        let mut attempts = 0;
-        retry_rebalance_metadata_access(None, 3, || {
-            attempts += 1;
-            std::future::ready(if attempts == 1 {
-                resolve_rebalance_meta_save_result(
-                    Err(rebalance_meta_lock_error(
-                        rustfs_lock::LockError::QuorumNotReached {
-                            required: 1,
-                            achieved: 0,
-                        },
-                        "write",
-                    )),
-                    "stop_rebalance",
-                )
-            } else {
-                Ok(())
-            })
-        })
-        .await
-        .expect("a later quorum-protected admission must be allowed to succeed");
-        assert_eq!(attempts, 2);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
-        for max_attempts in [0, 1, 3] {
-            let mut attempts = 0;
-            let result = retry_rebalance_metadata_access(None, max_attempts, || {
-                attempts += 1;
-                std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
-                    rustfs_lock::LockError::QuorumNotReached {
-                        required: 1,
-                        achieved: 0,
-                    },
-                    "write",
-                )))
-            })
-            .await;
-            let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
-            assert_eq!(attempts, max_attempts.max(1));
-            assert!(matches!(
-                rebalance_error_source(&error),
-                Error::NamespaceLockQuorumUnavailable {
-                    mode: "write",
-                    required: 1,
-                    achieved: 0,
-                    ..
-                }
-            ));
-        }
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
-        let mut attempts = 0;
-        let result = retry_rebalance_metadata_access(None, 3, || {
-            attempts += 1;
-            std::future::ready(Err::<(), _>(if attempts == 1 {
-                rebalance_meta_lock_error(
-                    rustfs_lock::LockError::QuorumNotReached {
-                        required: 1,
-                        achieved: 0,
-                    },
-                    "write",
-                )
-            } else {
-                Error::OperationCanceled
-            }))
-        })
-        .await;
-        assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
-        assert!(matches!(result, Err(Error::OperationCanceled)));
-    }
-
     #[tokio::test]
     async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
         for err in [
@@ -820,20 +731,6 @@ mod error_source_tests {
                 required: 3,
                 achieved: 2,
             },
-            Error::NamespaceLockQuorumUnavailable {
-                mode: "write",
-                bucket: "data-bucket".to_string(),
-                object: REBAL_META_NAME.to_string(),
-                required: 1,
-                achieved: 0,
-            },
-            Error::NamespaceLockQuorumUnavailable {
-                mode: "write",
-                bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
-                object: "other-metadata.bin".to_string(),
-                required: 1,
-                achieved: 0,
-            },
             Error::other("stale rebalance run rejected: lock acquisition timed out"),
             Error::other("rebalance distributed run fence lost"),
             Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -2538,6 +2538,7 @@ pub(in crate::set_disk) struct RenameDataCommit {
     pub(in crate::set_disk) old_current_source: Option<FileInfo>,
     pub(in crate::set_disk) committed_file_info: FileInfo,
     pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
+    pub(in crate::set_disk) tail_has_metadata_reservations: bool,
 }
 
 pub(in crate::set_disk) struct RenameTailCleanup {
@@ -3468,6 +3469,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
     rollback_receipt: Option<RenameRollbackReceipt>,
     multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
     multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
+    reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
+    prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
 }
 
 impl<'a> RenameDataFenceOptions<'a> {
@@ -3483,6 +3486,8 @@ impl<'a> RenameDataFenceOptions<'a> {
             rollback_receipt: None,
             multipart_proof_authority: None,
             multipart_commit_owner: None,
+            reservation_ctx: None,
+            prepared_reservations: None,
         }
     }
 
@@ -3519,6 +3524,165 @@ impl<'a> RenameDataFenceOptions<'a> {
         self.namespace_commit_guard = namespace_commit_guard;
         self
     }
+
+    pub(in crate::set_disk) fn with_reservation_context(
+        mut self,
+        ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
+        reservations: Option<Vec<Option<RenameMetadataReservation>>>,
+    ) -> Self {
+        self.reservation_ctx = ctx;
+        self.prepared_reservations = reservations;
+        self
+    }
+}
+
+pub(crate) struct RenameMetadataReservation {
+    disk: DiskStore,
+    token: Option<Uuid>,
+    prepared_at: tokio::time::Instant,
+}
+
+impl RenameMetadataReservation {
+    async fn cancel(&mut self) -> disk::error::Result<()> {
+        let Some(token) = self.token else { return Ok(()) };
+        match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
+            Ok(Ok(true)) => {
+                self.token = None;
+                Ok(())
+            }
+            Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
+            Ok(Err(error)) => Err(error),
+            Err(_) => Err(DiskError::Timeout),
+        }
+    }
+}
+
+impl Drop for RenameMetadataReservation {
+    fn drop(&mut self) {
+        let Some(token) = self.token.take() else { return };
+        if let Ok(runtime) = tokio::runtime::Handle::try_current() {
+            let disk = self.disk.clone();
+            runtime.spawn(async move {
+                let _ = disk.cancel_object_metadata_reservation(token).await;
+            });
+        }
+    }
+}
+
+pub(crate) async fn prepare_rename_metadata_reservations(
+    disks: &[Option<DiskStore>],
+    bucket: &str,
+    object: &str,
+    incarnation: Uuid,
+    ctx: &Arc<crate::runtime::instance::InstanceContext>,
+    owns_bucket_fence: bool,
+) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
+    let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
+        let Some(disk) = disk else { return Ok(None) };
+        if disk.is_local() {
+            crate::store::admit_local_disk(ctx, disk, None, false).await?;
+        }
+        let prepared_at = tokio::time::Instant::now();
+        let token = if disk.is_local() && owns_bucket_fence {
+            let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
+                .await
+                .ok_or(DiskError::DiskOngoingReq)?;
+            store
+                .reserve_local_object_metadata_at_incarnation(
+                    &disk.endpoint().to_string(),
+                    bucket,
+                    object,
+                    incarnation,
+                    Duration::from_secs(5),
+                )
+                .await?
+        } else {
+            disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
+                .await?
+        };
+        // No await separates receipt from installing the cancellation owner.
+        Ok(Some(RenameMetadataReservation {
+            disk: disk.clone(),
+            token: Some(token),
+            prepared_at,
+        }))
+    }))
+    .await;
+    let mut reservations = Vec::with_capacity(prepared.len());
+    let mut failed = false;
+    for result in prepared {
+        match result {
+            Ok(reservation) => reservations.push(reservation),
+            Err(_) => {
+                failed = true;
+                reservations.push(None);
+            }
+        }
+    }
+    if failed {
+        // Prepared tokens are inert; dropping them cannot leave a queued writer.
+        return Ok(None);
+    }
+    Ok(Some(reservations))
+}
+
+pub(crate) fn align_rename_metadata_reservations(
+    mut reservations: Vec<Option<RenameMetadataReservation>>,
+    disks: &[Option<DiskStore>],
+) -> Option<Vec<Option<RenameMetadataReservation>>> {
+    if reservations.len() != disks.len() {
+        return None;
+    }
+    let mut aligned = Vec::with_capacity(disks.len());
+    for disk in disks {
+        let Some(disk) = disk else {
+            aligned.push(None);
+            continue;
+        };
+        let index = reservations
+            .iter()
+            .position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
+        aligned.push(reservations[index].take());
+    }
+    // Disks lost while staging need no activation; dropping their still-inert
+    // reservations schedules cancellation without blocking the commit.
+    Some(aligned)
+}
+
+async fn activate_rename_metadata_reservations(
+    mut reservations: Vec<Option<RenameMetadataReservation>>,
+    bucket: &str,
+    object: &str,
+    ctx: &Arc<crate::runtime::instance::InstanceContext>,
+) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
+    // Tokens expire after 60 seconds. Before sending any activation, fall
+    // back if a long upload consumed the safety margin for its RPC deadline.
+    if reservations
+        .iter()
+        .flatten()
+        .any(|reservation| reservation.prepared_at.elapsed() >= Duration::from_secs(50))
+    {
+        return Ok(None);
+    }
+    let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
+        if reservation.disk.is_local() {
+            crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
+        }
+        let token = reservation.token.ok_or(DiskError::Unexpected)?;
+        reservation
+            .disk
+            .activate_object_metadata(bucket, object, token, Duration::from_secs(5))
+            .await
+    }))
+    .await;
+    if activated.into_iter().any(|result| result.is_err()) {
+        let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
+        if cancelled.into_iter().any(|result| result.is_err()) {
+            return Err(DiskError::DiskOngoingReq);
+        }
+        return Ok(None);
+    }
+    Ok(Some(reservations))
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3530,7 +3694,7 @@ type RenameDataLegacyTuple = (
     Option<OldCurrentSize>,
 );
 
-fn put_rename_early_ack_enabled() -> bool {
+pub(crate) fn put_rename_early_ack_enabled() -> bool {
     rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
 }
 
@@ -3896,6 +4060,7 @@ impl SetDisks {
             old_current_source,
             committed_file_info,
             tail_drain: None,
+            tail_has_metadata_reservations: false,
         })
     }
 
@@ -3992,6 +4157,8 @@ impl SetDisks {
             rollback_receipt,
             multipart_proof_authority: _,
             multipart_commit_owner: _,
+            reservation_ctx,
+            prepared_reservations,
         } = fence_options;
         if let Some(file_info) = disks
             .iter()
@@ -4007,6 +4174,11 @@ impl SetDisks {
 
         let disk_count = disks.len();
         let fanout_disks = disks.to_vec();
+        let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
+            (Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
+            (_, _) => None,
+        };
+        let tail_has_metadata_reservations = reservations.is_some();
         let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
         let coordinator_disks = fanout_disks.clone();
         let src_bucket = Arc::new(src_bucket.to_string());
@@ -4039,6 +4211,11 @@ impl SetDisks {
                     let file_info = file_info.clone();
                     let successful_rename_completion_rank = successful_rename_completion_rank.clone();
                     let namespace_commit_guard = namespace_commit_guard.clone();
+                    let reservation = reservations
+                        .as_mut()
+                        .and_then(|entries| entries.get_mut(i))
+                        .and_then(Option::take);
+                    let reservation_ctx = reservation_ctx.clone();
                     tasks.spawn(async move {
                         let _namespace_commit_guard = namespace_commit_guard.clone();
                         let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4049,6 +4226,12 @@ impl SetDisks {
                             let Some(disk) = disk else {
                                 return Err(DiskError::DiskNotFound);
                             };
+                            let disk_admission = if reservation.is_some() && disk.is_local() {
+                                let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
+                                crate::store::admit_local_disk(ctx, &disk, None, true).await?
+                            } else {
+                                None
+                            };
                             let disk_wait_location_stage = if disk.is_local() {
                                 rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
                             } else {
@@ -4074,6 +4257,11 @@ impl SetDisks {
                             let disk_wait_started = rustfs_io_metrics::put_stage_timer();
                             dispatch_state = RenameDispatchState::MayHavePublished;
                             let _foreground_active_guard = rename_foreground_active_guard(i);
+                            let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
+                                Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
+                            } else {
+                                namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
+                            };
                             let observed = disk
                                 .rename_data_borrowed_with_fence_observed(
                                     &src_bucket,
@@ -4083,9 +4271,8 @@ impl SetDisks {
                                     &dst_object,
                                     disk::RenameDataGuards {
                                         scanner_publication_lease_token,
-                                        namespace_owner: namespace_commit_guard
-                                            .clone()
-                                            .map(|owner| owner as Arc<dyn Send + Sync>),
+                                        metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
+                                        namespace_owner,
                                         ..Default::default()
                                     },
                                 )
@@ -4363,6 +4550,7 @@ impl SetDisks {
         let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
         commit.map(|mut commit| {
             commit.tail_drain = Some(tail_drain);
+            commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
             rustfs_io_metrics::record_put_object_stage_duration_from(
                 rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
                 attach_tail_owner_started,
@@ -4432,6 +4620,8 @@ impl SetDisks {
             rollback_receipt,
             multipart_proof_authority,
             multipart_commit_owner,
+            reservation_ctx: _,
+            prepared_reservations: _,
         } = fence_options;
         if let Some(file_info) = disks
             .iter()
@@ -4901,6 +5091,7 @@ impl SetDisks {
             old_current_source,
             committed_file_info,
             tail_drain: None,
+            tail_has_metadata_reservations: false,
         })
     }
 
@@ -5340,14 +5531,6 @@ impl SetDisks {
         }
 
         let disks = self.get_disks_internal().await;
-        if disks.len() == self.set_drive_count
-            && disks.iter().all(|disk| {
-                disk.as_ref()
-                    .is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
-            })
-        {
-            return Ok(false);
-        }
         let transaction_path = part_transaction_path(dst_object);
         let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
 
@@ -5547,14 +5730,6 @@ impl SetDisks {
         write_quorum: usize,
     ) -> disk::error::Result<()> {
         let disks = self.get_disks_internal().await;
-        if disks.len() == self.set_drive_count
-            && disks.iter().all(|disk| {
-                disk.as_ref()
-                    .is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
-            })
-        {
-            return Ok(());
-        }
         let listings = join_all(disks.iter().map(|disk| {
             let disk = disk.clone();
             async move {
@@ -5762,7 +5937,7 @@ impl SetDisks {
         let all_prepared_succeeded = errs
             .iter()
             .zip(prepared_disks.iter())
-            .all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
+            .all(|(err, disk)| disk.is_none() || err.is_none());
         let committed = if all_prepared_succeeded {
             let settles = prepared_disks.iter().map(|disk| {
                 let disk = disk.clone();

crates/ecstore/src/set_disk/metadata.rs

@@ -161,7 +161,7 @@ impl SetDisks {
             )
         });
         let mut metadata = FileMeta::new();
-        for (version_id, parts) in versions {
+        for parts in versions.into_values() {
             let errors = parts
                 .iter()
                 .zip(&missing)
@@ -173,25 +173,15 @@ impl SetDisks {
                     }
                 })
                 .collect::<Vec<_>>();
-            let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
-                Ok((read, write)) => (
-                    usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
-                    usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
-                ),
+            let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
+                Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
                 Err(DiskError::ErasureReadQuorum) if complete => continue,
                 Err(error) => return Err(error),
             };
-            let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
-                // A null-version system record must carry the same commit proof
-                // as GET. Listing only needs the selected metadata, so no disk
-                // handles are retained from the shared read selector.
-                Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
-            } else {
-                let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
-                Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
-            };
+            let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
+            let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
             metadata.add_version(selected)?;
         }
         if metadata.versions.is_empty() {

crates/ecstore/src/set_disk/mod.rs

@@ -410,6 +410,10 @@ impl ObjectLockDiagGuard {
         self
     }
 
+    async fn downgrade_committed_tail(&mut self) -> bool {
+        self.guard.downgrade_committed_tail().await
+    }
+
     /// Whether the underlying namespace lock's heartbeat has observed a
     /// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
     /// point on this so a stale lock holder does not race a double-write.
@@ -902,6 +906,10 @@ mod core;
 pub(crate) use core::io_primitives::disk_call_counters;
 #[cfg(test)]
 pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
+pub(crate) use core::io_primitives::{
+    RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
+    put_rename_early_ack_enabled,
+};
 mod ctx;
 mod metadata;
 mod ops;
@@ -4696,17 +4704,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
             .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
 }
 
-// Reusing a held writer additionally requires the same quorum, so repeated
-// clients must not give different weight to an otherwise identical domain.
-fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
-    !left.is_empty()
-        && left.len() == right.len()
-        && left.iter().all(|client| {
-            left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
-                && right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
-        })
-}
-
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5656,9 +5653,9 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            // Distributed lock clients receive the raw ObjectKey; the pool
-            // namespace is not part of the key stored by their backends.
-            (true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
+            (true, true) => {
+                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
+            }
             _ => false,
         }
     }
@@ -9049,13 +9046,6 @@ mod tests {
             "constructing an object lock must not clone each client Arc"
         );
 
-        // Check shared ownership before acquisition can spawn pending cleanup.
-        let read_lock = set
-            .new_ns_lock("bucket", "read-object")
-            .await
-            .expect("second namespace lock should be created");
-        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
-
         let write_error = write_lock
             .get_write_lock(Duration::from_millis(500))
             .await
@@ -9070,6 +9060,11 @@ mod tests {
             ),
             "the shared client representation must preserve the exact write quorum result: {write_error}"
         );
+        let read_lock = set
+            .new_ns_lock("bucket", "read-object")
+            .await
+            .expect("second namespace lock should be created");
+        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
         let read_guard = read_lock
             .get_read_lock(Duration::from_millis(500))
             .await
@@ -9893,7 +9888,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
+    async fn namespace_lock_domain_includes_pool_namespace() {
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
 
         let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9932,25 +9927,11 @@ mod tests {
             "sets in the same pool share the object namespace lock domain"
         );
         assert!(
-            same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pools using identical backends lock the same raw object key"
+            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
+            "different pool namespaces must not be deduplicated solely by identical clients"
         );
     }
 
-    #[test]
-    fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
-        let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
-        assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
-        assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
-        assert!(!same_distributed_lock_quorum(
-            &[a.clone(), a.clone(), b.clone()],
-            &[a.clone(), b.clone(), b]
-        ));
-        assert!(!same_distributed_lock_quorum(&[], &[]));
-    }
-
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
 #[derive(Clone, Copy)]
 struct ExplicitVersionHeal<'a> {
     opts: &'a HealOpts,
+    staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
     allow_regeneration: bool,
     preserve_unrepairable: bool,
     commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
     retirement: Option<&'a MarkerRetirementContext<'a>>,
 }
 
+async fn restore_heal_exclusive_guard(
+    guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
+) -> disk::error::Result<()> {
+    tokio::time::timeout(get_lock_acquire_timeout(), async {
+        loop {
+            {
+                let mut guard = guard.lock().await;
+                if guard.is_lock_lost() {
+                    return Err(DiskError::DiskOngoingReq);
+                }
+                let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
+                if !matches!(waiting, Ok(Some(false))) {
+                    return Err(DiskError::Timeout);
+                }
+                if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
+                    .await
+                    .unwrap_or(false)
+                {
+                    // A writer may have waited on a peer that upgraded while
+                    // another peer was still waiting for its readers to drain.
+                    let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
+                    return if matches!(waiting, Ok(Some(false))) {
+                        Ok(())
+                    } else {
+                        Err(DiskError::Timeout)
+                    };
+                }
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .map_err(|_| DiskError::Timeout)?
+}
+
+async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
+    loop {
+        tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+        let waiting =
+            tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
+        if !matches!(waiting, Ok(Some(false))) {
+            return;
+        }
+    }
+}
+
+fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
+    drop(tokio::spawn(async move {
+        let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
+    }));
+}
+
 // Keep the large heal state machine and its construction temporary out of
 // each dispatch caller's poll frame. Ordinary object writes do not use this
 // heal-only allocation boundary.
@@ -844,6 +897,7 @@ impl SetDisks {
             version_id,
             ExplicitVersionHeal {
                 opts,
+                staging_guard: None,
                 allow_regeneration: true,
                 preserve_unrepairable: false,
                 commit_fence: None,
@@ -994,6 +1048,7 @@ impl SetDisks {
     ) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
         let ExplicitVersionHeal {
             opts,
+            staging_guard,
             allow_regeneration: allow_explicit_version_regen,
             preserve_unrepairable,
             commit_fence,
@@ -1701,7 +1756,29 @@ impl SetDisks {
                         let dst_data_dir = data_dir;
 
                         if !latest_meta.deleted && !latest_meta.is_remote() {
-                            let stage_result: disk::error::Result<Option<DiskError>> = async {
+                            let mut shared_staging = false;
+                            if let Some(guard) = staging_guard {
+                                let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
+                                    guard.lock().await.supports_heal_lock_transition().await
+                                })
+                                .await
+                                .unwrap_or(false);
+                                if supported {
+                                    let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
+                                        guard.lock().await.downgrade_committed_tail().await
+                                    })
+                                    .await
+                                    .unwrap_or(false);
+                                    if converted {
+                                        shared_staging = true;
+                                    } else {
+                                        // Some peers may have converted. Abort without committing;
+                                        // dropping the guard releases each retained lease.
+                                        return Err(DiskError::Timeout);
+                                    }
+                                }
+                            }
+                            let stage = async {
                                 let erasure_info = latest_meta.erasure.clone();
                                 let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
 
@@ -1926,17 +2003,43 @@ impl SetDisks {
 
                                 writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
                                 Ok(None)
-                            }
-                            .await;
+                            };
+                            let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
+                                if let Some(guard) = staging_guard {
+                                    tokio::select! {
+                                        result = stage => result,
+                                        _ = wait_for_heal_writer(guard) => {
+                                            // The UUID prefix is private to this attempt. Cleanup must not
+                                            // retain the namespace lock while a foreground writer waits;
+                                            // the stale-tmp sweeper also covers a cancelled disk operation.
+                                            cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
+                                            return Err(DiskError::Timeout);
+                                        }
+                                    }
+                                } else {
+                                    stage.await
+                                }
+                            } else {
+                                stage.await
+                            };
                             match stage_result {
                                 Ok(None) => {}
                                 failure => {
-                                    // Release readers and writers before removing all parts
-                                    // staged under this heal's exclusively owned prefix.
-                                    let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
+                                    if shared_staging {
+                                        cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
+                                    } else {
+                                        let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
+                                    }
                                     return failure.map(|error| (result, error));
                                 }
                             }
+                            if shared_staging
+                                && let Some(guard) = staging_guard
+                                && let Err(error) = restore_heal_exclusive_guard(guard).await
+                            {
+                                cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
+                                return Err(error);
+                            }
                         }
                         let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
                             if _read_repair_read_lock_guard
@@ -3616,6 +3719,7 @@ impl SetDisks {
             &version,
             ExplicitVersionHeal {
                 opts: &opts,
+                staging_guard: None,
                 allow_regeneration: false,
                 preserve_unrepairable: true,
                 commit_fence: Some(can_commit),
@@ -3676,6 +3780,7 @@ impl SetDisks {
                         version_id,
                         ExplicitVersionHeal {
                             opts: &read_opts,
+                            staging_guard: None,
                             allow_regeneration: false,
                             preserve_unrepairable: true,
                             commit_fence: Some(&can_commit),
@@ -3702,7 +3807,7 @@ impl SetDisks {
                 .new_ns_lock(bucket, object)
                 .await
                 .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
-            Some(Arc::new(
+            Some(Arc::new(tokio::sync::Mutex::new(
                 measure_heal_object_lock(
                     "heal_object",
                     HealObjectLockKind::Write,
@@ -3714,7 +3819,7 @@ impl SetDisks {
                         .narrow_to_disk()
                         .unwrap_or_else(DiskError::other)
                 })?,
-            ))
+            )))
         } else {
             None
         };
@@ -3732,10 +3837,18 @@ impl SetDisks {
             } else {
                 None
             };
+        let write_lock_loss = if let Some(guard) = &_write_lock_guard {
+            guard.lock().await.lock_lost_signal()
+        } else {
+            None
+        };
         let locks_lost = || {
-            _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
+            write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
                 || transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
         };
+        // Recovery cleanup can outlive its future. Never permit reads until
+        // every physical operation holding its recovery owner has finished.
+        let recovery_tail = Arc::new(());
         let mut recovery_removed_version = false;
         if !opts.read_repair
             && _write_lock_guard.is_some()
@@ -3762,6 +3875,7 @@ impl SetDisks {
                             Arc::new((
                                 scope.clone(),
                                 self.ctx.begin_namespace_commit(),
+                                Arc::clone(&recovery_tail),
                                 _write_lock_guard.clone(),
                                 transaction_guard.clone(),
                             )) as Arc<dyn Send + Sync>
@@ -3885,6 +3999,9 @@ impl SetDisks {
                 version_id,
                 ExplicitVersionHeal {
                     opts: &inner_opts,
+                    staging_guard: (Arc::strong_count(&recovery_tail) == 1)
+                        .then_some(_write_lock_guard.as_ref())
+                        .flatten(),
                     allow_regeneration: true,
                     preserve_unrepairable: false,
                     commit_fence: Some(&commit_fence),
@@ -3909,6 +4026,9 @@ impl SetDisks {
                             version_id,
                             ExplicitVersionHeal {
                                 opts: &inner_opts,
+                                staging_guard: (Arc::strong_count(&recovery_tail) == 1)
+                                    .then_some(_write_lock_guard.as_ref())
+                                    .flatten(),
                                 allow_regeneration: true,
                                 preserve_unrepairable: false,
                                 commit_fence: Some(&commit_fence),
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
                 "",
                 ExplicitVersionHeal {
                     opts: &opts,
+                    staging_guard: None,
                     allow_regeneration: true,
                     preserve_unrepairable: false,
                     commit_fence: Some(&can_commit),
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
         assert_eq!(actual, expected);
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
+        use crate::object_api::WriteCompletion;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "background-heal-storage-entry";
+        let object = "object.bin";
+        let expected = vec![0x37; 1024 * 1024];
+        set.make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(expected.clone()),
+            &ObjectOptions {
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("write object");
+        let metadata = disks[0]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("read metadata");
+        let missing = dirs[0]
+            .path()
+            .join(bucket)
+            .join(object)
+            .join(metadata.data_dir.expect("external object data").to_string())
+            .join("part.1");
+        tokio::fs::remove_file(&missing).await.expect("remove shard");
+
+        let (_pause, arrived, release) =
+            ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
+        let heal_set = set.clone();
+        let heal = tokio::spawn(async move {
+            let mut absence = None;
+            let result = heal_set
+                .heal_object_with_retirement(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        scan_mode: HealScanMode::Deep,
+                        read_repair: false,
+                        ..Default::default()
+                    },
+                    &mut absence,
+                    AbsenceProofRequest {
+                        retirement: None,
+                        allow_unversioned: false,
+                    },
+                )
+                .await;
+            (result, absence)
+        });
+        tokio::time::timeout(Duration::from_secs(10), arrived.notified())
+            .await
+            .expect("read-repair reached shard reconstruction");
+        let mut reader = tokio::time::timeout(
+            Duration::from_millis(200),
+            set.get_object_reader(
+                bucket,
+                object,
+                None,
+                http::HeaderMap::new(),
+                &ObjectOptions {
+                    suppress_read_repair: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("background heal reconstruction should allow concurrent readers")
+        .expect("open object while reconstruction is paused");
+        let mut during_heal = Vec::new();
+        tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
+            .await
+            .expect("read completes while reconstruction is paused")
+            .expect("read object during reconstruction");
+        assert_eq!(during_heal, expected);
+        drop(reader);
+        release.notify_one();
+        let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
+            .await
+            .expect("repair finishes")
+            .expect("repair task joins");
+        let (_, error) = result.expect("repair operation succeeds");
+        assert!(error.is_none(), "repair failed: {error:?}");
+        assert!(absence.is_none());
+        assert!(missing.exists(), "missing shard was not restored");
+        let mut reader = set
+            .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read repaired object");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read full object");
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
+        use crate::object_api::WriteCompletion;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "background-heal-long-put";
+        let object = "object.bin";
+        set.make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
+            &ObjectOptions {
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("write original object");
+        let metadata = disks[0]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("read metadata");
+        let missing = dirs[0]
+            .path()
+            .join(bucket)
+            .join(object)
+            .join(metadata.data_dir.expect("external data").to_string())
+            .join("part.1");
+        tokio::fs::remove_file(missing).await.expect("remove shard");
+        let (_pause, arrived, release) =
+            ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
+        let heal_set = set.clone();
+        let heal = tokio::spawn(async move {
+            let mut absence = None;
+            heal_set
+                .heal_object_with_retirement(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        scan_mode: HealScanMode::Deep,
+                        ..Default::default()
+                    },
+                    &mut absence,
+                    AbsenceProofRequest {
+                        retirement: None,
+                        allow_unversioned: false,
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(10), arrived.notified())
+            .await
+            .expect("reconstruction started");
+        let put = tokio::time::timeout(
+            Duration::from_secs(7),
+            set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
+        )
+        .await;
+        release.notify_one();
+        let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
+            .await
+            .expect("heal finishes after releasing reconstruction")
+            .expect("heal task joins");
+        put.expect("foreground PUT finishes during reconstruction")
+            .expect("foreground PUT must not time out");
+        assert!(
+            matches!(heal_result, Err(Error::Timeout)),
+            "interrupted heal must be retryable: {heal_result:?}"
+        );
+        drop(_pause);
+        let mut absence = None;
+        let (_, retry_error) = set
+            .heal_object_with_retirement(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    scan_mode: HealScanMode::Deep,
+                    ..Default::default()
+                },
+                &mut absence,
+                AbsenceProofRequest {
+                    retirement: None,
+                    allow_unversioned: false,
+                },
+            )
+            .await
+            .expect("a later heal must retry the current object");
+        assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
+        let mut reader = set
+            .get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read foreground write after heal");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read full object");
+        assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
+        use crate::object_api::WriteCompletion;
+        use std::time::Duration;
+        use tokio::io::AsyncReadExt;
+
+        let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "background-heal-version-delete";
+        let object = "object.bin";
+        set.make_bucket(
+            bucket,
+            &MakeBucketOptions {
+                versioning_enabled: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("create versioned bucket");
+        let original = set
+            .put_object(
+                bucket,
+                object,
+                &mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
+                &ObjectOptions {
+                    versioned: true,
+                    write_completion: WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write original version");
+        let version = original.version_id.expect("version id").to_string();
+        let survivor = set
+            .put_object(
+                bucket,
+                object,
+                &mut PutObjReader::from_vec(vec![0x52; 1024]),
+                &ObjectOptions {
+                    versioned: true,
+                    write_completion: WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write surviving version")
+            .version_id
+            .expect("surviving version id");
+        let metadata = disks[0]
+            .read_version("", bucket, object, &version, &ReadOptions::default())
+            .await
+            .expect("read original metadata");
+        let missing = dirs[0]
+            .path()
+            .join(bucket)
+            .join(object)
+            .join(metadata.data_dir.expect("external data").to_string())
+            .join("part.1");
+        tokio::fs::remove_file(missing).await.expect("remove shard");
+        let (_pause, arrived, release) =
+            ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
+        let heal_set = set.clone();
+        let heal_version = version.clone();
+        let heal = tokio::spawn(async move {
+            heal_set
+                .heal_object_with_retirement(
+                    bucket,
+                    object,
+                    &heal_version,
+                    &HealOpts {
+                        scan_mode: HealScanMode::Deep,
+                        ..Default::default()
+                    },
+                    &mut None,
+                    AbsenceProofRequest {
+                        retirement: None,
+                        allow_unversioned: false,
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(10), arrived.notified())
+            .await
+            .expect("reconstruction started");
+        let delete = tokio::time::timeout(
+            Duration::from_secs(7),
+            set.delete_object(
+                bucket,
+                object,
+                ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version.clone()),
+                    object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
+                        crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await;
+        release.notify_one();
+        let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
+            .await
+            .expect("heal finishes")
+            .expect("heal task joins");
+        delete
+            .expect("foreground delete finishes")
+            .expect("foreground delete succeeds");
+        assert!(
+            matches!(heal_result, Err(Error::Timeout)),
+            "interrupted heal must be retryable: {heal_result:?}"
+        );
+        let result = set
+            .get_object_info(
+                bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version.clone()),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
+            "stale heal must not resurrect the deleted version: {result:?}"
+        );
+        let mut survivor = set
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                http::HeaderMap::new(),
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(survivor.to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("surviving version remains readable");
+        let mut body = Vec::new();
+        survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
+        assert_eq!(body, vec![0x52; 1024]);
+        for disk in &disks {
+            let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
+            assert!(
+                matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
+                "deleted version must remain absent on each disk: {physical:?}"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
         let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -41,8 +41,8 @@ use super::super::{
     SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
     Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
     complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
-    create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
-    get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
+    create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
+    get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
     log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
     prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
     reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
@@ -124,10 +124,9 @@ async fn create_part_writers(
         let Some(disk) = disk else {
             return (None, Some(DiskError::DiskNotFound));
         };
-        match create_bitrot_writer_with_orig_volume(
+        match create_bitrot_writer(
             false,
             Some(disk),
-            RUSTFS_META_MULTIPART_BUCKET,
             RUSTFS_META_TMP_BUCKET,
             path,
             length,
@@ -978,7 +977,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
         return Err(DiskError::DiskNotFound);
     }
 
-    if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
+    if !root_prefix.is_empty() {
         let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
             Ok(entries) => entries,
             Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
@@ -1003,13 +1002,6 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
         .into_iter()
         .map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
         .filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
-        .map(|sha_dir| {
-            if root_prefix.is_empty() {
-                sha_dir
-            } else {
-                format!("{root_prefix}/{sha_dir}")
-            }
-        })
         .collect::<Vec<_>>();
     let listed_upload_dirs = stream::iter(sha_dirs)
         .map(|sha_dir| {
@@ -1132,7 +1124,6 @@ impl SetDisks {
         orig_bucket: &str,
         error_path: &str,
         root_prefix: &str,
-        require_all_disks: bool,
     ) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
         let disks = self.disks.read().await.clone();
         if disks.is_empty() {
@@ -1174,8 +1165,7 @@ impl SetDisks {
             }
         }
 
-        let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
-        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
+        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
             return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
         }
 
@@ -1197,7 +1187,7 @@ impl SetDisks {
 
     pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
         let (_, paths, _, _) = self
-            .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
+            .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
             .await?;
         Ok(paths.into_iter().next())
     }
@@ -1210,9 +1200,8 @@ impl SetDisks {
         upload_identity: &str,
     ) -> Result<Vec<String>> {
         let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
-        let (_, candidate_paths, _, has_minority_candidate) = self
-            .discover_multipart_upload_paths(bucket, object, &expected_parent, false)
-            .await?;
+        let (_, candidate_paths, _, has_minority_candidate) =
+            self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
         if has_minority_candidate {
             return Err(Error::DecommissionCapacityBlocked {
                 message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
@@ -1761,54 +1750,6 @@ impl SetDisks {
         })
     }
 
-    pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
-        // Every disk must confirm absence in both namespaces before cleanup can
-        // remove catalog protection from the bucket.
-        for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
-            let (disks, paths, _, has_minority_candidate) =
-                self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
-            if has_minority_candidate {
-                return Err(Error::ErasureReadQuorum);
-            }
-            for upload_path in paths {
-                let (sha_dir, _) = upload_path
-                    .rsplit_once('/')
-                    .filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
-                    .ok_or(DiskError::CorruptedFormat)?;
-                let (metadata, errors) =
-                    Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
-                        .await?;
-                let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
-                let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
-                if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
-                    return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
-                }
-                let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
-                let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
-                let (Some(owner_bucket), Some(owner_object)) = (
-                    file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
-                    file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
-                ) else {
-                    return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
-                };
-                if owner_bucket.is_empty()
-                    || owner_object.is_empty()
-                    || Self::get_multipart_sha_dir(owner_bucket, owner_object)
-                        != sha_dir
-                            .strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
-                            .and_then(|suffix| suffix.strip_prefix('/'))
-                            .unwrap_or(sha_dir)
-                {
-                    return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
-                }
-                if owner_bucket == bucket {
-                    return Err(Error::BucketNotEmpty(bucket.to_string()));
-                }
-            }
-        }
-        Ok(())
-    }
-
     #[allow(clippy::too_many_arguments)]
     pub(crate) async fn list_multipart_uploads_for_incarnation(
         &self,
@@ -1820,8 +1761,7 @@ impl SetDisks {
         max_uploads: usize,
         expected_incarnation_id: Option<Uuid>,
     ) -> Result<ListMultipartsInfo> {
-        let (disks, candidate_paths, discovery_quorum, _) =
-            self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
+        let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
         let listed_uploads = stream::iter(candidate_paths)
             .map(|upload_path| {
                 let disks = &disks;
@@ -2432,7 +2372,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             let multipart_part_size = data.size();
             let stream = mem::replace(
                 &mut data.stream,
-                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
+                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
             );
 
             let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
@@ -2496,16 +2436,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             }
 
             let part_integrity = if let Some(integrity) = integrity {
-                Some(
-                    integrity
-                        .write(
-                            &mut shuffle_disks,
-                            RUSTFS_META_MULTIPART_BUCKET,
-                            RUSTFS_META_TMP_BUCKET,
-                            &tmp_part,
-                        )
-                        .await?,
-                )
+                Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
             } else { None };
             if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
                 return Err(Error::ErasureWriteQuorum);
@@ -4527,13 +4458,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts; deleting them before the commit would strand the upload
             // permanently. This mirrors the "clean up only after commit" pattern
             // already used for the old data-dir GC and the upload-dir delete_all below.
-            let can_skip_part_cleanup = !needs_upgrade
-                && commit_disks
-                    .iter()
-                    .all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
-            if !can_skip_part_cleanup {
-                commit_set.cleanup_multipart_path(&parts).await;
-            }
+            commit_set.cleanup_multipart_path(&parts).await;
 
             if let Some(old_dir) = op_old_dir {
                 // backlog#898: best-effort reclaim of the dereferenced old data dir.
@@ -10060,7 +9985,7 @@ mod tests {
 
         // Real disk callbacks and health probes must share wall-clock deadlines.
         // Advancing Tokio time can expire unrelated healthy-drive probes.
-        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
             let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
             let bucket = "mpu-quorum-success-physical";
@@ -10087,8 +10012,6 @@ mod tests {
                 local.observe_multipart_cleanup_for_test(&upload_path)
             }).collect::<Vec<_>>();
             let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
-            // Fault only the held leg; healthy disks keep their normal I/O budgets.
-            let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
             let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
             let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
             let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10245,15 +10168,7 @@ mod tests {
                 let size = copy_info.get_actual_size().expect("actual source size");
                 copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
                 tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
-                    .await.expect("cross-key Copy cannot wait on the old successful logical owner")
-                    .unwrap_or_else(|error| {
-                        let health = disks.iter().enumerate().map(|(slot, disk)| {
-                            let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
-                            let metrics = local.metrics_snapshot();
-                            (slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
-                        }).collect::<Vec<_>>();
-                        panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
-                    });
+                    .await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
                 drop(copy_info);
                 let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
                 let mut actual = Vec::new();
@@ -11282,163 +11197,6 @@ mod tests {
         assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
     }
 
-    #[tokio::test]
-    async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "multipart-cleanup-target";
-        let other = "multipart-cleanup-other";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("create target bucket");
-            disk.make_volume(other).await.expect("create other bucket");
-        }
-        set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect("an empty upload namespace permits cleanup");
-        set_disks
-            .new_multipart_upload(other, "object.bin", &ObjectOptions::default())
-            .await
-            .expect("create upload in other bucket");
-        set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect("an upload owned by another bucket permits cleanup");
-        set_disks
-            .new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
-            .await
-            .expect("create upload in target bucket");
-        assert!(
-            matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
-        );
-    }
-
-    #[tokio::test]
-    async fn multipart_absence_inspection_requires_every_disk_online() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "multipart-cleanup-offline";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("create bucket");
-        }
-        set_disks.disks.write().await[3] = None;
-        let listed = set_disks
-            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
-            .await
-            .expect("ordinary listing preserves its discovery quorum");
-        assert!(listed.uploads.is_empty());
-        let error = set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
-        assert!(
-            matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
-            "the offline disk must produce a bucket-scoped read quorum error: {error:?}"
-        );
-
-        set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
-        set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect("all disks can confirm the empty namespace after recovery");
-    }
-
-    #[tokio::test]
-    async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
-        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "multipart-cleanup-list-error";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("create bucket");
-        }
-        let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
-        let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
-        tokio::fs::create_dir_all(&multipart_dir)
-            .await
-            .expect("create multipart staging directory");
-        tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
-            .await
-            .expect("replace one upload parent directory with a file");
-        multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
-            .await
-            .expect_err("the affected disk must fail directory enumeration");
-        let listed = set_disks
-            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
-            .await
-            .expect("ordinary listing still accepts the remaining discovery quorum");
-        assert!(listed.uploads.is_empty());
-        let error = set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
-        assert!(
-            matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
-            "the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
-        );
-
-        tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
-            .await
-            .expect("remove the invalid directory entry");
-        set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect("all disks can confirm the empty namespace after repair");
-    }
-
-    #[tokio::test]
-    async fn multipart_absence_inspection_rejects_data_movement_uploads() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "multipart-cleanup-data-movement";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("create bucket");
-        }
-        let mut metadata = HashMap::new();
-        rustfs_utils::http::insert_str(
-            &mut metadata,
-            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
-            format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
-        );
-        set_disks
-            .new_multipart_upload(
-                bucket,
-                "object.bin",
-                &ObjectOptions {
-                    data_movement: true,
-                    user_defined: metadata,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("create data movement upload");
-        assert!(
-            matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
-        );
-    }
-
-    #[tokio::test]
-    async fn multipart_absence_inspection_rejects_unreadable_ownership() {
-        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "multipart-cleanup-corrupt";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("create bucket");
-        }
-        set_disks
-            .new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
-            .await
-            .expect("create multipart upload");
-        let mut corrupted = 0;
-        for temp_dir in &temp_dirs {
-            for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
-                tokio::fs::write(path, b"invalid upload metadata")
-                    .await
-                    .expect("corrupt upload metadata");
-                corrupted += 1;
-            }
-        }
-        assert!(corrupted > 0, "upload metadata must exist before corruption");
-        set_disks
-            .ensure_no_multipart_uploads(bucket)
-            .await
-            .expect_err("unreadable upload ownership must block cleanup");
-    }
-
     /// Regression (issue #5716): a single upload directory whose `xl.meta` was
     /// destroyed (crash mid-write, torn disk state) must degrade to that upload
     /// alone. Failing the whole ListMultipartUploads turns one piece of stale

crates/ecstore/src/set_disk/ops/object.rs

@@ -4184,7 +4184,19 @@ impl SetDisks {
         data: &mut PutObjReader,
         opts: &ObjectOptions,
     ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
-        self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
+        self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
+            .await
+    }
+
+    pub(crate) async fn put_object_with_prepared_reservations(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut PutObjReader,
+        opts: &ObjectOptions,
+        reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
+    ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
+        self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
             .await
     }
 
@@ -4203,7 +4215,7 @@ impl SetDisks {
         // The owned publication capability supplies the exact target write
         // domain at commit time; the generic PUT path must not reacquire it.
         opts.no_lock = true;
-        self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
+        self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
             .await
     }
 
@@ -4214,8 +4226,9 @@ impl SetDisks {
         data: &'a mut PutObjReader,
         opts: &'a ObjectOptions,
         publication_fence: Option<RemoteTuplePublicationFence>,
+        reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
     ) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
-        Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
+        Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
     }
 
     async fn put_object_with_old_current_size_inner(
@@ -4225,6 +4238,7 @@ impl SetDisks {
         data: &mut PutObjReader,
         opts: &ObjectOptions,
         mut publication_fence: Option<RemoteTuplePublicationFence>,
+        reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
     ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
         let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
         let protect_write = opts.shard_integrity_write_enabled();
@@ -4534,7 +4548,7 @@ impl SetDisks {
 
             let stream = mem::replace(
                 &mut data.stream,
-                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
+                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
             );
 
             let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
@@ -4789,6 +4803,41 @@ impl SetDisks {
                 .await?;
             }
 
+            // Remote publication owners were prepared before bucket metadata
+            // locks. Local-only fixtures can prepare before the object lock.
+            let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
+                && crate::set_disk::put_rename_early_ack_enabled()
+                && write_context.acquires_namespace()
+                && write_context.allows_early_ack(opts.write_completion)
+                && publication_fence.is_none()
+                && opts.scanner_publication_commit_scope.is_none()
+                && opts.decommission_capacity_admission.is_none()
+                && !opts.data_movement
+                && !is_meta_bucketname(bucket)
+            {
+                if let Some(prepared) = reservations {
+                    crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
+                } else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
+                    && crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
+                        .await
+                        .is_none()
+                    && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
+                {
+                    crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
+                        &shuffle_disks,
+                        bucket,
+                        object,
+                        incarnation,
+                        &self.ctx,
+                        false,
+                    )
+                    .await?
+                } else {
+                    None
+                }
+            } else {
+                None
+            };
             if let Some(store) = opts.decommission_capacity_admission.as_ref() {
                 #[cfg(test)]
                 {
@@ -5268,6 +5317,11 @@ impl SetDisks {
             let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
             let commit_decommission_capacity_guard = decommission_capacity_guard.take();
             let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
+            let commit_object_lock_snapshot = opts
+                .object_lock_config_snapshot
+                .as_ref()
+                .filter(|snapshot| snapshot.owns_lifecycle_guard())
+                .cloned();
             // A scanner publication scope owns the movement permit until the
             // complete rename fan-out drains. Keep this path synchronous so
             // its terminal state is known before the coordinator releases
@@ -5281,6 +5335,26 @@ impl SetDisks {
                 && !(opts.data_movement && opts.has_decommission_capacity_reservation())
                 && commit_owns_namespace_guard
                 && commit_scanner_publication_scope.is_none();
+            let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
+                || commit_object_lock_snapshot
+                    .as_ref()
+                    .is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
+            let reservation_ctx = (commit_allows_early_ack
+                && write_context.acquires_namespace()
+                && commit_object_lock_guard.is_some()
+                && commit_decommission_object_lock_guard.is_none()
+                && commit_decommission_capacity_guard.is_none()
+                && commit_publication_guard.is_none()
+                && !quota_mutation_fence
+                && !opts.data_movement
+                && !is_meta_bucketname(bucket)
+                && lifecycle_owned)
+                .then(|| Arc::clone(&self.ctx));
+            let prepared_metadata_reservations = if reservation_ctx.is_some() {
+                prepared_metadata_reservations
+            } else {
+                None
+            };
             // Full-tail callers also transfer owned guards to the coordinator:
             // cancelling their ACK waiter must not cancel an in-flight rename.
             let detach_commit_owner = commit_scanner_publication_scope.is_some()
@@ -5604,6 +5678,7 @@ impl SetDisks {
                     )
                     .with_publication_scope(commit_scanner_publication_scope.clone())
                     .with_rollback_receipt(commit_rollback_receipt.clone())
+                    .with_reservation_context(reservation_ctx, prepared_metadata_reservations)
                     .with_namespace_commit_guard(
                         (!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
                     ),
@@ -5627,6 +5702,7 @@ impl SetDisks {
                     pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
                 }
                 let mut rename_guard_release = None;
+                let mut committed_tail_guard = None;
                 let mut needs_immediate_heal = false;
                 let mut tail_owns_tmp_cleanup = false;
                 rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -5658,7 +5734,14 @@ impl SetDisks {
                             .or_else(|| commit_version_suspended.then(Uuid::nil))
                             .map(|version_id| version_id.to_string());
                         request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
-                        let object_lock_guard = _object_lock_guard.take();
+                        let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
+                            None
+                        } else {
+                            _object_lock_guard
+                                .take()
+                                .map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
+                        };
+                        committed_tail_guard = object_lock_guard.clone();
                         let publication_guard = _publication_guard.take();
                         let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
                         let decommission_object_lock_guard = _decommission_object_lock_guard.take();
@@ -5683,6 +5766,7 @@ impl SetDisks {
                                 write_context,
                                 publication_guard,
                                 bucket_lifecycle_guard,
+                                commit_object_lock_snapshot,
                                 decommission_object_lock_guard,
                                 decommission_capacity_guard,
                             ),
@@ -5704,6 +5788,7 @@ impl SetDisks {
                                 write_context,
                                 publication_guard,
                                 bucket_lifecycle_guard,
+                                object_lock_snapshot,
                                 decommission_object_lock_guard,
                                 decommission_capacity_guard,
                             ),
@@ -5717,6 +5802,7 @@ impl SetDisks {
                                 drop(write_context);
                                 drop(publication_guard);
                                 drop(bucket_lifecycle_guard);
+                                drop(object_lock_snapshot);
                                 cleanup_runtime::run(cleanup_reservation, async move {
                                     let zero_target_cleanup = targets.is_empty();
                                     rustfs_io_metrics::record_put_object_stage_duration(
@@ -5972,6 +6058,13 @@ impl SetDisks {
                     lock_held_post_rename_started,
                 );
                 let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
+                if let Some(guard) = committed_tail_guard.take() {
+                    // The committed tail still owns every physical rename. Convert
+                    // only after metadata invalidation and before acknowledging PUT.
+                    // Unsupported peers keep their exclusive leases and the tail
+                    // retains ownership until its physical work has drained.
+                    let _ = guard.lock().await.downgrade_committed_tail().await;
+                }
                 if let Some(release) = rename_guard_release.take() {
                     let _ = release.send(true);
                 }
@@ -6129,11 +6222,7 @@ impl SetDisks {
                     );
                 }
 
-                let all_local_disks_cleaned_tmp = all_commit_disks_local
-                    && online_disks.len() == commit_set.set_drive_count
-                    && online_disks.iter().all(Option::is_some)
-                    && !issue3031_diag_enabled();
-                if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
+                if !tail_owns_tmp_cleanup {
                     let cleanup_set = commit_set.clone();
                     let cleanup_tmp_dir = commit_tmp_dir.clone();
                     cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
@@ -22181,6 +22270,117 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
+        use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let (_dirs, disks, set) = hermetic_set_disks(4).await;
+            let bucket = "put-tail-lifecycle";
+            let object = "object";
+            for disk in &disks {
+                disk.make_volume(bucket).await.expect("create volume");
+            }
+            let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
+            let lifecycle = lock
+                .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
+                .await
+                .expect("lifecycle read lock");
+            let metadata = lock
+                .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
+                .await
+                .expect("metadata read lock");
+            let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
+                Uuid::new_v4(),
+                bucket,
+                Uuid::new_v4(),
+                OffsetDateTime::now_utc(),
+                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                lifecycle,
+                metadata,
+            ));
+            let opts = ObjectOptions {
+                object_lock_config_snapshot: Some(snapshot),
+                ..Default::default()
+            };
+            let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let writer_set = Arc::clone(&set);
+            let put = tokio::spawn(async move {
+                writer_set
+                    .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                .await
+                .expect("rename tail pauses");
+            put.await.expect("PUT joins").expect("PUT acknowledged");
+            let mut bucket_delete =
+                Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
+            let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
+            barrier.release();
+            drop(barrier);
+            let guard = match early {
+                Ok(result) => {
+                    drop(result.expect("early bucket lock result"));
+                    panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
+                }
+                Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
+                    .await
+                    .expect("lifecycle lock released after tail")
+                    .expect("bucket delete acquires lock"),
+            };
+            drop(guard);
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
+                (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
+            ],
+            async {
+                let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+                let bucket = "put-early-ack-tail-read";
+                let object = "early-ack-tail-read-object";
+                for disk in &disk_stores {
+                    disk.make_volume(bucket).await.expect("bucket volume should be created");
+                }
+                let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+                let put_set = Arc::clone(&set_disks);
+                let put = tokio::spawn(async move {
+                    let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+                    put_set
+                        .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+                        .await
+                });
+                tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
+                    .await
+                    .expect("tail disk should pause during rename");
+                put.await
+                    .expect("PUT task should join")
+                    .expect("PUT should be acknowledged at quorum");
+                let result = set_disks
+                    .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await;
+                let mut reader = result.expect("acknowledged object should remain readable while tail drains");
+                let mut body = Vec::new();
+                tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
+                    .await
+                    .expect("object body should not wait for the tail")
+                    .expect("object body should drain");
+                assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
+                rename_barrier.release();
+                drop(rename_barrier);
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
@@ -22253,6 +22453,200 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn early_ack_tail_does_not_timeout_the_next_put() {
+        use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let (_dirs, disks, set) = hermetic_set_disks(4).await;
+            let bucket = "put-early-ack-next-writer";
+            let object = "object";
+            for disk in &disks {
+                disk.make_volume(bucket).await.expect("create volume");
+            }
+            {
+                let map = set.ctx.local_disk_map();
+                let mut active = map.write().await;
+                for disk in &disks {
+                    active.insert(disk.endpoint().to_string(), Some(disk.clone()));
+                }
+            }
+            let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
+            let lifecycle = lifecycle_lock
+                .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
+                .await
+                .expect("lifecycle read lock");
+            let metadata = lifecycle_lock
+                .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
+                .await
+                .expect("metadata read lock");
+            let incarnation = Uuid::new_v4();
+            let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
+                Uuid::new_v4(),
+                bucket,
+                incarnation,
+                OffsetDateTime::now_utc(),
+                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                lifecycle,
+                metadata,
+            ));
+            let mut opts = ObjectOptions {
+                expected_bucket_incarnation_id: Some(incarnation),
+                object_lock_config_snapshot: Some(snapshot.clone()),
+                ..Default::default()
+            };
+            snapshot.add_lock_fences(&mut opts);
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
+            let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let first_set = Arc::clone(&set);
+            let first_opts = opts.clone();
+            let first = tokio::spawn(async move {
+                first_set
+                    .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                .await
+                .expect("first rename paused");
+            first.await.expect("first PUT joins").expect("first PUT acknowledged");
+            let second = tokio::time::timeout(
+                Duration::from_secs(7),
+                set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
+            )
+            .await;
+            barrier.release();
+            drop(barrier);
+            let second = second.expect("second PUT finishes").expect("second PUT must not time out");
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while rename_tasks.running() != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("both rename tails must drain");
+            let mut reader = set
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("read newest object");
+            let mut body = Vec::new();
+            reader.stream.read_to_end(&mut body).await.expect("read full body");
+            assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
+            let expected_etag = second.etag.as_ref().expect("second PUT ETag");
+            for disk in &disks {
+                let info = disk
+                    .read_version("", bucket, object, "", &ReadOptions::default())
+                    .await
+                    .expect("each disk must retain the latest version");
+                assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
+                let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
+                assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn early_ack_tail_cannot_restore_a_deleted_version() {
+        use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let (_dirs, disks, set) = hermetic_set_disks(4).await;
+            let bucket = "put-tail-version-delete";
+            let object = "object";
+            for disk in &disks {
+                disk.make_volume(bucket).await.expect("create volume");
+            }
+            {
+                let map = set.ctx.local_disk_map();
+                let mut active = map.write().await;
+                for disk in &disks {
+                    active.insert(disk.endpoint().to_string(), Some(disk.clone()));
+                }
+            }
+            let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
+            let lifecycle = lock
+                .get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
+                .await
+                .expect("lifecycle read lock");
+            let metadata = lock
+                .get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
+                .await
+                .expect("metadata read lock");
+            let incarnation = Uuid::new_v4();
+            let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
+                Uuid::new_v4(),
+                bucket,
+                incarnation,
+                OffsetDateTime::now_utc(),
+                crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                lifecycle,
+                metadata,
+            ));
+            let mut opts = ObjectOptions {
+                versioned: true,
+                expected_bucket_incarnation_id: Some(incarnation),
+                object_lock_config_snapshot: Some(snapshot.clone()),
+                ..Default::default()
+            };
+            snapshot.add_lock_fences(&mut opts);
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
+            let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let writer = Arc::clone(&set);
+            let put_opts = opts.clone();
+            let put = tokio::spawn(async move {
+                writer
+                    .put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                .await
+                .expect("rename pauses");
+            let version = put
+                .await
+                .expect("PUT joins")
+                .expect("PUT succeeds")
+                .version_id
+                .expect("version ID")
+                .to_string();
+            let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
+            let delete_set = Arc::clone(&set);
+            let mut delete_opts = opts;
+            delete_opts.version_id = Some(version.clone());
+            let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
+            delete_barrier.wait_until_paused().await;
+            delete_barrier.release();
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !delete_barrier.namespace_acquired() {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("version delete acquires namespace while old tail is paused");
+            barrier.release();
+            drop(barrier);
+            delete.await.expect("delete joins").expect("version delete succeeds");
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while rename_tasks.running() != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("rename tails drain");
+            for disk in &disks {
+                assert!(
+                    matches!(
+                        disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
+                        Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
+                    ),
+                    "late tail must not restore the deleted version"
+                );
+            }
+        })
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -44,16 +44,15 @@ pub(crate) mod lifecycle {
 
 pub(crate) mod list {
     pub(crate) use rustfs_storage_api::{
-        AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
-        ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
-        WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
+        ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
+        ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
+        WalkVersionsSortOrder,
     };
 }
 
 pub(crate) mod multipart {
     pub(crate) use rustfs_storage_api::{
-        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
-        MultipartUploadResult, PartInfo,
+        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
     };
 }
 

crates/ecstore/src/store/bucket.rs

@@ -523,22 +523,7 @@ impl ECStore {
     where
         F: FnOnce() -> Result<()> + Send,
     {
-        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
-    }
-
-    /// Validate an external fence and the bucket incarnation in one metadata transaction.
-    pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
-        &self,
-        bucket: &str,
-        config_file: &str,
-        data: Vec<u8>,
-        expected_incarnation_id: Uuid,
-        validate: F,
-    ) -> Result<OffsetDateTime>
-    where
-        F: FnOnce() -> Result<()> + Send,
-    {
-        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
+        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
     }
 
     pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {

crates/ecstore/src/store/bucket_fence.rs

@@ -45,7 +45,8 @@
 use crate::error::{Result, StorageError};
 use std::collections::HashMap;
 use std::future::Future;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, Weak};
+use std::time::Duration;
 
 use rustfs_lock::NamespaceLockGuard;
 use rustfs_lock::distributed_lock::LockLostSignal;
@@ -98,9 +99,74 @@ pub(super) struct FenceRegistration {
 #[derive(Default)]
 pub(crate) struct BucketFenceRegistry {
     entries: Mutex<HashMap<String, FenceEntry>>,
+    reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
+}
+
+#[derive(Default)]
+pub(super) struct ReservationFenceSlot {
+    pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
+}
+
+/// Shared by short bursts of reservations, but never kept alive by the cache.
+pub(super) struct ReservationFenceGuard {
+    publication: NamespaceLockGuard,
+    lifecycle: BucketIncarnationFenceGuard,
+    created_at: tokio::time::Instant,
+    _slot: Arc<ReservationFenceSlot>,
+}
+
+impl ReservationFenceGuard {
+    pub(super) fn new(
+        publication: NamespaceLockGuard,
+        lifecycle: BucketIncarnationFenceGuard,
+        slot: Arc<ReservationFenceSlot>,
+    ) -> Self {
+        Self {
+            publication,
+            lifecycle,
+            created_at: tokio::time::Instant::now(),
+            _slot: slot,
+        }
+    }
+
+    pub(super) fn is_reusable(&self) -> bool {
+        // Stop admitting new reservations through this shared reader so a
+        // queued bucket writer can progress after the existing tails drain.
+        self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
+    }
+
+    fn is_held(&self) -> bool {
+        !self.publication.is_lock_lost()
+            && !self.publication.is_released()
+            && !self.lifecycle.is_lock_lost()
+            && self
+                .lifecycle
+                .namespace_lock_guard()
+                .is_some_and(|guard| !guard.is_released())
+    }
+}
+
+impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
+    fn is_valid(&self) -> bool {
+        self.is_held()
+    }
 }
 
 impl BucketFenceRegistry {
+    pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
+        let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
+        let key = (bucket.to_owned(), incarnation);
+        if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
+            return slot;
+        }
+        if slots.len() >= 1024 {
+            slots.retain(|_, slot| slot.strong_count() != 0);
+        }
+        let slot = Arc::new(ReservationFenceSlot::default());
+        slots.insert(key, Arc::downgrade(&slot));
+        slot
+    }
+
     /// Register a new live guard for `bucket` and return the memoized
     /// incarnation id if one is valid for the current coverage window.
     fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {

crates/ecstore/src/store/heal.rs

@@ -3692,177 +3692,6 @@ mod tests {
         );
     }
 
-    // The RPC lock service uses the raw ObjectKey and one backend per node,
-    // even when the requesting sets carry different pool namespace strings.
-    async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
-        use rustfs_lock::client::{LockClient, local::LocalClient};
-
-        let clients = || {
-            (0..2)
-                .map(|_| {
-                    Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
-                })
-                .collect::<Vec<_>>()
-        };
-        let first = clients();
-        let second = if shared { first.clone() } else { clients() };
-        let mut pools = source.pools.clone();
-        for (pool, lockers) in pools.iter_mut().zip([first, second]) {
-            for set in &mut Arc::make_mut(pool).disk_set {
-                Arc::make_mut(set).lockers = lockers.clone();
-            }
-        }
-        let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
-        let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
-        Arc::new(ECStore {
-            id: Uuid::new_v4(),
-            disk_map: Default::default(),
-            pools,
-            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
-            pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
-            rebalance_meta: tokio::sync::RwLock::new(None),
-            decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
-            start_gate: tokio::sync::Mutex::new(()),
-            pool_meta_save_gate: tokio::sync::Mutex::new(state),
-            ctx: Arc::clone(&source.ctx),
-            bucket_fence_registry: Arc::default(),
-        })
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
-        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
-        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
-        let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
-        let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
-        let first_lock = first
-            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
-            .await
-            .expect("first pool lock");
-        let second_lock = second
-            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
-            .await
-            .expect("second pool lock");
-        let guard = first_lock
-            .get_write_lock(std::time::Duration::from_secs(2))
-            .await
-            .expect("first writer");
-        assert!(
-            second_lock
-                .get_write_lock(std::time::Duration::from_millis(150))
-                .await
-                .is_err(),
-            "different pool namespaces still conflict on the same backend and object key"
-        );
-        drop(guard);
-        second_lock
-            .get_write_lock(std::time::Duration::from_secs(3))
-            .await
-            .expect("second writer should acquire after the first releases");
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
-        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
-        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
-        let (item, error) = tokio::time::timeout(
-            std::time::Duration::from_secs(3),
-            store.handle_heal_object(
-                RUSTFS_META_BUCKET,
-                POOL_META_NAME,
-                "",
-                &HealOpts {
-                    pool: Some(1),
-                    dry_run: true,
-                    ..Default::default()
-                },
-            ),
-        )
-        .await
-        .expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
-        .expect("pool metadata heal inspection should return a result");
-        assert!(error.is_none(), "inspection should succeed: {error:?}");
-        assert_eq!(item.object, POOL_META_NAME);
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
-        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
-        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
-        let missing = remove_pool_meta_shard(&store, 1).await;
-        let (item, error) = tokio::time::timeout(
-            std::time::Duration::from_secs(10),
-            store.handle_heal_object(
-                RUSTFS_META_BUCKET,
-                POOL_META_NAME,
-                "",
-                &HealOpts {
-                    pool: Some(1),
-                    ..Default::default()
-                },
-            ),
-        )
-        .await
-        .expect("cross-pool repair must not reenter the shared writer")
-        .expect("pool metadata repair should return a result");
-        assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
-        assert_eq!(item.object, POOL_META_NAME);
-        assert!(
-            missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
-            "the selected missing shard must be restored"
-        );
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
-        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
-        let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
-        let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
-        let lock = target
-            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
-            .await
-            .expect("target lock");
-        let guard = lock
-            .get_write_lock(std::time::Duration::from_secs(2))
-            .await
-            .expect("hold target lock");
-        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
-        let heal_store = Arc::clone(&store);
-        let mut heal = tokio::spawn(async move {
-            heal_store
-                .handle_heal_object(
-                    RUSTFS_META_BUCKET,
-                    POOL_META_NAME,
-                    "",
-                    &HealOpts {
-                        pool: Some(1),
-                        dry_run: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-        });
-        tokio::time::timeout(
-            std::time::Duration::from_secs(3),
-            barrier.wait_until_external_heal_target_lock_attempted(),
-        )
-        .await
-        .expect("inspection must attempt the independent target lock");
-        assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
-        drop(guard);
-        let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
-            .await
-            .expect("inspection should progress after target lock release")
-            .expect("heal task should not panic")
-            .expect("pool metadata heal inspection should return a result");
-        assert!(error.is_none(), "inspection should succeed: {error:?}");
-        assert_eq!(item.object, POOL_META_NAME);
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {

crates/ecstore/src/store/list_objects.rs

@@ -37,8 +37,8 @@ use crate::error::{
 use crate::object_api::{ObjectInfo, ObjectOptions};
 use crate::set_disk::SetDisks;
 use crate::storage_api_contracts::list::{
-    AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
-    StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
+    ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
+    StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
 };
 use crate::store::ECStore;
 use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -80,21 +80,6 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
 type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
 
-#[async_trait::async_trait]
-impl AuthoritativeListOperations for ECStore {
-    async fn walk_authoritative(
-        self: Arc<Self>,
-        rx: CancellationToken,
-        bucket: &str,
-        prefix: &str,
-        result: Sender<ObjectInfoOrErr>,
-        opts: WalkOptions,
-    ) -> Result<()> {
-        check_list_objs_args(bucket, prefix, &None)?;
-        self.walk_authoritatively(rx, bucket, prefix, result, opts).await
-    }
-}
-
 /// Build the versioned listing options used by recursive object walks.
 ///
 /// `list_path_result` normally derives these scan bounds before dispatching to
@@ -1565,57 +1550,6 @@ impl ECStore {
 
         result
     }
-
-    async fn walk_authoritatively(
-        self: Arc<Self>,
-        cancellation: CancellationToken,
-        bucket: &str,
-        prefix: &str,
-        result: Sender<ObjectInfoOrErr>,
-        opts: WalkOptions,
-    ) -> Result<()> {
-        let cancel = cancellation.child_token();
-        let _cancel_on_drop = cancel.clone().drop_guard();
-        let mut scans = Vec::new();
-        let mut inputs = Vec::new();
-        for pool in &self.pools {
-            for set in &pool.disk_set {
-                let (sender, receiver) = mpsc::channel(1);
-                inputs.push(receiver);
-                let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
-                // Internal absence checks must include reserved object prefixes.
-                scan.skip_hidden_prefix_check = true;
-                scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
-            }
-        }
-        if scans.is_empty() {
-            return Err(StorageError::ErasureReadQuorum);
-        }
-
-        let (sender, receiver) = mpsc::channel(100);
-        let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
-        let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
-        let scan_results = join_all(scans).await;
-        let merge_result = merge.await.map_err(Error::from)?;
-        let forward_result = forwarder.await.map_err(Error::from);
-
-        // An unavailable set cannot establish absence, even when another set
-        // completed. Only an explicitly missing directory or volume is empty.
-        for scan in scan_results {
-            match scan {
-                Ok(outcome) if !outcome.limit_reached => {}
-                Ok(_) => return Err(StorageError::ErasureReadQuorum),
-                Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
-                Err(err) => return Err(err),
-            }
-        }
-        merge_result?;
-        forward_result?;
-        if cancellation.is_cancelled() {
-            return Err(Error::other("authoritative object walk was cancelled"));
-        }
-        Ok(())
-    }
 }
 
 /// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
@@ -2586,8 +2520,7 @@ impl SetDisks {
     /// Version scan for set-scoped callers that act on absence: set walks
     /// (tier reference proofs, tier-delete journals, pool maintenance) and the
     /// recursive prefix-delete scan. Every online drive is read and each
-    /// version uses the read quorum of its own erasure layout. Null versions
-    /// in the system bucket also retain GET's commit-proof selection. A
+    /// version is resolved at the read quorum of its own erasure layout; a
     /// version that cannot be ruled out fails the scan instead of being
     /// omitted. Merged `ListObjectVersions` pages go through `list_path`.
     async fn list_versions_authoritatively(
@@ -2926,7 +2859,6 @@ mod test {
     use crate::error::{Result, StorageError};
     use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
     use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
-    use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
     use crate::storage_api_contracts::namespace::NamespaceLocking as _;
     use crate::storage_api_contracts::object::ObjectIO as _;
     use bytes::Bytes;
@@ -5787,126 +5719,6 @@ mod test {
         ));
     }
 
-    #[test]
-    fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
-        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
-        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
-        for new_etag in ["new", "old"] {
-            let new =
-                test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
-            let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
-
-            assert!(matches!(
-                SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
-                Err(DiskError::ErasureReadQuorum)
-            ));
-        }
-    }
-
-    #[test]
-    fn authoritative_system_bucket_scan_selects_committed_null_version() {
-        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
-        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
-        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
-        for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
-            let entries = MetaCacheEntries(
-                (0..4)
-                    .map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
-                    .collect(),
-            );
-            let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
-                .expect("committed generation should resolve")
-                .expect("committed record should be present");
-            let info = resolved
-                .to_fileinfo(RUSTFS_META_BUCKET)
-                .expect("resolved metadata should decode");
-
-            assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
-        }
-    }
-
-    #[test]
-    fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
-        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
-        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
-        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
-        for (old_copies, new_copies, expected) in [
-            (2, 1, Some("old")),
-            (1, 2, None),
-            (0, 3, Some("new")),
-            (0, 2, Some("new")),
-            (0, 1, None),
-        ] {
-            let entries = MetaCacheEntries(
-                (0..4)
-                    .map(|slot| {
-                        if slot < old_copies {
-                            Some(old.clone())
-                        } else if slot < old_copies + new_copies {
-                            Some(new.clone())
-                        } else {
-                            None
-                        }
-                    })
-                    .collect(),
-            );
-            let errors = (0..4)
-                .map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
-                .collect::<Vec<_>>();
-            let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
-
-            if let Some(expected) = expected {
-                let info = resolved
-                    .expect("degraded committed record should resolve")
-                    .expect("degraded committed record should be present")
-                    .to_fileinfo(RUSTFS_META_BUCKET)
-                    .expect("resolved metadata should decode");
-                assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
-            } else {
-                assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
-            }
-        }
-    }
-
-    #[test]
-    fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
-        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
-        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
-        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
-        for old_copies in [1, 2] {
-            let entries = MetaCacheEntries(vec![
-                Some(new.clone()),
-                Some(new.clone()),
-                Some(old.clone()),
-                (old_copies == 2).then(|| old.clone()),
-            ]);
-            let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
-            let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
-                .expect("user bucket read quorum should resolve")
-                .expect("user bucket record should be present");
-            let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
-
-            assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
-        }
-    }
-
-    #[test]
-    fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
-        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
-        let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
-        let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
-        let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
-        let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
-            .expect("explicit version read quorum should resolve")
-            .expect("explicit version should be present");
-        let info = resolved
-            .to_fileinfo(RUSTFS_META_BUCKET)
-            .expect("resolved metadata should decode");
-
-        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
-        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
-    }
-
     #[test]
     fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
         let entry = test_object_meta_entry_with_erasure_versions(
@@ -6248,49 +6060,6 @@ mod test {
         Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
     }
 
-    #[tokio::test]
-    async fn authoritative_store_walk_includes_reserved_prefix_versions() {
-        let bucket = "catalog-cleanup-versions";
-        let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
-        let (_dirs, store) = single_set_store(16).await;
-        let set = &store.pools[0].disk_set[0];
-        for (slot, disk) in set.disks.read().await.iter().enumerate() {
-            let disk = disk.as_ref().expect("fixture drive is online");
-            disk.make_volume(bucket).await.expect("create bucket volume");
-            disk.write_all(
-                bucket,
-                &format!("{name}/{STORAGE_FORMAT_FILE}"),
-                versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
-            )
-            .await
-            .expect("persist reserved metadata versions");
-        }
-        let (sender, mut receiver) = mpsc::channel(1);
-        let walk = store
-            .clone()
-            .walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
-        let collect = async {
-            let mut versions = Vec::new();
-            while let Some(item) = receiver.recv().await {
-                assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
-                let item = item.item.expect("walk must return an object");
-                versions.push((item.name, item.version_id));
-            }
-            versions
-        };
-        let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
-            .await
-            .expect("authoritative reserved-prefix walk must terminate");
-        walk.expect("authoritative walk must succeed");
-        assert_eq!(
-            versions,
-            [
-                (name.to_string(), Some(Uuid::from_u128(1))),
-                (name.to_string(), Some(Uuid::from_u128(2)))
-            ]
-        );
-    }
-
     // A versioned DeleteObject(versionId) can reach write quorum while three
     // drives are offline; they keep the deleted version after they return. A
     // different drive going offline later must not turn that sub-quorum
@@ -6361,16 +6130,6 @@ mod test {
                 .await
                 .expect("set walk must terminate");
             assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
-            let (sender, mut receiver) = mpsc::channel(16);
-            let walk =
-                store
-                    .clone()
-                    .walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
-            let drain = async { while receiver.recv().await.is_some() {} };
-            let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
-                .await
-                .expect("authoritative store walk must terminate");
-            assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
             let recursive_delete_scan = set
                 .clone()
                 .inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)

crates/ecstore/src/store/mod.rs

@@ -456,6 +456,7 @@ mod rebalance;
 pub(crate) use rebalance::PoolSpaceForTest;
 pub(crate) mod utils;
 
+pub(crate) use peer::admit_local_disk;
 use peer::init_local_peer;
 pub use peer::{
     BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,

crates/ecstore/src/store/multipart.rs

@@ -19,28 +19,10 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
 use crate::storage_api_contracts::object::ObjectOperations as _;
 use futures::{StreamExt, stream};
-use rustfs_utils::path::encode_dir_object_ref;
 use std::collections::HashSet;
 
 const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
 
-#[async_trait::async_trait]
-impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
-    type Error = Error;
-
-    async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
-        let (_, lifecycle_guard) = self
-            .guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
-            .await?;
-        for pool in &self.pools {
-            for set in &pool.disk_set {
-                set.ensure_no_multipart_uploads(bucket).await?;
-            }
-        }
-        ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
-    }
-}
-
 #[cfg(test)]
 static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
     std::sync::OnceLock::new();
@@ -1186,9 +1168,11 @@ impl ECStore {
         let oi = if require_all_pools {
             lookup_opts.metadata_cache_safe = false;
             lookup_opts.suppress_read_repair = true;
-            // Only completion uses this branch, after validating bucket and object.
-            let encoded_object = encode_dir_object_ref(object);
-            self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
+            if let Err(err) = check_object_args(bucket, object) {
+                return Some(err);
+            }
+            let encoded_object = encode_dir_object(object);
+            self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
                 .await
                 .and_then(|(info, _)| {
                     lookup_opts.precondition_check(&info)?;

crates/ecstore/src/store/object.rs

@@ -4814,7 +4814,44 @@ impl ECStore {
         data: &mut PutObjReader,
         opts: &ObjectOptions,
     ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
-        let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
+        let mut early_opts = opts.clone();
+        // Bucket publication precedes lifecycle and metadata transaction locks.
+        let prepared = if self.single_pool()
+            && !self.ctx.lock_manager().is_disabled()
+            && crate::set_disk::put_rename_early_ack_enabled()
+            && early_opts.write_completion == crate::object_api::WriteCompletion::Quorum
+            && !is_meta_bucketname(bucket)
+            && !early_opts.no_lock
+            && !early_opts.data_movement
+            && early_opts.write_lock_context.is_none()
+            && early_opts.object_lock_config_snapshot.is_none()
+            && early_opts.bucket_lifecycle_lock_fence.is_none()
+            && early_opts.namespace_lock_fence.is_none()
+            && early_opts.scanner_publication_commit_scope.is_none()
+            && early_opts.decommission_capacity_admission.is_none()
+            && early_opts.decommission_capacity.is_none()
+            && early_opts.quota_admission.is_none()
+            && early_opts
+                .put_object_cancellation
+                .as_ref()
+                .is_none_or(|token| !token.is_cancelled())
+            && rustfs_utils::path::object_name_disk_identity(object).as_ref()
+                != rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
+        {
+            check_new_object_args(bucket, object, false)?;
+            let incarnation = match early_opts.expected_bucket_incarnation_id {
+                Some(incarnation) => incarnation,
+                None => self.bucket_incarnation_id(bucket).await?,
+            };
+            early_opts.expected_bucket_incarnation_id = Some(incarnation);
+            let encoded = encode_dir_object(object);
+            let set = self.pools[0].get_disks_by_key(&encoded);
+            let disks = set.disks.read().await.clone();
+            crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true).await?
+        } else {
+            None
+        };
+        let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?;
         let idx = self
             .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
             .await?;
@@ -4823,7 +4860,8 @@ impl ECStore {
             opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
         }
         let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
-        let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
+        let result =
+            Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await;
         enqueue_recorded_tier_free_versions(self, receipt_sink).await;
         match result {
             Ok((object_info, old_current_size)) => {
@@ -9938,6 +9976,305 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() {
+        use crate::disk::ReadOptions;
+        use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
+
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+            let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+            let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
+            let set = store.pools[0].get_disks_by_key("object");
+            let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
+            {
+                let map = ctx.local_disk_map();
+                let mut active = map.write().await;
+                for disk in &disks {
+                    active.insert(disk.endpoint().to_string(), Some(disk.clone()));
+                }
+            }
+            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+            let bucket = "ecstore-reservation-order";
+            let object = "object";
+            store
+                .handle_make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("create bucket");
+            let tasks = rename_fanout_barrier::observe_tasks(object);
+            let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let first_store = Arc::clone(&store);
+            let first = tokio::spawn(async move {
+                first_store
+                    .handle_put_object(
+                        bucket,
+                        object,
+                        &mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
+                        &ObjectOptions::default(),
+                    )
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
+                .await
+                .expect("first tail pauses");
+            first.await.expect("first task").expect("first PUT");
+            let second = tokio::time::timeout(
+                Duration::from_secs(7),
+                store.handle_put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
+                    &ObjectOptions::default(),
+                ),
+            )
+            .await;
+            barrier.release();
+            drop(barrier);
+            let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while tasks.running() != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("rename tails drain");
+            let expected_etag = second.etag.as_ref().expect("second ETag");
+            for disk in &disks {
+                let info = disk
+                    .read_version("", bucket, object, "", &ReadOptions::default())
+                    .await
+                    .expect("read latest metadata");
+                assert_eq!(info.metadata.get("etag"), Some(expected_etag));
+                let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
+                assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
+            }
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_prepared_reservations_hold_bucket_publication_fence() {
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
+        let set = store.pools[0].get_disks_by_key("object");
+        let disks = set.disks.read().await.clone();
+        {
+            let map = ctx.local_disk_map();
+            let mut active = map.write().await;
+            for disk in disks.iter().flatten() {
+                active.insert(disk.endpoint().to_string(), Some(disk.clone()));
+            }
+        }
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "local-reservation-bucket-fence";
+        store
+            .handle_make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+        let reservations =
+            crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true)
+                .await
+                .expect("prepare reservations")
+                .expect("all disks prepared");
+        let lock = store
+            .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
+            .await
+            .expect("publication lock");
+        let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
+        assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err());
+        drop(reservations);
+        let guard = tokio::time::timeout(Duration::from_secs(5), writer)
+            .await
+            .expect("reservations release fence")
+            .expect("writer lock");
+        drop(guard);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn bucket_reservations_do_not_exhaust_shared_lock_holders() {
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
+        let set = store.pools[0].get_disks_by_key("object");
+        let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
+        {
+            let map = ctx.local_disk_map();
+            let mut active = map.write().await;
+            for disk in &disks {
+                active.insert(disk.endpoint().to_string(), Some(disk.clone()));
+            }
+        }
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "reservation-reader-capacity";
+        store
+            .handle_make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+        let disk = disks[0].endpoint().to_string();
+        let mut tokens = Vec::new();
+        let mut failure = None;
+        for index in 0..256 {
+            match store
+                .reserve_local_object_metadata_at_incarnation(
+                    &disk,
+                    bucket,
+                    &format!("object-{index}"),
+                    incarnation,
+                    Duration::from_secs(1),
+                )
+                .await
+            {
+                Ok(token) => tokens.push(token),
+                Err(error) => {
+                    failure = Some((index, error));
+                    break;
+                }
+            }
+        }
+        for token in tokens {
+            assert!(
+                store
+                    .cancel_local_object_metadata_reservation(&disk, token)
+                    .await
+                    .expect("cancel token")
+            );
+        }
+        assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() {
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
+        let set = store.pools[0].get_disks_by_key("object");
+        let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone();
+        ctx.local_disk_map()
+            .write()
+            .await
+            .insert(disk.endpoint().to_string(), Some(disk.clone()));
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "reservation-writer-fairness";
+        store
+            .handle_make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
+        let disk_ref = disk.endpoint().to_string();
+        let first = store
+            .reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5))
+            .await
+            .expect("first reservation");
+        let lock = store
+            .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
+            .await
+            .expect("publication lock");
+        let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
+        assert!(
+            tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(),
+            "writer waits for first token"
+        );
+        tokio::time::pause();
+        tokio::time::advance(Duration::from_secs(2)).await;
+        tokio::time::resume();
+        let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation(
+            &disk_ref,
+            bucket,
+            "second",
+            incarnation,
+            Duration::from_secs(5),
+        ));
+        assert!(
+            tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
+            "expired cached reader must not bypass writer"
+        );
+        assert!(
+            store
+                .cancel_local_object_metadata_reservation(&disk_ref, first)
+                .await
+                .expect("cancel first token")
+        );
+        let guard = tokio::time::timeout(Duration::from_secs(2), writer)
+            .await
+            .expect("writer proceeds")
+            .expect("writer lock");
+        assert!(
+            tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
+            "new reader waits while writer holds the lock"
+        );
+        drop(guard);
+        let token = tokio::time::timeout(Duration::from_secs(2), second)
+            .await
+            .expect("reader proceeds")
+            .expect("second reservation");
+        assert!(
+            store
+                .cancel_local_object_metadata_reservation(&disk_ref, token)
+                .await
+                .expect("cancel second token")
+        );
+    }
+
+    #[tokio::test]
+    async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() {
+        let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
+            rustfs_lock::FastObjectLockManager::new(),
+        )));
+        let client = Arc::new(RefreshLossLockClient::with_manager(manager));
+        let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
+            "reservation-loss".to_string(),
+            vec![client.clone() as Arc<dyn rustfs_lock::LockClient>],
+            1,
+        );
+        let acquire = |object: &'static str| {
+            rustfs_lock::LockRequest::new(
+                rustfs_lock::ObjectKey::new("bucket", object),
+                rustfs_lock::LockType::Shared,
+                "reservation-loss",
+            )
+            .with_ttl(Duration::from_secs(2))
+            .with_refresh_interval(Duration::from_millis(20))
+        };
+        let publication = lock
+            .acquire_guard(&acquire("publication"))
+            .await
+            .expect("publication request")
+            .expect("publication quorum");
+        let signal = publication.lock_lost_signal().expect("distributed loss signal");
+        let lifecycle = lock
+            .acquire_guard(&acquire("lifecycle"))
+            .await
+            .expect("lifecycle request")
+            .expect("lifecycle quorum");
+        let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
+        let lifecycle =
+            crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
+        let incarnation = Uuid::new_v4();
+        let slot = bucket_registry.reservation_slot("bucket", incarnation);
+        let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
+        let registry = crate::disk::os::MetadataReservationRegistry::new(1);
+        let key = std::path::Path::new("bucket/object");
+        let disk = Uuid::new_v4();
+        let token = registry
+            .prepare_with_owner(key, disk, Some(owner))
+            .expect("prepare reservation");
+        client.reject_refreshes();
+        tokio::time::timeout(Duration::from_secs(5), signal.notified())
+            .await
+            .expect("publication lock loss");
+        assert!(
+            registry.activate(token, key, disk).await.is_err(),
+            "a lost bucket fence must not enter the mutation queue"
+        );
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn borrowed_copy_tail_drained_waits_for_rename_tail() {

crates/ecstore/src/store/peer.rs

@@ -85,6 +85,171 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
 }
 
 impl ECStore {
+    /// Prepare an inert user-object reservation on the currently admitted disk.
+    /// An unclaimed token expires and does not hold the object's metadata lock.
+    #[cfg(test)]
+    pub async fn reserve_local_object_metadata(
+        &self,
+        disk_ref: &str,
+        bucket: &str,
+        object: &str,
+        timeout: std::time::Duration,
+    ) -> DiskResult<Uuid> {
+        self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None)
+            .await
+    }
+
+    /// Prepare before the caller acquires bucket lifecycle or object locks.
+    /// The token holds the bucket publication fence through physical completion.
+    pub async fn reserve_local_object_metadata_at_incarnation(
+        &self,
+        disk_ref: &str,
+        bucket: &str,
+        object: &str,
+        incarnation: Uuid,
+        timeout: std::time::Duration,
+    ) -> DiskResult<Uuid> {
+        if self.ctx.lock_manager().is_disabled()
+            || crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
+            || is_meta_bucketname(bucket)
+            || incarnation.is_nil()
+            || timeout.is_zero()
+            || rustfs_utils::path::object_name_disk_identity(object).as_ref()
+                == rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
+        {
+            return Err(DiskError::InvalidPath);
+        }
+        tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
+            let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
+            let mut cached = slot.owner.lock().await;
+            let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
+                Some(owner) => owner,
+                None => {
+                    // Bucket deletion takes publication before lifecycle and metadata.
+                    let lock = self
+                        .handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
+                        .await
+                        .map_err(|error| DiskError::other(error.to_string()))?;
+                    let publication = lock
+                        .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
+                        .await
+                        .map_err(|error| DiskError::other(error.to_string()))?;
+                    // Older peers may acquire lifecycle without publication.
+                    let lifecycle = self
+                        .acquire_bucket_incarnation_fence(bucket, incarnation)
+                        .await
+                        .map_err(|error| DiskError::other(error.to_string()))?;
+                    let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
+                    if !owner.is_reusable() {
+                        return Err(DiskError::DiskOngoingReq);
+                    }
+                    *cached = Arc::downgrade(&owner);
+                    owner
+                }
+            };
+            drop(cached);
+            self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner))
+                .await
+        })
+        .await
+        .map_err(|_| DiskError::Timeout)?
+    }
+
+    async fn reserve_local_object_metadata_with_owner(
+        &self,
+        disk_ref: &str,
+        bucket: &str,
+        object: &str,
+        timeout: std::time::Duration,
+        owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
+    ) -> DiskResult<Uuid> {
+        if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
+            return Err(DiskError::InvalidPath);
+        }
+        if timeout.is_zero() {
+            return Err(DiskError::Timeout);
+        }
+        let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
+        let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
+        let Disk::Local(wrapper) = disk.as_ref() else {
+            return Err(DiskError::DiskNotFound);
+        };
+        tokio::time::timeout(
+            timeout.min(std::time::Duration::from_secs(5)),
+            wrapper.prepare_object_metadata_with_owner(bucket, object, owner),
+        )
+        .await
+        .map_err(|_| DiskError::Timeout)?
+    }
+
+    /// Atomically enqueue a prepared token on its original disk and object.
+    pub async fn activate_local_object_metadata_reservation(
+        &self,
+        disk_ref: &str,
+        bucket: &str,
+        object: &str,
+        token: Uuid,
+    ) -> DiskResult<()> {
+        if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
+            return Err(DiskError::InvalidPath);
+        }
+        let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
+        let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
+        let Disk::Local(wrapper) = disk.as_ref() else {
+            return Err(DiskError::DiskNotFound);
+        };
+        wrapper.activate_object_metadata(bucket, object, token).await
+    }
+
+    /// Cancel an unclaimed token on the currently admitted disk.
+    /// A token already claimed by a physical mutation cannot be cancelled.
+    pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
+        let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
+        let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
+        let Disk::Local(wrapper) = disk.as_ref() else {
+            return Err(DiskError::DiskNotFound);
+        };
+        Ok(wrapper.cancel_object_metadata_reservation(token))
+    }
+
+    /// Publish an ordinary staged user-object write using a one-shot disk reservation.
+    /// The token is bound to the admitted disk instance and destination. A claimed
+    /// physical mutation continues to own its lease if the RPC waiter is cancelled.
+    /// The caller must retain the bucket lifecycle fence until the physical tail
+    /// finishes; the reservation does not identify a bucket incarnation.
+    pub async fn rename_local_data_reserved(
+        &self,
+        disk_ref: &str,
+        source: (&str, &str),
+        fi: &FileInfo,
+        destination: (&str, &str),
+        token: Uuid,
+    ) -> DiskResult<RenameDataResp> {
+        if token.is_nil()
+            || source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
+            || crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
+            || is_meta_bucketname(destination.0)
+            || rustfs_utils::http::metadata_compat::contains_key_str(
+                &fi.metadata,
+                crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
+            )
+        {
+            return Err(DiskError::InvalidPath);
+        }
+        rename_local_data_with_ctx(
+            &self.ctx,
+            disk_ref,
+            source,
+            fi,
+            destination,
+            RenameDataGuards {
+                metadata_reservation_token: Some(token),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
     /// Resolve a quota lease against this request's captured storage instance.
     pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
         let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
@@ -270,6 +435,7 @@ impl ECStore {
                 scanner_publication_lease_token: scanner_token,
                 external_guard,
                 namespace_owner: None,
+                ..Default::default()
             },
         )
         .await
@@ -351,7 +517,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
     Ok(())
 }
 
-async fn admit_local_disk(
+pub(crate) async fn admit_local_disk(
     ctx: &Arc<InstanceContext>,
     disk: &DiskStore,
     disk_id: Option<Uuid>,
@@ -783,6 +949,116 @@ mod tests {
             .expect("seeded metadata bytes")
     }
 
+    #[tokio::test]
+    async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
+        let ctx = Arc::new(InstanceContext::new());
+        let dir = tempfile::tempdir().expect("disk root");
+        let id = Uuid::new_v4();
+        let disk = target_disk(&ctx, dir.path(), id).await;
+        let disk_ref = disk.endpoint().to_string();
+        let store = super::super::tests::build_store_with_ctx(ctx.clone());
+        let version = Uuid::new_v4();
+        let fi = target_file_info("object", version, b"reserved");
+        seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
+        tokio::fs::create_dir_all(dir.path().join("target-bucket"))
+            .await
+            .expect("user bucket");
+        assert!(matches!(
+            store
+                .reserve_local_object_metadata(
+                    &disk_ref,
+                    "target-bucket/../.rustfs.sys",
+                    "object",
+                    std::time::Duration::from_secs(5),
+                )
+                .await,
+            Err(DiskError::InvalidPath)
+        ));
+        assert!(matches!(
+            store
+                .reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
+                .await,
+            Err(DiskError::InvalidPath)
+        ));
+        let token = store
+            .reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
+            .await
+            .expect("reserve current disk");
+        assert!(matches!(
+            store
+                .rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
+                .await,
+            Err(DiskError::InvalidPath)
+        ));
+        store
+            .activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
+            .await
+            .expect("activate token");
+        store
+            .rename_local_data_reserved(
+                &disk_ref,
+                (crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
+                &fi,
+                ("target-bucket", "object"),
+                token,
+            )
+            .await
+            .expect("reserved publication");
+        assert!(
+            !store
+                .cancel_local_object_metadata_reservation(&disk_ref, token)
+                .await
+                .expect("consumed token")
+        );
+        let read = disk
+            .read_version(
+                "target-bucket",
+                "target-bucket",
+                "object",
+                &version.to_string(),
+                &crate::disk::ReadOptions {
+                    read_data: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("read reserved publication");
+        assert_eq!(read.data, fi.data);
+
+        let stale = store
+            .reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
+            .await
+            .expect("reserve before disk replacement");
+        let replacement = target_disk(&ctx, dir.path(), id).await;
+        assert!(matches!(
+            store
+                .rename_local_data_reserved(
+                    &disk_ref,
+                    (crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
+                    &fi,
+                    ("target-bucket", "object"),
+                    stale,
+                )
+                .await,
+            Err(DiskError::DiskOngoingReq)
+        ));
+        let after = replacement
+            .read_version(
+                "target-bucket",
+                "target-bucket",
+                "object",
+                &version.to_string(),
+                &crate::disk::ReadOptions {
+                    read_data: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("replacement retains publication");
+        assert_eq!(after.data, fi.data);
+        assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
+    }
+
     #[tokio::test]
     async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
         for warm in [false, true] {

crates/filemeta/src/metacache.rs

@@ -925,13 +925,6 @@ impl MetaCacheEntriesSorted {
 
 const METACACHE_STREAM_VERSION: u8 = 2;
 const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
-/// Upper bound on one incremental reservation while accumulating a single
-/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
-/// up to this bound so a legitimately large xl.meta does not pay one small
-/// read per chunk, while a corrupt length prefix still cannot force a
-/// reservation larger than the payload that has already arrived (see
-/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
-const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
 /// msgpack `false`: the entry-stream terminator, and the only byte a legacy
 /// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
 const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1164,14 +1157,13 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         // A whole xl.meta can legitimately exceed the limit on its individual
         // msgpack values. Grow only as payload arrives, so a corrupt length
         // prefix cannot cause a speculative allocation of the advertised size.
-        let mut chunk = METACACHE_READ_CHUNK_SIZE;
         while self.buf.len() < read_size {
             let start = self.buf.len();
             let extra = if lookahead {
                 let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
-                (read_size - start).clamp(min_chunk, chunk)
+                (read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
             } else {
-                (read_size - start).min(chunk)
+                (read_size - start).min(METACACHE_READ_CHUNK_SIZE)
             };
             if let Err(e) = self.buf.try_reserve(extra) {
                 let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1202,7 +1194,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
                     return Err(err);
                 }
             }
-            chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
         }
 
         self.pos = read_size;
@@ -2164,76 +2155,6 @@ mod tests {
         }
     }
 
-    /// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
-    /// decode completely even when the aggregate stream exceeds the per-value
-    /// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
-    /// element.
-    #[tokio::test]
-    async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
-        // ~2.9 KiB of metadata per entry matches the per-entry cost observed in
-        // the #9120 production report; 6,000 entries push the stream past 16 MiB.
-        const ENTRIES: usize = 6_000;
-        let metadata = vec![0xab; 2_880];
-        let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
-            .map(|i| MetaCacheEntry {
-                name: format!("bigdir/object-{i:05}"),
-                metadata: metadata.clone(),
-                ..Default::default()
-            })
-            .collect();
-        let mut output = Cursor::new(Vec::new());
-        let mut writer = MetacacheWriter::new(&mut output);
-        writer.write(&entries).await.expect("wide listing must encode");
-        writer.close().await.expect("stream must close");
-        let bytes = output.into_inner();
-        assert!(
-            bytes.len() > 16 << 20,
-            "fixture must exceed the per-value limit, got {} bytes",
-            bytes.len()
-        );
-        let mut reader = MetacacheReader::new(Cursor::new(bytes));
-        let decoded = reader.read_all().await.expect("wide listing must decode");
-        assert_eq!(decoded, entries);
-    }
-
-    /// The #9120 failure shape seen in production: one entry whose metadata is
-    /// an xl.meta that grew past 16 MiB with its version history, listed among
-    /// ordinary neighbors and skipped past without losing the framing.
-    #[tokio::test]
-    async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
-        let small = |i: u8| MetaCacheEntry {
-            name: format!("bigdir/object-{i}"),
-            metadata: vec![i; 64],
-            ..Default::default()
-        };
-        let oversized = MetaCacheEntry {
-            name: "bigdir/huge-versions".to_string(),
-            metadata: vec![0xcd; (16 << 20) + 12_345],
-            ..Default::default()
-        };
-        let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
-        let mut output = Cursor::new(Vec::new());
-        let mut writer = MetacacheWriter::new(&mut output);
-        writer.write(&entries).await.expect("mixed listing must encode");
-        writer.close().await.expect("stream must close");
-
-        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
-        assert_eq!(
-            reader.peek().await.expect("first entry").expect("present"),
-            entries[0],
-            "an ordinary entry before the oversized one must decode"
-        );
-        reader.skip(1).await.expect("skip the first entry");
-        assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
-        assert_eq!(
-            reader.peek().await.expect("oversized entry").expect("present"),
-            oversized,
-            "an entry past the per-value limit must round-trip whole"
-        );
-        reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
-        assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
-    }
-
     #[test]
     fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
         let version_id = Uuid::new_v4();
@@ -3854,74 +3775,6 @@ mod tests {
         assert!(corrupt_crc_entry.is_latest_delete_marker());
     }
 
-    #[test]
-    fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
-        let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
-            .expect("load realistic xl.meta");
-        let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
-        let mut free_meta_sys = HashMap::new();
-        rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
-        let free = FileMetaShallowVersion::try_from(FileMetaVersion {
-            version_type: VersionType::Delete,
-            delete_marker: Some(MetaDeleteMarker {
-                version_id: Some(Uuid::from_u128(42)),
-                mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
-                meta_sys: free_meta_sys,
-            }),
-            write_version: 1,
-            ..Default::default()
-        })
-        .expect("encode cleanup record");
-        fm.versions.insert(0, free);
-        let entry = MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: fm.marshal_msg().expect("encode metadata"),
-            cached: None,
-            reusable: false,
-        };
-        let scanned = fm
-            .get_file_info_versions("bucket", "object", false)
-            .expect("scanner version decode");
-        assert_eq!(scanned.free_versions.len(), 1);
-        assert_eq!(scanned.versions.len(), 1);
-        assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
-        assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
-        let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
-        assert_eq!(listed.version_id, object.version_id);
-        assert_eq!(listed.version_id, scanned.versions[0].version_id);
-        assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
-        assert_eq!(listed.size, scanned.versions[0].size);
-        assert!(!listed.deleted);
-
-        let marker_id = Uuid::from_u128(43);
-        fm.add_version(FileInfo {
-            volume: "bucket".to_string(),
-            name: "object".to_string(),
-            version_id: Some(marker_id),
-            deleted: true,
-            mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
-            ..Default::default()
-        })
-        .expect("add current delete marker");
-        let mut marker_entry = MetaCacheEntry {
-            name: "object".to_string(),
-            metadata: fm.marshal_msg().expect("encode marker metadata"),
-            cached: None,
-            reusable: false,
-        };
-        let scanned = fm
-            .get_file_info_versions("bucket", "object", false)
-            .expect("scan marker stack");
-        assert_eq!(scanned.versions.len(), 2);
-        assert_eq!(scanned.versions[0].version_id, Some(marker_id));
-        assert!(scanned.versions[0].deleted);
-        assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
-        assert!(marker_entry.is_latest_delete_marker());
-        let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
-        assert_eq!(listed.version_id, scanned.versions[0].version_id);
-        assert!(listed.deleted);
-    }
-
     #[test]
     fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
         let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))

crates/heal/src/error.rs

@@ -128,20 +128,6 @@ impl Error {
         Error::TransientSkip { message: message.into() }
     }
 
-    /// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
-    /// Callers must cut their own concurrency instead of retrying at full speed.
-    pub(crate) fn is_slow_down(&self) -> bool {
-        match self {
-            Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
-            Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
-            Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
-                message.contains("Please reduce your request rate")
-            }
-            Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
-            _ => false,
-        }
-    }
-
     /// Whether a heal operation can be retried without changing its inputs.
     pub(crate) fn is_recoverable_heal(&self) -> bool {
         match classify_heal_error(self) {
@@ -463,22 +449,6 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
-    #[test]
-    fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
-        assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
-        assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
-        assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
-        assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
-        assert!(
-            Error::TaskExecutionFailed {
-                message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
-            }
-            .is_slow_down()
-        );
-        assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
-        assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
-    }
-
     #[test]
     fn pool_metadata_failures_keep_their_typed_recovery_classification() {
         use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};

crates/heal/src/heal/erasure_healer.rs

@@ -14,7 +14,6 @@
 
 use crate::error::{HealErrorClass, classify_heal_error};
 use crate::heal::{
-    pacing::HealPageRate,
     progress::{HealProgress, add_bytes, increment_counter},
     resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -129,12 +128,12 @@ pub struct ErasureSetHealer {
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
     replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
     work_parent: Option<super::manager::HealWork>,
-    mainline_pacer: Option<super::pacing::ForegroundPacing>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
 }
 
 async fn acquire_page_permit(
     semaphore: Arc<Semaphore>,
-    rate: &HealPageRate,
+    pacer: Option<&super::pacing::MainlinePacer>,
     cancel: &tokio_util::sync::CancellationToken,
 ) -> Result<tokio::sync::OwnedSemaphorePermit> {
     let acquire = || async {
@@ -147,16 +146,18 @@ async fn acquire_page_permit(
     let mut paid_pause = false;
     loop {
         let permit = acquire().await?;
-        // Keep the real permit on the low-pressure path. Every acquisition
-        // gets a fresh decision, including a waiter that queued a second
-        // time. One completed pause is a bounded minimum-progress grant.
-        match rate.admission_decision() {
-            super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
-                drop(permit);
-                paid_pause = rate.wait_after_admission(cancel, pressure).await?;
-                continue;
+        if let Some(pacer) = pacer {
+            // Keep the real permit on the low-pressure path. Every acquisition
+            // gets a fresh decision, including a waiter that queued a second
+            // time. One completed pause is a bounded minimum-progress grant.
+            match pacer.admission_decision() {
+                super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
+                    drop(permit);
+                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
+                    continue;
+                }
+                _ => {}
             }
-            _ => {}
         }
         return Ok(permit);
     }
@@ -165,7 +166,7 @@ async fn acquire_page_permit(
 #[cfg(test)]
 mod mainline_pacing_tests {
     use super::*;
-    use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
+    use crate::heal::pacing::{MainlinePacer, TestPressure};
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
     use tokio_util::sync::CancellationToken;
@@ -179,15 +180,15 @@ mod mainline_pacing_tests {
             .await
             .expect("existing object owns capacity");
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
-        let pacer =
-            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
-        let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
+        let pacer = Arc::new(
+            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
+        );
         let cancel = CancellationToken::new();
         let waiting = tokio::spawn({
             let semaphore = semaphore.clone();
-            let rate = rate.clone();
+            let pacer = pacer.clone();
             let cancel = cancel.clone();
-            async move { acquire_page_permit(semaphore, &rate, &cancel).await }
+            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
         });
         tokio::task::yield_now().await;
         provider.active.store(100, Ordering::SeqCst);
@@ -199,13 +200,12 @@ mod mainline_pacing_tests {
         assert_eq!(semaphore.available_permits(), 1);
         let deadline = tokio::time::timeout(
             Duration::from_millis(10),
-            acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
+            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
         )
         .await;
         assert!(deadline.is_err());
         assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
-        let unpaced = HealPageRate::new(1, None);
-        let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
+        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
             .await
             .expect("unpaced admission");
         assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,11 +221,10 @@ mod mainline_pacing_tests {
             let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
             let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
             let pause = Duration::from_millis(250);
-            let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
-            let rate = HealPageRate::new(1, Some(pacer));
+            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
             let cancel = CancellationToken::new();
-            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
-            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
             assert!(futures::poll!(first.as_mut()).is_pending());
             assert!(futures::poll!(second.as_mut()).is_pending());
             drop(occupied);
@@ -330,20 +329,6 @@ impl ErasureSetHealer {
         }
     }
 
-    /// Page fan-out for one whole erasure-set pass (stable across buckets).
-    fn pass_page_concurrency_limit(&self) -> usize {
-        if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
-            // Versions of the same object must not race without its namespace lock.
-            1
-        } else {
-            Self::effective_heal_page_object_concurrency_for_source(
-                self.source,
-                self.heal_opts.scan_mode,
-                self.replacement_task_id.is_some(),
-            )
-        }
-    }
-
     /// Map the shared heal error class onto this healer's per-object actions.
     ///
     /// `classify_heal_error` is the only classifier; this adapter decides what
@@ -388,7 +373,7 @@ impl ErasureSetHealer {
         }
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
         self.mainline_pacer = pacer;
         self
     }
@@ -479,7 +464,12 @@ impl ErasureSetHealer {
             None
         };
         let buckets = admin_buckets.as_deref().unwrap_or(buckets);
-        if checkpoint_manager.admin_completed().await {
+        if checkpoint_manager
+            .get_checkpoint()
+            .await
+            .admin
+            .is_some_and(|admin| admin.completed)
+        {
             self.initialize_progress(buckets, &resume_manager.get_state().await).await;
             return Ok(());
         }
@@ -500,7 +490,14 @@ impl ErasureSetHealer {
         }
 
         if self.admin_task.is_some() {
-            checkpoint_manager.complete_admin().await?;
+            let mut admin = checkpoint_manager
+                .get_checkpoint()
+                .await
+                .admin
+                .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
+            admin.completed = true;
+            admin.outcome.finish(None);
+            checkpoint_manager.set_admin(admin).await?;
         } else {
             checkpoint_manager.cleanup().await?;
             resume_manager.cleanup().await?;
@@ -721,10 +718,7 @@ impl ErasureSetHealer {
             }
             return Err(Error::DanglingDeleteDeferred { retry_not_before });
         }
-        // One rate controller for the whole pass: SlowDown cool-down and page
-        // fan-out persist across buckets instead of resetting at each one.
-        let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
-        let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
+        let checkpoint = checkpoint_manager.get_checkpoint().await;
 
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -732,17 +726,14 @@ impl ErasureSetHealer {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
             set_disk_id,
-            current_bucket_index,
-            current_object_index,
+            current_bucket_index = checkpoint.current_bucket_index,
+            current_object_index = checkpoint.current_object_index,
             state = "resuming",
             "Erasure set resumed"
         );
 
         // 2. initialize progress
         self.initialize_progress(buckets, &state).await;
-        // Snapshot before any baseline write: set_progress_baseline overwrites
-        // the saved generation, which would hide a real generation mismatch.
-        let checkpoint = checkpoint_manager.progress_snapshot().await;
         let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
             let baseline = self.progress.read().await;
             (
@@ -769,7 +760,17 @@ impl ErasureSetHealer {
         let mut successful_objects = state.successful_objects;
         let mut failed_objects = state.failed_objects;
         let mut skipped_objects = state.skipped_objects;
-        let checkpoint_has_progress = checkpoint.has_progress();
+        let checkpoint_has_progress = checkpoint.baseline_known
+            || checkpoint.successful_objects > 0
+            || checkpoint.failed_object_count > 0
+            || checkpoint.skipped_object_count > 0
+            || checkpoint.skipped_new_versions > 0
+            || checkpoint.skipped_ilm_expired > 0
+            || checkpoint.processed_bytes > 0
+            || checkpoint.total_objects > 0
+            || checkpoint.total_bytes > 0
+            || checkpoint.baseline_generation.is_some()
+            || checkpoint.counter_unknown;
         let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
         let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
         if checkpoint_has_progress {
@@ -853,7 +854,6 @@ impl ErasureSetHealer {
                     resume_manager,
                     checkpoint_manager,
                     state.start_time,
-                    &page_rate,
                 )
                 .await;
 
@@ -1020,14 +1020,14 @@ impl ErasureSetHealer {
         if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
             if self.replacement_task_id.is_some() {
                 let state = resume_manager.get_state().await;
-                let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
+                let checkpoint = checkpoint_manager.get_checkpoint().await;
                 if state.can_retry()
                     && failed_objects == 0
                     && failed_buckets == 0
                     && skipped_objects > 0
-                    && skipped_objects == grace_objects
-                    && !counter_unknown
-                    && let Some(retry_not_before) = retry_not_before
+                    && skipped_objects == checkpoint.dangling_delete_grace_objects
+                    && !checkpoint.counter_unknown
+                    && let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
                 {
                     // Publish the generation's rewind and wait first. If the
                     // checkpoint reset is interrupted, initialization reapplies
@@ -1167,9 +1167,14 @@ impl ErasureSetHealer {
 
         let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
         let checkpoint_key = compose_key(&object_key, None);
-        if checkpoint_manager.is_object_settled(&checkpoint_key).await {
+        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        if checkpoint.processed_objects.contains(&checkpoint_key)
+            || checkpoint.failed_objects.contains(&checkpoint_key)
+            || checkpoint.skipped_objects.contains(&checkpoint_key)
+        {
             return Ok(());
         }
+        drop(checkpoint);
 
         self.verify_replacement_identity_fence("pool metadata").await?;
         resume_manager
@@ -1423,7 +1428,7 @@ impl ErasureSetHealer {
 
     /// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
     #[allow(clippy::too_many_arguments)]
-    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
+    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
     async fn heal_bucket_with_resume(
         &self,
         bucket: &str,
@@ -1437,7 +1442,6 @@ impl ErasureSetHealer {
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
         started_at_secs: u64,
-        page_rate: &Arc<HealPageRate>,
     ) -> Result<bool> {
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -1487,6 +1491,17 @@ impl ErasureSetHealer {
         // detects a non-advancing cursor even though next_marker embeds a
         // transient cache id that changes between identical pages.
         let mut previous_page_last: Option<(String, Option<String>)> = None;
+        let page_concurrency_limit =
+            if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
+                // Versions of the same object must not race without its namespace lock.
+                1
+            } else {
+                Self::effective_heal_page_object_concurrency_for_source(
+                    self.source,
+                    self.heal_opts.scan_mode,
+                    self.replacement_task_id.is_some(),
+                )
+            };
         let in_flight = Arc::new(AtomicUsize::new(0));
         // Per-bucket sample caps for per-object warn! lines: a flapping rebuild
         // disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1508,7 +1523,9 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
-            page_rate.wait_before_page(&self.cancel_token).await?;
+            if let Some(pacer) = &self.mainline_pacer {
+                pacer.wait(&self.cancel_token).await?;
+            }
             self.verify_replacement_identity_fence("page scan").await?;
             if let Some(expected) = incarnation {
                 self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1530,8 +1547,9 @@ impl ErasureSetHealer {
                     .await?
             };
             let page_is_empty = objects.is_empty();
+            let checkpoint = checkpoint_manager.get_checkpoint().await;
             let page_resume_index = *current_object_index;
-            let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
+            let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
             let mut page_tasks = FuturesUnordered::new();
             let mut completed_in_page = 0usize;
 
@@ -1545,7 +1563,10 @@ impl ErasureSetHealer {
 
                 // Per-version dedup identity — the single canonical key.
                 let key = compose_key(&item.name, item.version_id.as_deref());
-                if checkpoint_manager.is_object_settled(&key).await {
+                if checkpoint.processed_objects.contains(&key)
+                    || checkpoint.failed_objects.contains(&key)
+                    || checkpoint.skipped_objects.contains(&key)
+                {
                     continue;
                 }
 
@@ -1703,7 +1724,7 @@ impl ErasureSetHealer {
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
                 let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
-                let page_rate = page_rate.clone();
+                let mainline_pacer = self.mainline_pacer.clone();
 
                 let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
                 let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1715,15 +1736,14 @@ impl ErasureSetHealer {
                     .map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
                 let work = async move {
                     let mut retries = 0;
-                    let mut saw_slow_down = false;
                     let mut admin_identity = admin_identity;
                     let mut admin_task_id = admin_task_id;
                     loop {
-                        let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
+                        let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
 
                         let permit = match permit {
                             Ok(permit) => permit,
-                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
+                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
                         };
 
                         let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1731,8 +1751,7 @@ impl ErasureSetHealer {
                         if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
                             let (result, canonical) =
                                 admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
-                            let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
-                            return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
+                            return (dedup_key, object_name, version_id, result, canonical);
                         }
 
                         // Always go through heal_object. Genuine absence flows through
@@ -1834,23 +1853,18 @@ impl ErasureSetHealer {
 
                         drop(in_flight_guard);
                         drop(permit);
-                        if matches!(&result.1, Err(err) if err.is_slow_down()) {
-                            saw_slow_down = true;
-                        }
                         if retry_replacement
                             && retries < REPLACEMENT_OBJECT_MAX_RETRIES
                             && matches!(&result.1, Err(Error::TransientSkip { .. }))
                         {
                             retries += 1;
                             tokio::select! {
-                                _ = cancel_token.cancelled() => {
-                                    return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
-                                }
+                                _ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
                                 _ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
                             }
                             continue;
                         }
-                        return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
+                        return (dedup_key, object_name, version_id, result, None);
                     }
                 };
                 page_tasks.push(async move {
@@ -1861,7 +1875,7 @@ impl ErasureSetHealer {
                         };
                         match result {
                             Ok(result) => result,
-                            Err(error) => (key, object, version, (0, Err(error), false), None),
+                            Err(error) => (key, object, version, (0, Err(error)), None),
                         }
                     } else {
                         work.await
@@ -1870,16 +1884,7 @@ impl ErasureSetHealer {
             }
 
             while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
-                let (object_size, result, saw_slow_down) = result;
-                // Exactly one rate observation per finished object, including
-                // SlowDown that later succeeded on a replacement retry.
-                page_rate.observe_object_outcome(
-                    &match &result {
-                        Ok(_) => Ok(true),
-                        Err(err) => Err(err),
-                    },
-                    saw_slow_down,
-                );
+                let (object_size, result) = result;
                 let mut telemetry_unknown = false;
                 let checkpoint_outcome = match result {
                     Ok(true) => {
@@ -2477,7 +2482,6 @@ mod resume_loop_tests {
     use super::{
         ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
     };
-    use crate::heal::pacing::HealPageRate;
     use crate::heal::progress::HealProgress;
     use crate::heal::resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -3059,7 +3063,6 @@ mod resume_loop_tests {
         let mut successful = 0u64;
         let mut failed = 0u64;
         let mut skipped = 0u64;
-        let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
         let result = env
             .healer
             .heal_bucket_with_resume(
@@ -3074,7 +3077,6 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 state.start_time,
-                &page_rate,
             )
             .await;
         (processed, successful, failed, skipped, result.map(|_| ()))
@@ -3142,7 +3144,6 @@ mod resume_loop_tests {
         let mut skipped = 0;
         let started_at = env.resume.get_state().await.start_time;
 
-        let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
         let error = healer
             .heal_bucket_with_resume(
                 "b",
@@ -3156,7 +3157,6 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 started_at,
-                &page_rate,
             )
             .await
             .expect_err("a remounted target must not begin a new page scan");

crates/heal/src/heal/manager.rs

@@ -810,14 +810,7 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
 fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
     active_heals
         .iter()
-        .find(|(_, task)| {
-            PriorityHealQueue::make_dedup_key_for_owner(
-                &task.heal_type,
-                &task.options,
-                task.source,
-                task.expected_mrf_bucket_incarnation_id,
-            ) == key
-        })
+        .find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
         .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
 }
 

crates/heal/src/heal/manager/queue.rs

@@ -611,104 +611,7 @@ impl PriorityHealQueue {
 
     /// Create a deduplication key from a heal request
     pub(super) fn make_dedup_key(request: &HealRequest) -> String {
-        Self::make_dedup_key_for_owner(
-            &request.heal_type,
-            &request.options,
-            request.source,
-            request.expected_mrf_bucket_incarnation_id,
-        )
-    }
-
-    /// Bound MRF work can only merge with work for the same bucket lifetime.
-    /// Length-prefix user-controlled components so a key cannot forge another
-    /// version, scope, or incarnation through delimiter characters.
-    pub(super) fn make_dedup_key_for_owner(
-        heal_type: &HealType,
-        options: &HealOptions,
-        source: HealRequestSource,
-        incarnation: Option<Uuid>,
-    ) -> String {
-        if source == HealRequestSource::Mrf
-            && let Some(incarnation) = incarnation
-        {
-            use std::fmt::Write as _;
-
-            let (pool, set) = heal_scope_indices(heal_type, options);
-            let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
-            let mut part = |value: &str| {
-                // Formatting into a String is infallible.
-                let _ = write!(key, "{}:{value}", value.len());
-            };
-            match heal_type {
-                HealType::Object {
-                    bucket,
-                    object,
-                    version_id,
-                } => {
-                    part("object");
-                    part(bucket);
-                    part(object);
-                    match version_id {
-                        None => part("none"),
-                        Some(version_id) => {
-                            part("some");
-                            part(version_id);
-                        }
-                    }
-                }
-                HealType::Metadata { bucket, object } => {
-                    part("metadata");
-                    part(bucket);
-                    part(object);
-                }
-                HealType::ECDecode {
-                    bucket,
-                    object,
-                    version_id,
-                } => {
-                    part("ecdecode");
-                    part(bucket);
-                    part(object);
-                    match version_id {
-                        None => part("none"),
-                        Some(version_id) => {
-                            part("some");
-                            part(version_id);
-                        }
-                    }
-                }
-                HealType::DeleteMarkerPurge {
-                    bucket,
-                    object,
-                    version_id,
-                    purge,
-                } => {
-                    part("delete-marker-purge");
-                    part(bucket);
-                    part(object);
-                    part(version_id);
-                    part(&purge.bucket_incarnation_id.to_string());
-                    part(&purge.marker_incarnation_id.to_string());
-                    part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
-                }
-                HealType::Bucket { bucket } => {
-                    part("bucket");
-                    part(bucket);
-                }
-                HealType::Prefix { bucket, prefix } => {
-                    part("prefix");
-                    part(bucket);
-                    part(prefix);
-                }
-                // Cluster and erasure-set work are not bucket-bound; preserve
-                // their existing keys, including contains_erasure_set lookups.
-                HealType::Cluster | HealType::ErasureSet { .. } => {
-                    return Self::make_dedup_key_for_scope(heal_type, options);
-                }
-            }
-            return key;
-        }
-        Self::make_dedup_key_for_scope(heal_type, options)
+        Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
     }
 
     pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {

crates/heal/src/heal/manager/scheduler.rs

@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
     request: &HealRequest,
     config: &HealConfig,
     provider: &Option<WorkloadSnapshotProviderRef>,
-) -> Option<crate::heal::pacing::ForegroundPacing> {
+) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
     let pace_running = request.source == HealRequestSource::Admin
         || (request.source == HealRequestSource::AutoHeal
             && matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,13 +43,14 @@ pub(super) fn running_mainline_pacer(
         return None;
     }
     provider.as_ref().and_then(|provider| {
-        crate::heal::pacing::ForegroundPacing::new(
+        crate::heal::pacing::MainlinePacer::new(
             provider.clone(),
             config.mainline_read_utilization_high_percent,
             config.mainline_write_utilization_high_percent,
             config.mainline_max_sleep,
             request.source,
         )
+        .map(Arc::new)
     })
 }
 

crates/heal/src/heal/manager/tests.rs

@@ -117,453 +117,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
 
 struct MockStorage;
 
-fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
-    let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
-    request.source = HealRequestSource::Mrf;
-    request.expected_mrf_bucket_incarnation_id = Some(incarnation);
-    request.options.pool_index = Some(0);
-    request.options.set_index = Some(0);
-    request
-}
-
-async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
-    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-    let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
-    let old_incarnation = Uuid::new_v4();
-    let new_incarnation = Uuid::new_v4();
-    let old_request = incarnation_bound_mrf_fallback(old_incarnation);
-    let old_id = old_request.id.clone();
-    match existing_state {
-        "queued" => {
-            let receipt = manager
-                .submit_heal_request_with_receipt(old_request)
-                .await
-                .expect("old fallback must be admitted");
-            assert_eq!(receipt.result, HealAdmissionResult::Accepted);
-        }
-        "active" => {
-            manager
-                .active_heals
-                .lock()
-                .await
-                .insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
-        }
-        "retrying" => {
-            manager.retrying_heals.lock().await.insert(
-                old_id.clone(),
-                RetryingHeal {
-                    request: old_request,
-                    error: "retry pending".to_owned(),
-                    cancel_token: CancellationToken::new(),
-                },
-            );
-        }
-        other => panic!("unknown fixture state: {other}"),
-    }
-
-    let new_request = incarnation_bound_mrf_fallback(new_incarnation);
-    let new_id = new_request.id.clone();
-    assert_ne!(new_id, old_id, "the fallback must use a new request ID");
-    let receipt = manager
-        .submit_heal_request_with_receipt(new_request)
-        .await
-        .expect("new incarnation admission must be decided");
-    assert_eq!(
-        receipt.result,
-        HealAdmissionResult::Accepted,
-        "a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
-    );
-    assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
-    let queue = manager.heal_queue.lock().await;
-    assert!(
-        queue
-            .requests()
-            .any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
-    );
-    if existing_state == "queued" {
-        assert!(
-            queue
-                .requests()
-                .any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
-        );
-    }
-}
-
-#[tokio::test]
-async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
-    assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
-}
-
-#[tokio::test]
-async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
-    assert_recreated_bucket_fallback_has_its_own_owner("active").await;
-}
-
-#[tokio::test]
-async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
-    assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
-}
-
-fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
-    let mut request = incarnation_bound_mrf_fallback(incarnation);
-    if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
-        *id = version_id.map(str::to_owned);
-    } else {
-        unreachable!("fallback fixture must be an object request");
-    }
-    request
-}
-
-#[tokio::test]
-async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
-    for version in [Uuid::nil(), Uuid::new_v4()] {
-        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
-        let incarnation = Uuid::new_v4();
-        let version_id = version.to_string();
-        let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
-        let original_id = original.id.clone();
-        let accepted = manager
-            .submit_heal_request_with_receipt(original)
-            .await
-            .expect("first queued fallback");
-        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
-        let merged = manager
-            .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
-            .await
-            .expect("duplicate queued fallback");
-        assert_eq!(merged.result, HealAdmissionResult::Merged);
-        assert_eq!(merged.task_id, original_id);
-        assert_eq!(manager.heal_queue.lock().await.len(), 1);
-    }
-}
-
-#[tokio::test]
-async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
-    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
-    let incarnation = Uuid::new_v4();
-    let original = incarnation_bound_mrf_fallback(incarnation);
-    let original_id = original.id.clone();
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(original)
-            .await
-            .expect("first fallback")
-            .result,
-        HealAdmissionResult::Accepted
-    );
-    let same = manager
-        .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
-        .await
-        .expect("same-incarnation fallback");
-    assert_eq!(same.result, HealAdmissionResult::Merged);
-    assert_eq!(same.task_id, original_id);
-
-    let mut versioned = incarnation_bound_mrf_fallback(incarnation);
-    versioned.heal_type = HealType::Object {
-        bucket: "mrf-recreated-bucket".to_owned(),
-        object: "object".to_owned(),
-        version_id: Some(Uuid::new_v4().to_string()),
-    };
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(versioned)
-            .await
-            .expect("different version")
-            .result,
-        HealAdmissionResult::Accepted
-    );
-    let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
-    other_scope.options.pool_index = Some(1);
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(other_scope)
-            .await
-            .expect("different pool")
-            .result,
-        HealAdmissionResult::Accepted
-    );
-    let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
-    same_id_other_incarnation.id = original_id;
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(same_id_other_incarnation)
-            .await
-            .expect("same-ID conflict must be decided")
-            .result,
-        HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
-    );
-}
-
-#[test]
-fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
-    let incarnation = Uuid::new_v4();
-    let bound = incarnation_bound_mrf_fallback(incarnation);
-    let mut unbound = bound.clone();
-    unbound.expected_mrf_bucket_incarnation_id = None;
-    let mut non_mrf = bound.clone();
-    non_mrf.source = HealRequestSource::ReadRepair;
-    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
-    assert_eq!(
-        PriorityHealQueue::make_dedup_key(&non_mrf),
-        PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
-        "an unrelated source retains its previous scope key even if the MRF-only field is set"
-    );
-    assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
-
-    let mut left = bound.clone();
-    left.heal_type = HealType::Object {
-        bucket: "mrf-recreated-bucket".to_owned(),
-        object: "part:a".to_owned(),
-        version_id: Some("b".to_owned()),
-    };
-    let mut right = bound.clone();
-    right.heal_type = HealType::Object {
-        bucket: "mrf-recreated-bucket".to_owned(),
-        object: "part".to_owned(),
-        version_id: Some("a:b".to_owned()),
-    };
-    assert_eq!(
-        PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
-        PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
-        "the legacy delimiter-only key demonstrates the collision"
-    );
-    assert_ne!(
-        PriorityHealQueue::make_dedup_key(&left),
-        PriorityHealQueue::make_dedup_key(&right),
-        "length-prefixed object and version components must not alias"
-    );
-    let mut forged = unbound;
-    forged.heal_type = HealType::Object {
-        bucket: "mrf-recreated-bucket".to_owned(),
-        object: PriorityHealQueue::make_dedup_key(&bound),
-        version_id: None,
-    };
-    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
-}
-
-#[test]
-fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
-    let incarnation = Uuid::new_v4();
-    for decode in [false, true] {
-        let make_request = |object: &str, version_id: Option<&str>| {
-            let mut request = incarnation_bound_mrf_fallback(incarnation);
-            let bucket = "mrf-recreated-bucket".to_owned();
-            let object = object.to_owned();
-            let version_id = version_id.map(str::to_owned);
-            request.heal_type = if decode {
-                HealType::ECDecode {
-                    bucket,
-                    object,
-                    version_id,
-                }
-            } else {
-                HealType::Object {
-                    bucket,
-                    object,
-                    version_id,
-                }
-            };
-            request
-        };
-        let absent = make_request("object", None);
-        let empty = make_request("object", Some(""));
-        let forged_object = make_request("object4:some0:", None);
-        let forged_version = make_request("object", Some("4:none"));
-        let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
-        for (index, key) in keys.iter().enumerate() {
-            assert!(keys.iter().skip(index + 1).all(|other| key != other));
-        }
-
-        let mut queue = PriorityHealQueue::new();
-        assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
-        assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
-        assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
-        assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
-        assert_eq!(queue.len(), 2);
-
-        for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
-            let mut legacy_absent = absent.clone();
-            let mut legacy_empty = empty.clone();
-            legacy_absent.source = source;
-            legacy_empty.source = source;
-            if source == HealRequestSource::Mrf {
-                legacy_absent.expected_mrf_bucket_incarnation_id = None;
-                legacy_empty.expected_mrf_bucket_incarnation_id = None;
-            }
-            assert_eq!(
-                PriorityHealQueue::make_dedup_key(&legacy_absent),
-                PriorityHealQueue::make_dedup_key(&legacy_empty),
-                "unbound and non-MRF work retain the legacy empty-version key"
-            );
-        }
-    }
-}
-
-#[test]
-fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
-    let mut queue = PriorityHealQueue::new();
-    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
-    let old_id = old.id.clone();
-    let old_key = PriorityHealQueue::make_dedup_key(&old);
-    let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
-    let new_id = new.id.clone();
-    let new_key = PriorityHealQueue::make_dedup_key(&new);
-    let mut forced_old = old.clone();
-    forced_old.id = Uuid::new_v4().to_string();
-    forced_old.force_start = true;
-    let forced_id = forced_old.id.clone();
-
-    assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
-    assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
-    assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
-    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
-    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
-    queue
-        .remove_request_id(&old_id)
-        .expect("remove the old incarnation representative");
-    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
-    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
-    assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
-    assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
-    assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
-    assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
-}
-
-#[tokio::test]
-async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
-    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
-
-    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
-    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(old)
-            .await
-            .expect("old fallback")
-            .result,
-        HealAdmissionResult::Accepted
-    );
-    let incarnation = Uuid::new_v4();
-    let request = incarnation_bound_mrf_fallback(incarnation);
-    let request_id = request.id.clone();
-    let mut intent = MrfIntent {
-        bucket: Arc::from("mrf-recreated-bucket"),
-        object: Arc::from("object"),
-        version_id: None,
-        kind: MrfKind::PartialWrite,
-        delete_marker_purge: None,
-        scope: Some(MrfScope {
-            pool_index: 0,
-            set_index: 0,
-        }),
-        lease: None,
-        enqueued_at_ms: 1,
-        attempts: 0,
-    };
-    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
-    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
-    let target = MrfRepairNoticeTarget {
-        bucket: Arc::clone(&intent.bucket),
-        object: Arc::clone(&intent.object),
-        version_id: intent.version_id,
-        kind: intent.kind,
-        scope: intent.scope,
-        delete_marker_purge: None,
-        lease: intent.lease,
-        durable_anchor: Some(anchor.clone()),
-    };
-    let receipt = manager
-        .submit_mrf_heal_request_with_receipt_and_identity(request, target)
-        .await
-        .expect("durable admission must be decided");
-    assert_eq!(
-        receipt.result,
-        HealAdmissionResult::Accepted,
-        "a durable new incarnation needs its own owner"
-    );
-    assert_eq!(receipt.task_id, request_id);
-    assert!(
-        manager
-            .heal_queue
-            .lock()
-            .await
-            .requests()
-            .any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
-    );
-    assert!(
-        manager
-            .try_in_flight_durable_mrf_anchors()
-            .expect("stable owner snapshot")
-            .contains(&anchor)
-    );
-}
-
-#[tokio::test]
-async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
-    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
-
-    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
-    let incarnation = Uuid::new_v4();
-    let object = format!("same-generation-{}", Uuid::new_v4());
-    let make_request = || {
-        let mut request = incarnation_bound_mrf_fallback(incarnation);
-        request.heal_type = HealType::Object {
-            bucket: "mrf-recreated-bucket".to_owned(),
-            object: object.clone(),
-            version_id: None,
-        };
-        request
-    };
-    let old = make_request();
-    let old_id = old.id.clone();
-    assert_eq!(
-        manager
-            .submit_heal_request_with_receipt(old)
-            .await
-            .expect("old fallback")
-            .result,
-        HealAdmissionResult::Accepted
-    );
-    let mut intent = MrfIntent {
-        bucket: Arc::from("mrf-recreated-bucket"),
-        object: Arc::from(object.as_str()),
-        version_id: None,
-        kind: MrfKind::PartialWrite,
-        delete_marker_purge: None,
-        scope: Some(MrfScope {
-            pool_index: 0,
-            set_index: 0,
-        }),
-        lease: None,
-        enqueued_at_ms: 1,
-        attempts: 0,
-    };
-    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
-    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
-    let target = MrfRepairNoticeTarget {
-        bucket: Arc::clone(&intent.bucket),
-        object: Arc::clone(&intent.object),
-        version_id: intent.version_id,
-        kind: intent.kind,
-        scope: intent.scope,
-        delete_marker_purge: None,
-        lease: intent.lease,
-        durable_anchor: Some(anchor.clone()),
-    };
-    let merged = manager
-        .submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
-        .await
-        .expect("durable same-incarnation admission");
-    assert_eq!(merged.result, HealAdmissionResult::Merged);
-    assert_eq!(merged.task_id, old_id);
-    assert!(
-        manager
-            .try_in_flight_durable_mrf_anchors()
-            .expect("stable owner snapshot")
-            .contains(&anchor)
-    );
-}
-
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
         task_id: "fixture-token".to_string(),

crates/heal/src/heal/pacing.rs

@@ -7,14 +7,8 @@ use rustfs_concurrency::{
     workload::{ForegroundPressure, foreground_pressure},
 };
 use rustfs_heal_contracts::heal_channel::HealRequestSource;
-use std::{
-    sync::{
-        Arc,
-        atomic::{AtomicU32, Ordering},
-    },
-    time::Duration,
-};
-use tokio::{sync::Mutex as AsyncMutex, time::Instant};
+use std::{sync::Arc, time::Duration};
+use tokio::{sync::Mutex, time::Instant};
 use tokio_util::sync::CancellationToken;
 
 #[derive(Default)]
@@ -28,19 +22,17 @@ pub(crate) enum PacingDecision {
     Wait(Option<ForegroundPressure>),
 }
 
-/// Foreground watermarks for one heal execution. Constructed once per request;
-/// disabled when watermarks are zero or no workload provider is configured.
-#[derive(Clone)]
-pub(crate) struct ForegroundPacing {
+/// Cooperative pacing for one heal execution, not a storage admission permit.
+pub(crate) struct MainlinePacer {
     provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
     read_high: usize,
     write_high: usize,
     pause: Duration,
     source: HealRequestSource,
-    state: Arc<AsyncMutex<PacingState>>,
+    state: Mutex<PacingState>,
 }
 
-impl ForegroundPacing {
+impl MainlinePacer {
     pub(crate) fn new(
         provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
         read_high: usize,
@@ -57,7 +49,7 @@ impl ForegroundPacing {
             write_high: write_high.min(100),
             pause: pause.min(Duration::from_secs(1)),
             source,
-            state: Arc::new(AsyncMutex::new(PacingState::default())),
+            state: Mutex::new(PacingState::default()),
         })
     }
 
@@ -133,142 +125,6 @@ impl ForegroundPacing {
     }
 }
 
-/// Single owner of "how hard may this heal pass push storage?".
-///
-/// One instance spans the whole erasure-set scan (all buckets and pages):
-/// - foreground admission pressure (`ForegroundPacing`)
-/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
-///
-/// Call `observe_object_outcome` exactly once per finished object so a single
-/// SlowDown cannot double-reduce concurrency.
-pub(crate) struct HealPageRate {
-    base_concurrency: usize,
-    /// 0 = full base concurrency; each SlowDown increases the reduction.
-    slow_down_step: AtomicU32,
-    slow_down_until: std::sync::Mutex<Option<Instant>>,
-    foreground: Option<ForegroundPacing>,
-}
-
-impl HealPageRate {
-    /// Always-on controller. Foreground yield is optional; SlowDown control is not.
-    pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
-        Self {
-            base_concurrency: base_concurrency.max(1),
-            slow_down_step: AtomicU32::new(0),
-            slow_down_until: std::sync::Mutex::new(None),
-            foreground,
-        }
-    }
-
-    /// Concurrency after applying the current SlowDown reduction. Always ≥ 1
-    /// so maintenance can still make progress under sustained rate pressure.
-    pub(crate) fn effective_concurrency(&self) -> usize {
-        let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
-        (self.base_concurrency >> step).max(1)
-    }
-
-    /// Fold one finished object into the rate state. Call once per outcome.
-    /// `saw_slow_down` is true when any attempt of this object hit SlowDown,
-    /// including attempts that later succeeded on retry.
-    pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
-        if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
-            self.observe_slow_down();
-            return;
-        }
-        if result.is_ok() {
-            self.observe_success();
-        }
-    }
-
-    /// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
-    fn observe_slow_down(&self) {
-        // fetch_add returns the previous step; +1 so the first SlowDown uses
-        // exponent 1 (2s), not exponent 0 (1s).
-        let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
-        let delay = Self::backoff(step);
-        let ready = Instant::now() + delay;
-        if let Ok(mut until) = self.slow_down_until.lock()
-            && until.is_none_or(|current| current < ready)
-        {
-            *until = Some(ready);
-        }
-    }
-
-    /// Restore one concurrency step only after the SlowDown cool-down expires.
-    /// A same-page success must not undo the reduction while the pass is still
-    /// in the rate-limited window (the page semaphore is fixed until the next page).
-    fn observe_success(&self) {
-        let cooling = self
-            .slow_down_until
-            .lock()
-            .ok()
-            .and_then(|until| *until)
-            .is_some_and(|ready| ready > Instant::now());
-        if cooling {
-            return;
-        }
-        self.slow_down_step
-            .try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
-            .ok();
-    }
-
-    /// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
-    pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
-        self.wait_slow_down(cancel).await?;
-        if let Some(foreground) = &self.foreground {
-            foreground.wait(cancel).await?;
-        }
-        Ok(())
-    }
-
-    /// Foreground admission decision while the caller owns page capacity.
-    pub(crate) fn admission_decision(&self) -> PacingDecision {
-        match &self.foreground {
-            Some(foreground) => foreground.admission_decision(),
-            None => PacingDecision::Ready,
-        }
-    }
-
-    /// Foreground wait after a rejected admission. Returns whether a pause was paid.
-    pub(crate) async fn wait_after_admission(
-        &self,
-        cancel: &CancellationToken,
-        observed: Option<ForegroundPressure>,
-    ) -> Result<bool> {
-        match &self.foreground {
-            Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
-            None => Ok(false),
-        }
-    }
-
-    async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
-        let delay = self
-            .slow_down_until
-            .lock()
-            .ok()
-            .and_then(|until| *until)
-            .map(|ready| ready.saturating_duration_since(Instant::now()));
-        let Some(delay) = delay else {
-            return Ok(());
-        };
-        if delay.is_zero() {
-            return Ok(());
-        }
-        tokio::select! {
-            biased;
-            _ = cancel.cancelled() => Err(Error::TaskCancelled),
-            _ = tokio::time::sleep(delay) => Ok(()),
-        }
-    }
-
-    fn backoff(step: u32) -> Duration {
-        // step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
-        // rate with exponential backoff, and heal must do the same to itself.
-        let exponent = step.clamp(1, 5);
-        Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
-    }
-}
-
 #[cfg(test)]
 pub(crate) struct TestPressure {
     pub(crate) active: std::sync::atomic::AtomicUsize,
@@ -308,87 +164,12 @@ mod tests {
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
 
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_first_slow_down_waits_two_seconds() {
-        let rate = HealPageRate::new(8, None);
-        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
-        let cancel = CancellationToken::new();
-        let before = Instant::now();
-        rate.wait_before_page(&cancel).await.expect("cool-down wait");
-        assert_eq!(Instant::now() - before, Duration::from_secs(2));
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_reduces_concurrency_and_backs_off() {
-        let rate = HealPageRate::new(8, None);
-        assert_eq!(rate.effective_concurrency(), 8);
-        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
-        assert_eq!(rate.effective_concurrency(), 4);
-        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
-        assert_eq!(rate.effective_concurrency(), 2);
-        // Floor is one so maintenance never stops entirely.
-        for _ in 0..8 {
-            rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
-        }
-        assert_eq!(rate.effective_concurrency(), 1);
-
-        let cancel = CancellationToken::new();
-        assert!(rate.wait_before_page(&cancel).await.is_ok());
-
-        // After cool-down, successes restore concurrency one step at a time.
-        for _ in 0..12 {
-            rate.observe_object_outcome(&Ok(true), false);
-        }
-        assert!(rate.effective_concurrency() > 1);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
-        let rate = HealPageRate::new(8, None);
-        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
-        assert_eq!(rate.effective_concurrency(), 4);
-        // Same-page successes while cool-down is armed must not restore fan-out.
-        for _ in 0..4 {
-            rate.observe_object_outcome(&Ok(true), false);
-        }
-        assert_eq!(rate.effective_concurrency(), 4);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
-        let rate = HealPageRate::new(8, None);
-        // Replacement retry: SlowDown on attempt 1, success on attempt 2.
-        rate.observe_object_outcome(&Ok(true), true);
-        assert_eq!(rate.effective_concurrency(), 4);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_wait_is_cancellable() {
-        let rate = HealPageRate::new(4, None);
-        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
-        let cancel = CancellationToken::new();
-        cancel.cancel();
-        assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
-        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
-        let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
-            .expect("foreground pacer");
-        let rate = HealPageRate::new(4, Some(foreground));
-        let cancel = CancellationToken::new();
-        let before = Instant::now();
-        rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
-        assert_eq!(Instant::now() - before, Duration::from_millis(250));
-    }
-
     #[tokio::test(start_paused = true)]
     async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
         for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
             let provider = Arc::new(TestPressure::new(class, 0));
             let pause = Duration::from_millis(250);
-            let pacer = ForegroundPacing::new(
+            let pacer = MainlinePacer::new(
                 provider.clone(),
                 if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
                 if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
@@ -416,9 +197,9 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
-        assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
-        assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
-        let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
+        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
+        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
+        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
         let before = Instant::now();
         pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
         assert_eq!(Instant::now() - before, Duration::from_secs(1));
@@ -428,7 +209,7 @@ mod tests {
     async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
         let paced = Arc::new(
-            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
+            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
         );
         let cancel_first = CancellationToken::new();
         let first = tokio::spawn({
@@ -447,7 +228,7 @@ mod tests {
         cancel_second.cancel();
         assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
         provider.active.store(0, Ordering::SeqCst);
-        let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
+        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
             .expect("independent task");
         let before = Instant::now();
         other_task

crates/heal/src/heal/resume.rs

@@ -35,9 +35,7 @@ mod replacement;
 mod utils;
 
 pub(crate) use checkpoint::AdminErasureCheckpoint;
-pub use checkpoint::{
-    CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
-};
+pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
 pub(crate) use gc::ResumeGc;
 pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
 pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};

crates/heal/src/heal/resume/checkpoint.rs

@@ -152,42 +152,6 @@ impl AdminErasureCheckpoint {
     }
 }
 
-/// Lightweight progress view for resume decisions. Omits the per-page object
-/// ledgers so callers do not clone a million-object checkpoint just to read
-/// counters and positions.
-#[derive(Debug, Clone, Copy)]
-pub struct CheckpointProgressSnapshot {
-    pub current_bucket_index: usize,
-    pub current_object_index: usize,
-    pub successful_objects: u64,
-    pub failed_object_count: u64,
-    pub skipped_object_count: u64,
-    pub skipped_new_versions: u64,
-    pub skipped_ilm_expired: u64,
-    pub processed_bytes: u64,
-    pub total_objects: u64,
-    pub total_bytes: u64,
-    pub baseline_generation: Option<u64>,
-    pub baseline_known: bool,
-    pub counter_unknown: bool,
-}
-
-impl CheckpointProgressSnapshot {
-    pub fn has_progress(&self) -> bool {
-        self.baseline_known
-            || self.successful_objects > 0
-            || self.failed_object_count > 0
-            || self.skipped_object_count > 0
-            || self.skipped_new_versions > 0
-            || self.skipped_ilm_expired > 0
-            || self.processed_bytes > 0
-            || self.total_objects > 0
-            || self.total_bytes > 0
-            || self.baseline_generation.is_some()
-            || self.counter_unknown
-    }
-}
-
 /// resume checkpoint
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ResumeCheckpoint {
@@ -751,75 +715,6 @@ impl CheckpointManager {
         self.checkpoint.read().await.clone()
     }
 
-    /// True when a page object is already settled in the current ledger.
-    /// Membership is checked under the lock without cloning the checkpoint,
-    /// which used to copy every page's dedup sets on the small-object path.
-    pub async fn is_object_settled(&self, key: &str) -> bool {
-        let checkpoint = self.checkpoint.read().await;
-        checkpoint.processed_objects.contains(key)
-            || checkpoint.failed_objects.contains(key)
-            || checkpoint.skipped_objects.contains(key)
-    }
-
-    /// True when the administrator checkpoint already finished.
-    pub async fn admin_completed(&self) -> bool {
-        self.checkpoint
-            .read()
-            .await
-            .admin
-            .as_ref()
-            .is_some_and(|admin| admin.completed)
-    }
-
-    /// Current resume position without cloning the object ledgers.
-    pub async fn resume_position(&self) -> (usize, usize) {
-        let checkpoint = self.checkpoint.read().await;
-        (checkpoint.current_bucket_index, checkpoint.current_object_index)
-    }
-
-    /// Dangling-delete grace summary without cloning the object ledgers.
-    pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
-        let checkpoint = self.checkpoint.read().await;
-        (
-            checkpoint.dangling_delete_grace_objects,
-            checkpoint.dangling_delete_retry_not_before,
-            checkpoint.counter_unknown,
-        )
-    }
-
-    /// Aggregate progress counters without cloning the page dedup sets.
-    pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
-        let checkpoint = self.checkpoint.read().await;
-        CheckpointProgressSnapshot {
-            current_bucket_index: checkpoint.current_bucket_index,
-            current_object_index: checkpoint.current_object_index,
-            successful_objects: checkpoint.successful_objects,
-            failed_object_count: checkpoint.failed_object_count,
-            skipped_object_count: checkpoint.skipped_object_count,
-            skipped_new_versions: checkpoint.skipped_new_versions,
-            skipped_ilm_expired: checkpoint.skipped_ilm_expired,
-            processed_bytes: checkpoint.processed_bytes,
-            total_objects: checkpoint.total_objects,
-            total_bytes: checkpoint.total_bytes,
-            baseline_generation: checkpoint.baseline_generation,
-            baseline_known: checkpoint.baseline_known,
-            counter_unknown: checkpoint.counter_unknown,
-        }
-    }
-
-    /// Finish the administrator outcome in place and persist it.
-    pub async fn complete_admin(&self) -> Result<()> {
-        let mut checkpoint = self.checkpoint.write().await;
-        let admin = checkpoint
-            .admin
-            .as_mut()
-            .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
-        admin.completed = true;
-        admin.outcome.finish(None);
-        drop(checkpoint);
-        self.save_checkpoint().await
-    }
-
     /// update position
     pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;

crates/heal/src/heal/task.rs

@@ -485,7 +485,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
-    mainline_pacer: Option<super::pacing::ForegroundPacing>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
 }
 
 impl HealTask {
@@ -586,7 +586,7 @@ impl HealTask {
         task
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
         self.mainline_pacer = pacer;
         self
     }

crates/heal/src/heal/task/heal_object.rs

@@ -480,9 +480,8 @@ impl HealTask {
                 message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
             })?;
         if current_bucket_incarnation_id != bucket_incarnation_id {
-            return Err(Error::StaleBucketIncarnation {
-                bucket: bucket.to_owned(),
-                expected: Some(bucket_incarnation_id),
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
             });
         }
         expected.bucket_incarnation_id = Some(bucket_incarnation_id);

crates/heal/src/heal/task/tests.rs

@@ -3375,67 +3375,10 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
 
     let result = task.heal_object("bucket", "object", None).await;
 
-    assert!(matches!(
-        result,
-        Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
-    ));
+    assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
     assert!(storage.heal_object_calls.lock().unwrap().is_empty());
 }
 
-#[tokio::test]
-async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
-    for object in ["timed out", "network error", "ordinary"] {
-        let original_incarnation = Uuid::new_v4();
-        let recreated_incarnation = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
-            ..Default::default()
-        });
-        let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
-        request.source = HealRequestSource::Mrf;
-        request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
-        request.options.pool_index = Some(0);
-        request.options.set_index = Some(1);
-        let task = HealTask::from_request(request, storage.clone());
-
-        let error = task
-            .execute()
-            .await
-            .expect_err("a recreated bucket must reject the old MRF responsibility");
-        assert!(
-            !error.is_recoverable_heal(),
-            "user-controlled object names must not make a stale incarnation retryable: {object}"
-        );
-        assert!(
-            matches!(
-                &error,
-                Error::StaleBucketIncarnation { bucket, expected }
-                    if bucket == "bucket" && *expected == Some(original_incarnation)
-            ),
-            "the failure must retain the original bucket incarnation: {error:?}"
-        );
-        assert!(
-            matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
-            "stale responsibility must be classified independently of the object name"
-        );
-        assert!(
-            storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
-            "an old responsibility must not reach storage repair in the recreated bucket"
-        );
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
-        assert!(
-            outcome.objects.iter().all(|item| !matches!(
-                &item.disposition,
-                HealObjectDisposition::Repaired
-                    | HealObjectDisposition::VerifiedHealthy
-                    | HealObjectDisposition::AuthoritativelyAbsent
-            )),
-            "the stale task must not produce a positive repair proof"
-        );
-    }
-}
-
 #[tokio::test(start_paused = true)]
 async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
     let storage = Arc::new(MockStorage {

crates/iam/src/oidc/provider.rs

@@ -16,10 +16,9 @@ use super::{
     transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
 };
 use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
-use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
+use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
 use openidconnect::{
-    AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
-    ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
+    AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
 };
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -213,35 +212,6 @@ impl DiscoveredProviderMetadata {
     }
 }
 
-pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
-    token: &'a CoreIdToken,
-    verifier: &CoreIdTokenVerifier<'_>,
-    nonce_verifier: N,
-) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
-    let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
-        CoreJwsSigningAlgorithm::HmacSha256 => 32,
-        CoreJwsSigningAlgorithm::HmacSha384 => 48,
-        CoreJwsSigningAlgorithm::HmacSha512 => 64,
-        _ => return token.claims(verifier, nonce_verifier),
-    };
-    // RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
-    // openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
-    let invalid_length = || {
-        ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
-            "invalid HMAC signature length".to_owned(),
-        ))
-    };
-    let compact = token.to_string();
-    let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
-    let signature = base64_simd::URL_SAFE_NO_PAD
-        .decode_to_vec(signature)
-        .map_err(|_| invalid_length())?;
-    if signature.len() != expected_length {
-        return Err(invalid_length());
-    }
-    token.claims(verifier, nonce_verifier)
-}
-
 // This adapter is used only for discovery/JWKS fetches, never token exchange.
 struct JwksAcceptClient<'a> {
     inner: &'a ReqwestHttpClient,

crates/iam/src/oidc/provider_tests.rs

@@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
     let now = time::OffsetDateTime::now_utc().unix_timestamp();
     let payload = serde_json::json!({
         "iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
-        "iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
+        "iat": now, "exp": now + 300,
     });
 
     for (algorithm, name, signature_length) in [
@@ -275,28 +275,22 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
                 .expect("parse configured provider metadata");
             let verifier = metadata.verifier(&config);
             assert!(
-                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "valid {name} token must pass for workload={workload}"
             );
-            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
-            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
 
             for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
                 let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
                 let malformed: CoreIdToken = format!("{signing_input}.{signature}")
                     .parse()
-                    .expect("invalid signature length must remain parseable before verification");
+                    .expect("invalid signature length must still reach cryptographic verification");
                 assert!(
-                    verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
+                    malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
                     "{name} signature length {length} must return an error for workload={workload}"
                 );
-                assert!(
-                    verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
-                    "{name} signature length {length} must return an error with a nonce for workload={workload}"
-                );
             }
             assert!(
-                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "invalid inputs must not prevent later valid {name} verification"
             );
         }

crates/iam/src/oidc/runtime.rs

@@ -16,7 +16,7 @@ use super::{
         OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
         SourcedOidcProviderConfig, load_oidc_config_snapshot,
     },
-    provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
+    provider::{ProviderRuntime, discover_provider_for_recovery},
     state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
     transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
 };
@@ -535,7 +535,7 @@ impl OidcSys {
         let verifier = client
             .id_token_verifier()
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
+        let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
         if let Err(e) = verified {
             let verification_error = e.to_string();
             warn!(
@@ -583,7 +583,8 @@ impl OidcSys {
             let verifier = client
                 .id_token_verifier()
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
+            id_token
+                .claims(&verifier, &Nonce::new(session.nonce.clone()))
                 .map_err(|retry_err| {
                     error!(
                         event = EVENT_OIDC_DIAGNOSTICS,
@@ -766,7 +767,7 @@ impl OidcSys {
             .metadata
             .verifier(&config)
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
+        if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
             state = self
                 .provider_runtime
                 .refresh_provider_state(&provider_id, &config)
@@ -779,7 +780,8 @@ impl OidcSys {
                 .metadata
                 .verifier(&config)
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
+            id_token
+                .claims(&verifier, |_: Option<&Nonce>| Ok(()))
                 .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
         }
 

crates/io-metrics/src/internode_metrics.rs

@@ -1411,28 +1411,6 @@ pub fn record_peer_reachable(addr: &str) {
     publish_offline_gauge(&peers);
 }
 
-/// Record a successful interaction with a cluster peer, but only when the peer is currently
-/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
-/// and lock-path successes that do not go through a fresh dial.
-///
-/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
-/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
-/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
-pub fn record_peer_reachable_if_offline(addr: &str) {
-    let key = normalize_peer_key(addr);
-    // Fast path: read lock to check whether the peer is currently offline.
-    {
-        let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
-        match peers.get(key) {
-            None => return,
-            Some(entry) if entry.online => return,
-            Some(_) => {} // offline: fall through to the write path
-        }
-    }
-    // Slow path: the peer is offline, so flip it back online and republish the gauge.
-    record_peer_reachable(addr);
-}
-
 /// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
 /// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
 pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
@@ -2210,34 +2188,6 @@ mod tests {
         assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
     }
 
-    #[test]
-    fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
-        let addr = "http://cluster-peer-recover-if-offline-test:9000";
-
-        // Mark the peer offline.
-        record_peer_unreachable(addr, 1);
-        assert!(cluster_peer_is_offline(addr));
-
-        // A successful cached-channel RPC (no fresh dial) clears the offline state.
-        record_peer_reachable_if_offline(addr);
-        assert!(!cluster_peer_is_offline(addr));
-        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
-    }
-
-    #[test]
-    fn record_peer_reachable_if_offline_no_op_for_online_peer() {
-        let addr = "http://cluster-peer-recover-noop-test:9000";
-
-        record_peer_reachable(addr);
-        // Should not panic or change state for an already-online peer.
-        record_peer_reachable_if_offline(addr);
-        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
-
-        // Should also be a no-op for unknown peers.
-        record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
-        assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
-    }
-
     #[test]
     fn cluster_peer_should_bypass_is_self_healing() {
         let addr = "http://cluster-peer-bypass-selfheal-test:9000";

crates/lock/src/client/local.rs

@@ -382,6 +382,42 @@ impl LockClient for LocalClient {
         }
     }
 
+    async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
+        let shard = self.get_shard(lock_id);
+        let mut guards = shard.write().await;
+        let Some(entry) = guards.get_mut(lock_id) else {
+            return Ok(false);
+        };
+        if entry.is_expired() {
+            return Ok(false);
+        }
+        Ok(entry.guard.downgrade_committed_tail())
+    }
+
+    async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
+        let shard = self.get_shard(lock_id);
+        let mut guards = shard.write().await;
+        let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
+        if entry.is_expired() {
+            return Ok(false);
+        }
+        Ok(entry.guard.try_upgrade_committed_tail())
+    }
+
+    async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
+        self.refresh(lock_id).await
+    }
+
+    async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
+        let shard = self.get_shard(lock_id);
+        let guards = shard.read().await;
+        let Some(entry) = guards.get(lock_id) else { return Ok(None) };
+        if entry.is_expired() {
+            return Ok(None);
+        }
+        Ok(entry.guard.waiting_writer())
+    }
+
     async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
         self.release(lock_id).await
     }
@@ -547,6 +583,32 @@ mod tests {
         assert!(!response.success);
     }
 
+    #[tokio::test]
+    async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
+        let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
+        let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
+        let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
+        assert!(client.acquire_lock(&first).await.expect("initial lock").success);
+        assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
+        assert!(
+            client
+                .downgrade_committed_tail(&first.lock_id)
+                .await
+                .expect("retry downgrade")
+        );
+        let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
+            .with_acquire_timeout(Duration::from_millis(500));
+        assert!(client.acquire_lock(&reader).await.expect("reader").success);
+        let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
+        assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
+        assert!(client.release(&first.lock_id).await.expect("release tail"));
+        assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
+        assert!(client.release(&reader.lock_id).await.expect("release reader"));
+        assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
+        assert!(client.release(&writer.lock_id).await.expect("release writer"));
+        assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
+    }
+
     #[tokio::test]
     async fn acquire_rejects_a_lock_id_for_another_resource() {
         let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));

crates/lock/src/client/mod.rs

@@ -20,6 +20,12 @@ use async_trait::async_trait;
 use futures::future::join_all;
 use std::sync::Arc;
 
+/// Marker exchanged in a refresh request and response for a committed-tail conversion.
+pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
+pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
+pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
+pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
+
 /// Lock client trait
 #[async_trait]
 pub trait LockClient: Send + Sync + std::fmt::Debug {
@@ -48,6 +54,27 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
     /// Refresh lock
     async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
 
+    /// Convert an exclusive lease to a reader-compatible committed-tail lease.
+    /// Unsupported peers retain their original lease and report `false`.
+    async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
+        Ok(false)
+    }
+
+    /// Try to restore exclusive ownership, retaining the writer fence on failure.
+    async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
+        Ok(false)
+    }
+
+    /// Check support without changing the lease mode.
+    async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
+        Ok(false)
+    }
+
+    /// Observe foreground writer contention, or return `None` when unsupported.
+    async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
+        Ok(None)
+    }
+
     /// Force release lock
     async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
 

crates/lock/src/distributed_lock.rs

@@ -262,6 +262,8 @@ pub struct DistributedLockGuard {
     /// All underlying leases that should be refreshed and released with the guard.
     entries: Vec<HeldLockEntry>,
     lock_type: LockType,
+    /// Shared and exclusive quorums must intersect before a tail may convert.
+    committed_tail_quorums_intersect: bool,
     /// If true, Drop will not try to release (used if user manually released).
     disarmed: bool,
     /// Background lease task for renewable leases.
@@ -277,7 +279,13 @@ impl DistributedLockGuard {
     /// - `entries` is the full list of underlying leases that should be refreshed and released
     ///   with this guard.
     /// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
-    fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
+    fn new(
+        lock_id: LockId,
+        entries: Vec<HeldLockEntry>,
+        lock_type: LockType,
+        heartbeat: LockHeartbeatConfig,
+        committed_tail_quorums_intersect: bool,
+    ) -> Self {
         record_lock_held_acquire(lock_type);
         let lock_lost = Arc::new(LockLostSignal::default());
         match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
@@ -296,6 +304,7 @@ impl DistributedLockGuard {
             lock_id,
             entries,
             lock_type,
+            committed_tail_quorums_intersect,
             disarmed: false,
             refresh_task,
             lock_lost,
@@ -412,6 +421,70 @@ impl DistributedLockGuard {
         &self.lock_id
     }
 
+    /// Convert retained peer leases without ever releasing writer exclusion.
+    /// A peer without conversion support keeps its exclusive lease. Partial
+    /// conversion is safe: both lease modes continue to exclude writers.
+    pub async fn downgrade_committed_tail(&mut self) -> bool {
+        if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
+        {
+            return false;
+        }
+        let results = futures::future::join_all(
+            self.entries
+                .iter()
+                .map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
+        )
+        .await;
+        !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
+    }
+
+    /// Require every retained lease to confirm exclusivity before committing.
+    pub async fn try_upgrade_committed_tail(&mut self) -> bool {
+        if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
+            return false;
+        }
+        let results = futures::future::join_all(
+            self.entries
+                .iter()
+                .map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
+        )
+        .await;
+        !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
+    }
+
+    pub async fn supports_heal_lock_transition(&self) -> bool {
+        if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
+        {
+            return false;
+        }
+        let results = futures::future::join_all(
+            self.entries
+                .iter()
+                .map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
+        )
+        .await;
+        !self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
+    }
+
+    /// Observe contention on every held lease. Unknown peers prevent a false idle result.
+    pub async fn waiting_writer(&self) -> Option<bool> {
+        if self.disarmed || self.is_lock_lost() {
+            return None;
+        }
+        let results =
+            futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
+        if self.is_lock_lost() {
+            return None;
+        }
+        if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
+            return Some(true);
+        }
+        results
+            .iter()
+            .all(|result| matches!(result, Ok(Some(false))))
+            .then_some(false)
+    }
+
     /// Actual granted peer identities, which may differ from the logical guard ID.
     pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
         self.entries.iter().map(|entry| &entry.lock_id)
@@ -599,6 +672,7 @@ impl DistributedLock {
                 individual_locks,
                 request.lock_type,
                 heartbeat,
+                self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
             )))
         } else {
             // Check if it's a timeout or quorum failure

crates/lock/src/fast_lock/guard.rs

@@ -39,6 +39,8 @@ pub struct FastLockGuard {
     shard: Option<Arc<LockShard>>, // None when locks are disabled
     released: bool,
     disabled: bool, // True when locks are disabled globally
+    committed_tail: bool,
+    sealed_for_commit: bool,
     /// Unique ID for this guard instance to prevent double-release
     guard_id: u64,
 }
@@ -54,6 +56,8 @@ impl FastLockGuard {
             shard: Some(shard),
             released: false,
             disabled: false,
+            committed_tail: false,
+            sealed_for_commit: false,
             guard_id,
         }
     }
@@ -68,6 +72,8 @@ impl FastLockGuard {
             shard: None,
             released: false,
             disabled: true,
+            committed_tail: false,
+            sealed_for_commit: false,
             guard_id,
         }
     }
@@ -87,6 +93,83 @@ impl FastLockGuard {
         &self.owner
     }
 
+    /// The physical acquisition was force-released before registration.
+    pub(crate) fn abandon_unregistered(&mut self) {
+        if !self.released {
+            self.released = true;
+            record_lock_held_release(self.mode);
+        }
+    }
+
+    /// Keep excluding writers while permitting reads of the committed version.
+    pub fn downgrade_committed_tail(&mut self) -> bool {
+        if self.released || self.sealed_for_commit {
+            return false;
+        }
+        if self.committed_tail {
+            return true;
+        }
+        if self.mode != LockMode::Exclusive {
+            return false;
+        }
+        if self.disabled {
+            self.mode = LockMode::Shared;
+            self.committed_tail = true;
+            return true;
+        }
+        let Some(shard) = &self.shard else { return false };
+        if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
+            return false;
+        }
+        record_lock_held_release(LockMode::Exclusive);
+        record_lock_held_acquire(LockMode::Shared);
+        self.mode = LockMode::Shared;
+        self.committed_tail = true;
+        true
+    }
+
+    /// Try to restore exclusive ownership without releasing the writer fence.
+    pub fn try_upgrade_committed_tail(&mut self) -> bool {
+        if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
+            return false;
+        }
+        if !self.disabled {
+            let Some(shard) = &self.shard else { return false };
+            if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
+                return false;
+            }
+        }
+        if self.committed_tail {
+            record_lock_held_release(LockMode::Shared);
+            record_lock_held_acquire(LockMode::Exclusive);
+        }
+        self.mode = LockMode::Exclusive;
+        self.committed_tail = false;
+        self.sealed_for_commit = true;
+        true
+    }
+
+    /// Observe writers waiting on this guard's exact object key.
+    pub fn waiting_writer(&self) -> Option<bool> {
+        if self.released {
+            return None;
+        }
+        if self.disabled {
+            return Some(false);
+        }
+        self.shard
+            .as_ref()?
+            .waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
+    }
+
+    fn release_physical(&self, shard: &LockShard) -> bool {
+        if self.committed_tail {
+            shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
+        } else {
+            shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
+        }
+    }
+
     /// Manually release the lock early
     ///
     /// Returns true if the lock was successfully released, false if it was
@@ -106,7 +189,7 @@ impl FastLockGuard {
         }
 
         if let Some(shard) = &self.shard {
-            let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
+            let success = self.release_physical(shard);
             if success {
                 self.released = true;
                 record_lock_held_release(self.mode);
@@ -169,7 +252,7 @@ impl Drop for FastLockGuard {
     fn drop(&mut self) {
         if let Some(shard) = &self.shard {
             if !self.released && !self.disabled {
-                let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
+                let success = self.release_physical(shard);
                 if !success {
                     // For high-concurrency scenarios, this is likely due to:
                     // 1. Lock was already released by another thread

crates/lock/src/fast_lock/manager.rs

@@ -96,11 +96,14 @@ impl FastObjectLockManager {
     /// Acquire object lock
     pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
         let shard = self.get_shard(&request.key);
-        match shard.acquire_lock(&request).await {
-            Ok(()) => {
-                let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
+        match shard.acquire_guarded(&request).await {
+            Ok(acquired) => {
+                let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
                 // Register guard to prevent premature cleanup
-                shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
+                if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
+                    guard.abandon_unregistered();
+                    return Err(LockResult::Timeout);
+                }
                 Ok(guard)
             }
             Err(err) => Err(err),
@@ -201,31 +204,15 @@ impl FastObjectLockManager {
         let mut all_failed = Vec::new();
         let mut guards = Vec::new();
 
-        for (shard_id, requests) in shard_groups {
-            let shard = self.shards[*shard_id].clone();
-
+        for (_, requests) in shard_groups {
             for request in requests {
                 let key = request.key.clone();
-                let owner = request.owner.clone();
-                let mode = request.mode;
-
-                let acquired = if shard.try_fast_path_only(request) {
-                    true
-                } else {
-                    match shard.acquire_lock(request).await {
-                        Ok(()) => true,
-                        Err(err) => {
-                            all_failed.push((key.clone(), err));
-                            false
-                        }
+                match self.acquire_lock(request.clone()).await {
+                    Ok(guard) => {
+                        all_successful.push(key);
+                        guards.push(guard);
                     }
-                };
-
-                if acquired {
-                    let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
-                    shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
-                    all_successful.push(key);
-                    guards.push(guard);
+                    Err(err) => all_failed.push((key, err)),
                 }
             }
         }
@@ -245,14 +232,10 @@ impl FastObjectLockManager {
         let mut acquired_guards = Vec::new();
         let mut failed_locks = Vec::new();
 
-        'outer: for (shard_id, requests) in shard_groups {
-            let shard = self.shards[*shard_id].clone();
-
+        'outer: for (_, requests) in shard_groups {
             for request in requests {
-                match shard.acquire_lock(request).await {
-                    Ok(()) => {
-                        let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
-                        shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
+                match self.acquire_lock(request.clone()).await {
+                    Ok(guard) => {
                         acquired_guards.push(guard);
                     }
                     Err(err) => {

crates/lock/src/fast_lock/object_pool.rs

@@ -94,6 +94,7 @@ impl ObjectLockState {
     pub fn reset_for_reuse(&mut self) {
         // Reset atomic state
         self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
+        self.force_epoch.store(0, Ordering::Relaxed);
 
         // Clear owners
         *self.current_owner.write() = None;

crates/lock/src/fast_lock/shard.rs

@@ -15,6 +15,7 @@
 use parking_lot::RwLock;
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::sync::atomic::Ordering;
 use std::time::{Duration, Instant, SystemTime};
 use tokio::time::timeout;
 
@@ -30,6 +31,12 @@ struct ActiveGuardInfo {
     key: ObjectKey,
     mode: LockMode,
     owner: Arc<str>,
+    committed_tail: bool,
+}
+
+pub(crate) struct AcquiredLock {
+    state: Arc<ObjectLockState>,
+    force_epoch: u64,
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -46,6 +53,8 @@ enum TestPausePoint {
     BeforeFastWrite,
     FastWriteExisting,
     SlowLookup,
+    ForceAfterTailRelease,
+    ReleaseAfterUnregister,
 }
 
 #[cfg(test)]
@@ -69,7 +78,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
 
 /// Lock shard to reduce global contention.
 ///
-/// When both are needed, lock `objects` before a state's owner metadata.
+/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
 #[derive(Debug)]
 pub struct LockShard {
     /// Object lock states - using parking_lot for better performance
@@ -133,12 +142,16 @@ impl LockShard {
 
     /// Acquire lock with fast path optimization
     pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
+        self.acquire_guarded(request).await.map(|_| ())
+    }
+
+    pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
         let start_time = Instant::now();
 
         // Try fast path first
-        if let Some(_state) = self.try_fast_path(request) {
+        if let Some(acquired) = self.try_fast_path(request) {
             self.metrics.record_fast_path_success();
-            return Ok(());
+            return Ok(acquired);
         }
 
         // Slow path with waiting
@@ -157,7 +170,7 @@ impl LockShard {
     }
 
     /// Try fast path lock acquisition (lock-free when possible)
-    fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
+    fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
         // First try to get existing state without write lock
         {
             let objects = self.objects.read();
@@ -168,14 +181,16 @@ impl LockShard {
                 test_pause(TestPausePoint::FastExisting, &request.key);
 
                 // Try atomic acquisition
-                let success = match request.mode {
-                    LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
-                    LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
-                };
+                let force_epoch = state.force_epoch.load(Ordering::Acquire);
+                let success = force_epoch != u64::MAX
+                    && match request.mode {
+                        LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
+                        LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
+                    };
                 drop(objects);
 
                 if success {
-                    return Some(state);
+                    return Some(AcquiredLock { state, force_epoch });
                 }
             }
         }
@@ -195,10 +210,11 @@ impl LockShard {
                 #[cfg(test)]
                 test_pause(TestPausePoint::FastWriteExisting, &request.key);
 
-                let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
+                let force_epoch = state.force_epoch.load(Ordering::Acquire);
+                let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
                 drop(objects);
                 if acquired {
-                    return Some(state);
+                    return Some(AcquiredLock { state, force_epoch });
                 }
             } else {
                 // Create new state from pool and acquire immediately
@@ -206,7 +222,7 @@ impl LockShard {
                 let state = Arc::new(*state_box);
                 if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
                     objects.insert(request.key.clone(), state.clone());
-                    return Some(state);
+                    return Some(AcquiredLock { state, force_epoch: 0 });
                 }
             }
         }
@@ -215,7 +231,7 @@ impl LockShard {
     }
 
     /// Slow path with async waiting
-    async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
+    async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
         // Use adaptive timeout based on current load and request priority
         let adaptive_timeout = self.calculate_adaptive_timeout(request);
         let deadline = start_time + adaptive_timeout;
@@ -234,7 +250,7 @@ impl LockShard {
 
         loop {
             // Keep the state in the map until its acquisition attempt completes.
-            let (state, success) = {
+            let (state, success, force_epoch) = {
                 let mut objects = self.objects.write();
                 let state = match objects.get(&request.key) {
                     Some(state) => state.clone(),
@@ -249,17 +265,19 @@ impl LockShard {
 
                 #[cfg(test)]
                 test_pause(TestPausePoint::SlowLookup, &request.key);
-                let success = match request.mode {
-                    LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
-                    LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
-                };
+                let force_epoch = state.force_epoch.load(Ordering::Acquire);
+                let success = force_epoch != u64::MAX
+                    && match request.mode {
+                        LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
+                        LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
+                    };
                 drop(objects);
-                (state, success)
+                (state, success, force_epoch)
             };
 
             if success {
                 self.metrics.record_slow_path_success();
-                return Ok(());
+                return Ok(AcquiredLock { state, force_epoch });
             }
 
             // Check timeout
@@ -381,14 +399,10 @@ impl LockShard {
 
     /// Release lock with guard ID tracking for double-release prevention
     pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
-        // First, try to remove the guard from active set
-        let guard_was_active = {
-            let mut guards = self.active_guards.lock();
-            guards.remove(&guard_id).is_some()
-        };
-
-        // If guard was not active, this is a double-release attempt
-        if !guard_was_active {
+        // Keep force-unlock from releasing this holder and admitting a new
+        // same-owner holder between guard removal and physical release.
+        let mut guards = self.active_guards.lock();
+        if guards.remove(&guard_id).is_none() {
             tracing::debug!(
                 "Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
                 key,
@@ -399,6 +413,9 @@ impl LockShard {
             return false;
         }
 
+        #[cfg(test)]
+        test_pause(TestPausePoint::ReleaseAfterUnregister, key);
+
         // Proceed with normal release
         let should_cleanup;
         let result;
@@ -423,6 +440,8 @@ impl LockShard {
             }
         }
 
+        drop(guards);
+
         if should_cleanup {
             self.schedule_cleanup(key.clone());
         }
@@ -430,21 +449,110 @@ impl LockShard {
         result
     }
 
+    /// Convert an active exclusive guard without opening a writer gap.
+    pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
+        let mut guards = self.active_guards.lock();
+        let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
+        if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
+            return false;
+        }
+        let objects = self.objects.read();
+        let Some(state) = objects.get(key) else { return false };
+        if !state.downgrade_committed_tail(owner) {
+            return false;
+        }
+        info.mode = LockMode::Shared;
+        info.committed_tail = true;
+        true
+    }
+
+    pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
+        let mut guards = self.active_guards.lock();
+        let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
+        if info.key != *key || info.owner.as_ref() != owner.as_ref() {
+            return false;
+        }
+        if info.mode == LockMode::Exclusive && !info.committed_tail {
+            return true;
+        }
+        if info.mode != LockMode::Shared || !info.committed_tail {
+            return false;
+        }
+        let objects = self.objects.read();
+        let Some(state) = objects.get(key) else { return false };
+        if !state.try_upgrade_committed_tail(owner) {
+            return false;
+        }
+        info.mode = LockMode::Exclusive;
+        info.committed_tail = false;
+        true
+    }
+
+    pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
+        let guards = self.active_guards.lock();
+        let info = guards.get(&guard_id)?.as_ref()?;
+        if info.key != *key || info.owner.as_ref() != owner.as_ref() {
+            return None;
+        }
+        let objects = self.objects.read();
+        let state = objects.get(key)?;
+        Some(state.atomic_state.writer_observed_during_tail())
+    }
+
+    /// Release a converted guard and its special reader-admission state.
+    pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
+        let mut guards = self.active_guards.lock();
+        let Some(Some(info)) = guards.get(&guard_id) else { return false };
+        if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
+            return false;
+        }
+        let (success, should_cleanup) = {
+            let objects = self.objects.read();
+            let Some(state) = objects.get(key) else { return false };
+            let success = state.release_committed_tail(owner);
+            (success, success && !state.is_locked() && !state.atomic_state.has_waiters())
+        };
+        if success {
+            guards.remove(&guard_id);
+            self.metrics.record_release();
+        }
+        drop(guards);
+        if should_cleanup {
+            self.schedule_cleanup(key.clone());
+        }
+        success
+    }
+
     /// Register a guard to prevent premature cleanup
     pub fn register_guard(&self, guard_id: u64) {
         self.active_guards.lock().insert(guard_id, None);
     }
 
-    pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
+    pub(crate) fn register_guard_with_info(
+        &self,
+        guard_id: u64,
+        key: &ObjectKey,
+        mode: LockMode,
+        owner: &Arc<str>,
+        acquired: &AcquiredLock,
+    ) -> bool {
         let mut guards = self.active_guards.lock();
+        let objects = self.objects.read();
+        if !objects.get(key).is_some_and(|state| {
+            Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
+        }) {
+            return false;
+        }
         guards.insert(
             guard_id,
             Some(ActiveGuardInfo {
                 key: key.clone(),
                 mode,
                 owner: owner.clone(),
+                committed_tail: false,
             }),
         );
+        true
     }
 
     /// Unregister a guard (called when guard is dropped)
@@ -719,27 +827,57 @@ impl LockShard {
     /// Returns the number of owners that were released. Used by the admin
     /// force-unlock path to clear a stuck resource.
     pub fn force_release_all(&self, key: &ObjectKey) -> usize {
-        let owners_modes: Vec<(Arc<str>, LockMode)> = {
-            let objects = self.objects.read();
-            let Some(state) = objects.get(key) else {
-                return 0;
-            };
-            let mut pairs = Vec::new();
-            if let Some(info) = state.current_owner.read().clone() {
-                pairs.push((info.owner, LockMode::Exclusive));
-            }
-            for entry in state.shared_owners.read().iter() {
-                pairs.push((entry.owner.clone(), LockMode::Shared));
-            }
-            pairs
-        };
-
+        // Serialize tail conversion and prevent new acquisitions while taking
+        // and releasing the owner snapshot. A new tail must never be mistaken
+        // for an ordinary shared owner by force unlock.
+        let mut guards = self.active_guards.lock();
+        let objects = self.objects.write();
+        let Some(state) = objects.get(key) else { return 0 };
+        // Acquisitions completed before this point but not yet registered
+        // must not later create guards for the released generation.
+        let _ = state
+            .force_epoch
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
+        let tail = guards.iter().find_map(|(&id, info)| {
+            info.as_ref()
+                .filter(|info| info.committed_tail && info.key == *key)
+                .map(|info| (id, info.owner.clone()))
+        });
         let mut released = 0;
-        for (owner, mode) in owners_modes {
-            if self.release_lock(key, &owner, mode) {
+        if let Some((id, owner)) = tail
+            && state.release_committed_tail(&owner)
+        {
+            guards.remove(&id);
+            self.metrics.record_release();
+            released += 1;
+        }
+        #[cfg(test)]
+        test_pause(TestPausePoint::ForceAfterTailRelease, key);
+        let mut owners_modes = Vec::new();
+        if let Some(info) = state.current_owner.read().clone() {
+            owners_modes.push((info.owner, LockMode::Exclusive, 1));
+        }
+        for entry in state.shared_owners.read().iter() {
+            owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
+        }
+        for (owner, mode, count) in owners_modes {
+            let mut owner_released = false;
+            for _ in 0..count {
+                let success = match mode {
+                    LockMode::Shared => state.release_shared(&owner),
+                    LockMode::Exclusive => state.release_exclusive(&owner),
+                };
+                if success {
+                    self.metrics.record_release();
+                    owner_released = true;
+                }
+            }
+            if owner_released {
                 released += 1;
             }
         }
+        // Old guard drops must not release a later holder with the same owner.
+        guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
         released
     }
 
@@ -979,6 +1117,315 @@ mod tests {
         )
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
+        for reader in [false, true] {
+            let manager = test_manager();
+            let key = ObjectKey::new("bucket", "force-release-gap");
+            let old = if reader {
+                manager.acquire_read_lock(key.clone(), "same-owner").await
+            } else {
+                manager.acquire_write_lock(key.clone(), "same-owner").await
+            }
+            .expect("old holder");
+            let shard = manager.get_shard(&key).clone();
+            let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
+            let old_drop = tokio::task::spawn_blocking(move || drop(old));
+            entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
+
+            let can_force = shard.active_guards.try_lock().is_some();
+            let mut force = {
+                let manager = manager.clone();
+                let key = key.clone();
+                tokio::task::spawn_blocking(move || manager.force_unlock(&key))
+            };
+            let mut new_guard = if can_force {
+                assert_eq!(force.await.expect("force join"), 1);
+                let mut guard = manager
+                    .acquire_write_lock(key.clone(), "same-owner")
+                    .await
+                    .expect("new writer");
+                if reader {
+                    assert!(guard.downgrade_committed_tail());
+                }
+                resume.send(()).expect("resume old release");
+                old_drop.await.expect("old release join");
+                guard
+            } else {
+                assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
+                resume.send(()).expect("resume old release");
+                old_drop.await.expect("old release join");
+                assert_eq!(force.await.expect("force join"), 0);
+                manager
+                    .acquire_write_lock(key.clone(), "same-owner")
+                    .await
+                    .expect("new writer")
+            };
+            if reader && !can_force {
+                assert!(new_guard.downgrade_committed_tail());
+            }
+            assert!(
+                tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
+                    .await
+                    .is_err(),
+                "an old release must not unlock the new writer"
+            );
+            drop(new_guard);
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn force_unlock_does_not_capture_a_new_committed_tail() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "force-tail-race");
+        let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
+        assert!(old.downgrade_committed_tail());
+        let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
+        let force = {
+            let manager = manager.clone();
+            let key = key.clone();
+            tokio::task::spawn_blocking(move || manager.force_unlock(&key))
+        };
+        entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
+        let mut next = {
+            let manager = manager.clone();
+            let key = key.clone();
+            tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
+        };
+        assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
+        resume.send(()).expect("resume force unlock");
+        assert_eq!(force.await.expect("force unlock task"), 1);
+        let mut new = next.await.expect("new writer task").expect("new writer");
+        assert!(new.downgrade_committed_tail());
+        drop(old);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
+                .await
+                .is_err(),
+            "the new committed tail must still exclude writers"
+        );
+    }
+
+    #[tokio::test]
+    async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "upgrade-tail");
+        let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        let reader = manager
+            .acquire_read_lock(key.clone(), "reader")
+            .await
+            .expect("reader during tail");
+        assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
+                .await
+                .is_err(),
+            "new readers must not starve a pending upgrade"
+        );
+        drop(reader);
+        assert!(tail.try_upgrade_committed_tail());
+        assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
+        assert!(
+            !tail.downgrade_committed_tail(),
+            "a delayed downgrade must not reopen reads after upgrade"
+        );
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
+                .await
+                .is_err()
+        );
+        drop(tail);
+        let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
+    }
+
+    #[tokio::test]
+    async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "brief-writer-observation");
+        let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
+                .await
+                .is_err()
+        );
+        assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
+        assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
+        drop(tail);
+        let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
+        assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
+    }
+
+    #[tokio::test]
+    async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
+        let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        assert!(tail.try_upgrade_committed_tail());
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
+                .await
+                .is_err()
+        );
+        assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
+        drop(tail);
+        let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
+        assert_eq!(next.waiting_writer(), Some(false));
+    }
+
+    #[tokio::test]
+    async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "waiting-writer-observation");
+        let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        assert_eq!(tail.waiting_writer(), Some(false));
+        let writer = {
+            let manager = manager.clone();
+            let key = key.clone();
+            tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
+        };
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while tail.waiting_writer() != Some(true) {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("writer becomes observable");
+        assert_eq!(manager.force_unlock(&key), 1);
+        assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
+        let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
+    }
+
+    #[tokio::test]
+    async fn dropping_a_pending_upgrade_allows_readers_again() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "cancel-upgrade");
+        let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
+        assert!(!tail.try_upgrade_committed_tail());
+        let mut waiting = {
+            let manager = manager.clone();
+            let key = key.clone();
+            tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
+        };
+        assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
+        drop(tail);
+        let second = tokio::time::timeout(Duration::from_secs(2), waiting)
+            .await
+            .expect("reader wakes after cancellation")
+            .expect("reader task joins")
+            .expect("reader obtains lock");
+        drop((reader, second));
+    }
+
+    #[tokio::test]
+    async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "upgrade-stale");
+        let mut tail = manager
+            .acquire_write_lock(key.clone(), "same-owner")
+            .await
+            .expect("write lock");
+        assert!(tail.downgrade_committed_tail());
+        assert_eq!(manager.force_unlock(&key), 1);
+        let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
+        assert!(!tail.try_upgrade_committed_tail());
+    }
+
+    #[tokio::test]
+    async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "late-downgrade");
+        let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
+        assert!(guard.try_upgrade_committed_tail());
+        assert!(!guard.downgrade_committed_tail());
+        assert!(guard.try_upgrade_committed_tail());
+    }
+
+    #[tokio::test]
+    async fn force_unlock_old_readers_cannot_release_a_new_tail() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "force-old-readers");
+        let old1 = manager
+            .acquire_read_lock(key.clone(), "same-owner")
+            .await
+            .expect("first reader");
+        let old2 = manager
+            .acquire_read_lock(key.clone(), "same-owner")
+            .await
+            .expect("second reader");
+        assert_eq!(manager.force_unlock(&key), 1);
+        let mut tail = manager
+            .acquire_write_lock(key.clone(), "same-owner")
+            .await
+            .expect("new writer");
+        assert!(tail.downgrade_committed_tail());
+        drop((old1, old2));
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
+                .await
+                .is_err(),
+            "old reader drops must not release the new tail"
+        );
+    }
+
+    #[tokio::test]
+    async fn forced_unregistered_reader_cannot_release_a_new_tail() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "force-unregistered-reader");
+        let shard = manager.get_shard(&key).clone();
+        let owner: Arc<str> = Arc::from("same-owner");
+        let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
+        let acquired = shard
+            .acquire_guarded(&request)
+            .await
+            .expect("old reader acquired before registration");
+        assert_eq!(manager.force_unlock(&key), 1);
+        let mut tail = manager
+            .acquire_write_lock(key.clone(), owner.clone())
+            .await
+            .expect("new writer");
+        assert!(tail.downgrade_committed_tail());
+        let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
+        assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
+        old.abandon_unregistered();
+        drop(old);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
+                .await
+                .is_err(),
+            "an old reader must not release a new committed tail"
+        );
+    }
+
+    #[tokio::test]
+    async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
+        let manager = test_manager();
+        let key = ObjectKey::new("bucket", "unrelated-acquisition");
+        let shard = manager.get_shard(&key).clone();
+        let owner: Arc<str> = Arc::from("owner");
+        let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
+        let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
+        let other = (0..1000)
+            .map(|index| ObjectKey::new("bucket", format!("other-{index}")))
+            .find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
+            .expect("another key on the same shard");
+        let other_guard = manager
+            .acquire_write_lock(other.clone(), "other")
+            .await
+            .expect("other writer");
+        assert_eq!(manager.force_unlock(&other), 1);
+        drop(other_guard);
+        let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
+        assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
+        drop(guard);
+        assert!(manager.acquire_write_lock(key, "next").await.is_ok());
+    }
+
     fn start_acquisition(
         manager: &Arc<FastObjectLockManager>,
         request: ObjectLockRequest,

crates/lock/src/fast_lock/state.rs

@@ -23,6 +23,11 @@ use crate::fast_lock::types::{LockMode, LockPriority};
 /// Optimized atomic lock state encoding in u64
 /// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
 const WRITER_FLAG_MASK: u64 = 0x1;
+// A committed write tail is a reader-compatible owner that still excludes writers.
+const COMMITTED_TAIL_FLAG: u64 = 0x2;
+const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
+const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
+const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
 const READERS_SHIFT: u8 = 8;
 const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
 const READERS_WAITING_SHIFT: u8 = 16;
@@ -31,7 +36,6 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
 const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
 
 // Fast path check masks
-const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
 const COMPLETELY_UNLOCKED: u64 = 0;
 
 /// Fast atomic lock state for single version
@@ -66,8 +70,8 @@ impl AtomicLockState {
         let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
         match mode {
             LockMode::Shared => {
-                // No writer and no waiting writers
-                (state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
+                (state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
+                    && ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
             }
             LockMode::Exclusive => {
                 // Completely unlocked
@@ -83,8 +87,11 @@ impl AtomicLockState {
         loop {
             let current = self.state.load(Ordering::Acquire);
 
-            // Fast path check - cannot acquire if there's a writer or writers waiting
-            if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
+            // While a committed tail owns a shared slot, let reads through
+            // even if a subsequent writer is waiting for that tail to drain.
+            if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
+                || ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
+            {
                 return false;
             }
 
@@ -137,6 +144,78 @@ impl AtomicLockState {
         }
     }
 
+    /// Atomically retain writer exclusion while allowing reads of a committed tail.
+    fn downgrade_committed_tail(&self) -> bool {
+        loop {
+            let current = self.state.load(Ordering::Acquire);
+            if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
+                return false;
+            }
+            let next = (current & !WRITER_FLAG_MASK)
+                | COMMITTED_TAIL_FLAG
+                | (1 << READERS_SHIFT)
+                | if current & WRITERS_WAITING_MASK != 0 {
+                    TAIL_WRITER_OBSERVED_FLAG
+                } else {
+                    0
+                };
+            if self
+                .state
+                .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
+                .is_ok()
+            {
+                self.update_access_time();
+                return true;
+            }
+        }
+    }
+
+    /// Restore exclusive ownership only after every other reader has left.
+    fn try_upgrade_committed_tail(&self) -> bool {
+        loop {
+            let current = self.state.load(Ordering::Acquire);
+            if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
+                return false;
+            }
+            let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
+            let next = if readers_drained {
+                (current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
+                    | WRITER_FLAG_MASK
+                    | TAIL_TRACK_WRITERS_FLAG
+            } else {
+                current | TAIL_UPGRADE_PENDING_FLAG
+            };
+            if self
+                .state
+                .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
+                .is_ok()
+            {
+                self.update_access_time();
+                return readers_drained;
+            }
+        }
+    }
+
+    /// Release the committed tail's own shared slot and restore ordinary fairness.
+    fn release_committed_tail(&self) -> bool {
+        loop {
+            let current = self.state.load(Ordering::Acquire);
+            if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
+                return false;
+            }
+            let next =
+                (current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
+            if self
+                .state
+                .compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
+                .is_ok()
+            {
+                self.update_access_time();
+                return true;
+            }
+        }
+    }
+
     /// Release shared lock
     pub fn release_shared(&self) -> bool {
         loop {
@@ -169,7 +248,7 @@ impl AtomicLockState {
                 return false; // No exclusive lock to release
             }
 
-            let new_state = current & !WRITER_FLAG_MASK;
+            let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
 
             if self
                 .state
@@ -236,7 +315,10 @@ impl AtomicLockState {
                 return false; // Max waiting writers
             }
 
-            let new_state = current + (1 << WRITERS_WAITING_SHIFT);
+            let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
+            if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
+                new_state |= TAIL_WRITER_OBSERVED_FLAG;
+            }
 
             if self
                 .state
@@ -321,10 +403,15 @@ impl AtomicLockState {
     }
 
     #[cfg(test)]
-    pub fn writers_waiting_count(&self) -> u16 {
+    pub(crate) fn writers_waiting_count(&self) -> u16 {
         let state = self.state.load(Ordering::Acquire);
         self.writers_waiting(state)
     }
+
+    pub(crate) fn writer_observed_during_tail(&self) -> bool {
+        let state = self.state.load(Ordering::Acquire);
+        self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
+    }
 }
 
 /// Object lock state with version support - optimized memory layout
@@ -334,6 +421,8 @@ pub struct ObjectLockState {
     // First cache line: Most frequently accessed data
     /// Atomic state for fast operations
     pub atomic_state: AtomicLockState,
+    /// Changes whenever an administrator force-releases this state.
+    pub(crate) force_epoch: AtomicU64,
 
     // Second cache line: Notification mechanisms
     /// Notification for readers (traditional)
@@ -377,6 +466,7 @@ impl ObjectLockState {
     pub fn new() -> Self {
         Self {
             atomic_state: AtomicLockState::new(),
+            force_epoch: AtomicU64::new(0),
             read_notify: Notify::new(),
             write_notify: Notify::new(),
             optimized_notify: OptimizedNotify::new(),
@@ -425,6 +515,70 @@ impl ObjectLockState {
         true
     }
 
+    /// Convert the current exclusive owner into the committed-tail owner.
+    pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
+        let mut current = self.current_owner.write();
+        let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
+            return false;
+        };
+        let mut shared = self.shared_owners.write();
+        if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
+            return false;
+        }
+        shared.push(SharedOwnerEntry {
+            owner: info.owner,
+            count: 1,
+            acquired_at: info.acquired_at,
+            lock_timeout: info.lock_timeout,
+        });
+        *current = None;
+        drop(shared);
+        drop(current);
+        self.optimized_notify.notify_readers();
+        true
+    }
+
+    pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
+        let mut current = self.current_owner.write();
+        let mut shared = self.shared_owners.write();
+        let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
+            return false;
+        };
+        if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
+            return false;
+        }
+        let info = shared.remove(pos);
+        *current = Some(ExclusiveOwnerInfo {
+            owner: info.owner,
+            acquired_at: info.acquired_at,
+            lock_timeout: info.lock_timeout,
+        });
+        true
+    }
+
+    /// Release the committed-tail owner's shared slot without releasing readers.
+    pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
+        let mut shared = self.shared_owners.write();
+        let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
+            return false;
+        };
+        if !self.atomic_state.release_committed_tail() {
+            return false;
+        }
+        if shared[pos].count > 1 {
+            shared[pos].count -= 1;
+        } else {
+            shared.remove(pos);
+        }
+        let no_readers = shared.is_empty();
+        drop(shared);
+        if no_readers {
+            self.optimized_notify.notify_writer();
+        }
+        self.optimized_notify.notify_readers();
+        true
+    }
+
     /// Release shared lock
     pub fn release_shared(&self, owner: &Arc<str>) -> bool {
         let mut shared = self.shared_owners.write();
@@ -555,6 +709,24 @@ mod tests {
         assert!(!state.release_exclusive());
     }
 
+    #[test]
+    fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
+        let state = AtomicLockState::new();
+        assert!(state.try_acquire_exclusive());
+        assert!(state.inc_writers_waiting());
+        assert!(state.downgrade_committed_tail());
+        assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
+        assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
+        assert!(state.release_committed_tail());
+        assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
+        assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
+        assert!(state.release_shared());
+        assert!(state.try_acquire_exclusive());
+        assert!(state.release_exclusive());
+        state.dec_writers_waiting();
+        assert!(state.is_free());
+    }
+
     // Regression for the waiter-preserving exclusive CAS.
     //
     // The acquisition CAS used to demand a fully-zero state word, which

crates/lock/src/fast_lock/tests.rs

@@ -111,6 +111,67 @@ mod fast_lock_tests {
         assert!(guard.is_released(), "Guard should be marked as released");
     }
 
+    #[tokio::test]
+    async fn force_unlock_old_tail_cannot_release_new_tail() {
+        let manager = create_test_manager();
+        let key = ObjectKey::new("test-bucket", "force-tail-generation");
+        let owner: Arc<str> = Arc::from("same-owner");
+        let mut old = manager
+            .acquire_write_lock(key.clone(), owner.clone())
+            .await
+            .expect("old writer");
+        assert!(old.downgrade_committed_tail());
+        assert_eq!(manager.force_unlock(&key), 1);
+        let mut new = manager
+            .acquire_write_lock(key.clone(), owner.clone())
+            .await
+            .expect("new writer");
+        assert!(new.downgrade_committed_tail());
+        drop(old);
+        let attempt =
+            tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
+                .await;
+        assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
+        drop(new);
+    }
+
+    #[tokio::test]
+    async fn committed_tail_allows_reader_while_writer_waits() {
+        let manager = Arc::new(create_test_manager());
+        let key = ObjectKey::new("test-bucket", "tail-handoff");
+        let mut tail = manager
+            .acquire_write_lock(key.clone(), Arc::from("first-writer"))
+            .await
+            .expect("initial writer");
+        let writer = tokio::spawn({
+            let manager = Arc::clone(&manager);
+            let key = key.clone();
+            async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
+        });
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while manager.current_lock_counts().1 == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("second writer should be queued");
+        assert!(tail.downgrade_committed_tail());
+        let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
+            .await
+            .expect("reader should not wait behind committed tail")
+            .expect("reader should acquire");
+        assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
+        drop(tail);
+        assert!(!writer.is_finished(), "reader still excludes the next writer");
+        drop(reader);
+        let writer = tokio::time::timeout(Duration::from_secs(2), writer)
+            .await
+            .expect("writer should resume")
+            .expect("writer task should join")
+            .expect("writer should acquire");
+        drop(writer);
+    }
+
     #[tokio::test]
     async fn current_lock_counts_report_live_holders_and_waiters() {
         let manager = Arc::new(create_test_manager());

crates/lock/src/namespace/mod.rs

@@ -107,6 +107,38 @@ impl NamespaceLockGuard {
         }
     }
 
+    /// Permit reads while continuing to exclude competing writers.
+    pub async fn downgrade_committed_tail(&mut self) -> bool {
+        match self {
+            Self::Standard(guard) => guard.downgrade_committed_tail().await,
+            Self::Fast(guard) => guard.downgrade_committed_tail(),
+        }
+    }
+
+    /// Try to restore exclusive access while retaining the writer fence.
+    pub async fn try_upgrade_committed_tail(&mut self) -> bool {
+        match self {
+            Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
+            Self::Fast(guard) => guard.try_upgrade_committed_tail(),
+        }
+    }
+
+    /// Check whether the held leases can safely use heal lock transitions.
+    pub async fn supports_heal_lock_transition(&self) -> bool {
+        match self {
+            Self::Standard(guard) => guard.supports_heal_lock_transition().await,
+            Self::Fast(_) => true,
+        }
+    }
+
+    /// Observe whether a writer is waiting on this object's held leases.
+    pub async fn waiting_writer(&self) -> Option<bool> {
+        match self {
+            Self::Standard(guard) => guard.waiting_writer().await,
+            Self::Fast(guard) => guard.waiting_writer(),
+        }
+    }
+
     /// Manually release the lock early
     pub fn release(&mut self) -> bool {
         match self {

crates/lock/src/namespace/tests.rs

@@ -154,6 +154,43 @@ impl crate::client::LockClient for DelayedClient {
     }
 }
 
+#[derive(Debug)]
+struct PendingTailClient(LocalClient);
+
+#[async_trait::async_trait]
+impl crate::client::LockClient for PendingTailClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.0.acquire_lock(request).await
+    }
+    async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.0.release(lock_id).await
+    }
+    async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.0.refresh(lock_id).await
+    }
+    async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
+        std::future::pending().await
+    }
+    async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.0.force_release(lock_id).await
+    }
+    async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.0.check_status(lock_id).await
+    }
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.0.get_stats().await
+    }
+    async fn close(&self) -> crate::Result<()> {
+        self.0.close().await
+    }
+    async fn is_online(&self) -> bool {
+        self.0.is_online().await
+    }
+    async fn is_local(&self) -> bool {
+        self.0.is_local().await
+    }
+}
+
 #[derive(Debug)]
 struct FlakyAcquireClient {
     inner: LocalClient,
@@ -579,6 +616,286 @@ async fn test_namespace_lock_guard_lock_id() {
     }
 }
 
+#[tokio::test]
+async fn tail_conversion_rejects_non_intersecting_quorums() {
+    let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
+        .collect();
+    let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
+    let resource = ObjectKey::new("bucket", "non-intersecting-tail");
+    let mut tail = lock
+        .get_write_lock(resource, "writer", Duration::from_secs(2))
+        .await
+        .expect("write lease");
+    assert!(
+        !tail.downgrade_committed_tail().await,
+        "conversion requires intersecting read and write quorums"
+    );
+    assert!(tail.release());
+}
+
+#[tokio::test]
+async fn partial_tail_conversion_keeps_writers_excluded() {
+    let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
+    for index in 0..4 {
+        let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
+        if index < 2 {
+            clients.push(inner);
+        } else {
+            clients.push(Arc::new(DelayedClient {
+                inner,
+                delay: Duration::ZERO,
+            }));
+        }
+    }
+    let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
+    let resource = ObjectKey::new("bucket", "partial-tail");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "first", Duration::from_secs(2))
+        .await
+        .expect("first writer");
+    assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
+    assert!(
+        lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
+            .await
+            .is_err(),
+        "mixed leases still exclude writers"
+    );
+    assert!(tail.release());
+    let mut writer = lock
+        .get_write_lock(resource, "second", Duration::from_secs(3))
+        .await
+        .expect("writer after release");
+    assert!(writer.release());
+}
+
+#[tokio::test]
+async fn cancelled_tail_conversion_keeps_writers_excluded() {
+    let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
+        .collect();
+    clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
+    let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
+    let resource = ObjectKey::new("bucket", "cancel-tail");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "first", Duration::from_secs(2))
+        .await
+        .expect("first writer");
+    assert!(
+        tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
+            .await
+            .is_err()
+    );
+    let mut reader = lock
+        .get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
+        .await
+        .expect("other peers converted before cancellation");
+    assert!(reader.release());
+    assert!(
+        lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
+            .await
+            .is_err(),
+        "cancelled conversion must still exclude writers"
+    );
+    assert!(tail.release());
+    let mut writer = lock
+        .get_write_lock(resource, "second", Duration::from_secs(3))
+        .await
+        .expect("writer after release");
+    assert!(writer.release());
+}
+
+#[tokio::test]
+async fn unsupported_peer_keeps_exclusive_tail_lease() {
+    let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
+        .map(|_| {
+            let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
+            Arc::new(DelayedClient {
+                inner,
+                delay: Duration::ZERO,
+            }) as Arc<dyn crate::client::LockClient>
+        })
+        .collect();
+    let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
+    let resource = ObjectKey::new("bucket", "old-tail");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "first", Duration::from_secs(2))
+        .await
+        .expect("first writer");
+    assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
+    assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
+    assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
+    assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
+    assert!(
+        lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
+            .await
+            .is_err()
+    );
+    assert!(
+        lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
+            .await
+            .is_err()
+    );
+    assert!(tail.release());
+    let mut writer = lock
+        .get_write_lock(resource, "writer", Duration::from_secs(3))
+        .await
+        .expect("writer after tail");
+    assert!(writer.release());
+}
+
+#[tokio::test]
+async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
+    let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
+        .collect();
+    let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
+    let resource = ObjectKey::new("bucket", "committed-tail");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "first", Duration::from_secs(2))
+        .await
+        .expect("first writer");
+    assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
+    let mut reader = lock
+        .get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
+        .await
+        .expect("committed reader");
+    assert!(
+        lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
+            .await
+            .is_err(),
+        "tail and reader must exclude writer"
+    );
+    assert!(tail.release());
+    assert!(
+        lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
+            .await
+            .is_err(),
+        "reader still excludes writer"
+    );
+    assert!(reader.release());
+    let mut writer = lock
+        .get_write_lock(resource, "second", Duration::from_secs(3))
+        .await
+        .expect("writer after drain");
+    assert!(writer.release());
+}
+
+#[tokio::test]
+async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
+    let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
+        .collect();
+    let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
+    let resource = ObjectKey::new("bucket", "tail-upgrade");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
+        .await
+        .expect("write lock");
+    assert!(tail.supports_heal_lock_transition().await);
+    assert!(tail.downgrade_committed_tail().await);
+    let mut reader = lock
+        .get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
+        .await
+        .expect("reader");
+    assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
+    assert!(reader.release());
+    tokio::time::timeout(Duration::from_secs(2), async {
+        while !tail.try_upgrade_committed_tail().await {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("retry must finish partial upgrades after asynchronous reader release");
+    assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
+    assert!(
+        lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
+            .await
+            .is_err()
+    );
+    assert!(tail.release());
+    let mut writer = lock
+        .get_write_lock(resource, "writer", Duration::from_secs(2))
+        .await
+        .expect("writer after release");
+    assert!(writer.release());
+}
+
+#[tokio::test]
+async fn distributed_tail_observes_a_waiting_writer() {
+    let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
+        .collect();
+    let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
+    let resource = ObjectKey::new("bucket", "writer-observation");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
+        .await
+        .expect("heal lock");
+    assert!(tail.downgrade_committed_tail().await);
+    assert_eq!(tail.waiting_writer().await, Some(false));
+    let writer = {
+        let lock = Arc::clone(&lock);
+        let resource = resource.clone();
+        tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
+    };
+    tokio::time::timeout(Duration::from_secs(2), async {
+        while tail.waiting_writer().await != Some(true) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("waiting writer becomes visible on a held peer");
+    assert!(tail.release());
+    let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
+    assert!(guard.release());
+}
+
+#[tokio::test]
+async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
+    use crate::client::LockClient;
+
+    let clients: Vec<Arc<LocalClient>> = (0..3)
+        .map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
+        .collect();
+    let lock = NamespaceLock::with_clients(
+        "partial-upgrade-writer".to_owned(),
+        clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
+    );
+    let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
+    let mut tail = lock
+        .get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
+        .await
+        .expect("heal lock");
+    let ids = match &tail {
+        NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
+        NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
+    };
+    let mut holders = Vec::new();
+    for client in &clients {
+        for id in &ids {
+            if client.check_status(id).await.expect("peer status").is_some() {
+                holders.push(Arc::clone(client));
+                break;
+            }
+        }
+    }
+    assert!(holders.len() >= 2, "write quorum must hold multiple peers");
+    assert!(tail.downgrade_committed_tail().await);
+    let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
+    assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
+    assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
+    let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
+    assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
+    assert_eq!(
+        tail.waiting_writer().await,
+        Some(true),
+        "partial upgrade must retain the earlier contention"
+    );
+    assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
+    assert!(tail.release());
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_multi_node_simulation() {
     // Simulate a 3-node distributed environment where each node has its own lock backend

crates/protocols/Cargo.toml

@@ -30,7 +30,6 @@ workspace = true
 
 [features]
 default = []
-fips = ["rustfs-tls-runtime?/fips"]
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/protocols/src/ftps/server.rs

@@ -110,9 +110,6 @@ where
             ftps_required = self.config.ftps_required,
             "FTPS server starting"
         );
-        if self.config.tls_enabled && self.config.cert_dir.is_some() {
-            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
-        }
         let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
 
         let storage_clone = self.storage.clone();
@@ -194,6 +191,8 @@ where
                 );
 
                 // Build ServerConfig with SNI support
+                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
                 let server_config = rustls::ServerConfig::builder()
                     .with_no_client_auth()
                     .with_cert_resolver(resolver);

crates/protocols/src/lib.rs

@@ -43,70 +43,3 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
 
 #[cfg(feature = "sftp")]
 pub use sftp::{SftpConfig, SftpInitError, SftpServer};
-
-#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
-mod fips_tests {
-    use crate::common::dummy_storage::DummyBackend;
-    use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
-
-    #[test]
-    fn protocols_reject_incompatible_host_provider() {
-        const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
-        if std::env::var_os(CHILD).is_some() {
-            let mut provider = rustfs_tls_runtime::default_crypto_provider();
-            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
-            provider.install_default().expect("install incompatible host provider");
-            let cert_dir = tempfile::tempdir().expect("certificate directory");
-            let cert_dir = cert_dir.path().to_string_lossy().into_owned();
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("test runtime");
-            runtime.block_on(async {
-                let ftps_config = FtpsConfig {
-                    cert_dir: Some(cert_dir.clone()),
-                    ..Default::default()
-                };
-                let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
-                    .await
-                    .expect("FTPS config");
-                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
-                let error = ftps
-                    .start(receiver)
-                    .await
-                    .expect_err("FTPS must reject the incompatible provider");
-                assert!(error.to_string().contains("process-wide rustls crypto provider"));
-
-                let webdav_config = WebDavConfig {
-                    bind_addr: ([127, 0, 0, 1], 0).into(),
-                    cert_dir: Some(cert_dir),
-                    ..Default::default()
-                };
-                let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
-                    .await
-                    .expect("WebDAV config");
-                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
-                let error = webdav
-                    .start(receiver)
-                    .await
-                    .expect_err("WebDAV must reject the incompatible provider");
-                assert!(error.to_string().contains("process-wide rustls crypto provider"));
-            });
-            return;
-        }
-        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
-            .arg("protocols_reject_incompatible_host_provider")
-            .arg("--nocapture")
-            .env(CHILD, "1")
-            .env("NO_PROXY", "localhost,127.0.0.1,::1")
-            .env("no_proxy", "localhost,127.0.0.1,::1")
-            .output()
-            .expect("start isolated protocol TLS test");
-        assert!(
-            output.status.success(),
-            "stdout={} stderr={}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        );
-    }
-}

crates/protocols/src/webdav/server.rs

@@ -118,10 +118,6 @@ where
             "WebDAV server starting"
         );
 
-        if self.config.tls_enabled && self.config.cert_dir.is_some() {
-            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
-        }
-
         let listener = TcpListener::bind(self.config.bind_addr).await?;
         self.serve(listener, shutdown_rx).await
     }
@@ -159,6 +155,8 @@ where
                     reload_shutdown_rx.clone(),
                 );
 
+                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
                 let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
 
                 Some(TlsAcceptor::from(Arc::new(server_config)))

crates/protos/src/lib.rs

@@ -1092,6 +1092,51 @@ impl_canonical_mutation_body!(
 // its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
 // the JSON fallback decode path, breaks the signature-bound digest.
 
+pub fn canonical_metadata_reservation_request_body(
+    request: &proto_gen::node_service::MetadataReservationRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.bucket)?;
+    body.push_str(&request.object)?;
+    body.push_u64(request.timeout_ms);
+    body.push_bytes(&request.bucket_incarnation_id)?;
+    Ok(body.finish())
+}
+
+pub fn canonical_metadata_reservation_cancel_request_body(
+    request: &proto_gen::node_service::MetadataReservationCancelRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_bytes(&request.token)?;
+    Ok(body.finish())
+}
+
+pub fn canonical_metadata_reservation_activate_request_body(
+    request: &proto_gen::node_service::MetadataReservationActivateRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.bucket)?;
+    body.push_str(&request.object)?;
+    body.push_bytes(&request.token)?;
+    Ok(body.finish())
+}
+
+pub fn canonical_reserved_rename_data_request_body(
+    request: &proto_gen::node_service::ReservedRenameDataRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
+    if let Some(rename) = request.rename.as_ref() {
+        body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
+    } else {
+        body.push_bytes(&[])?;
+    }
+    body.push_bytes(&request.token)?;
+    Ok(body.finish())
+}
+
 pub fn canonical_rename_data_request_body(
     request: &proto_gen::node_service::RenameDataRequest,
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -1722,6 +1767,84 @@ mod disk_mutation_canonical_tests {
         }
     }
 
+    #[test]
+    fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
+        use proto_gen::node_service::{
+            MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
+            ReservedRenameDataRequest,
+        };
+
+        let request = MetadataReservationRequest {
+            disk: "disk".into(),
+            bucket: "bucket".into(),
+            object: "object".into(),
+            timeout_ms: 5000,
+            bucket_incarnation_id: vec![1; 16].into(),
+        };
+        let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
+        for mutate in [
+            |r: &mut MetadataReservationRequest| r.disk = "other".into(),
+            |r: &mut MetadataReservationRequest| r.bucket = "other".into(),
+            |r: &mut MetadataReservationRequest| r.object = "other".into(),
+            |r: &mut MetadataReservationRequest| r.timeout_ms = 1,
+            |r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
+        ] {
+            let mut changed = request.clone();
+            mutate(&mut changed);
+            bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
+        }
+        assert_all_distinct(&bodies);
+        let activation = MetadataReservationActivateRequest {
+            disk: "disk".into(),
+            bucket: "bucket".into(),
+            object: "object".into(),
+            token: vec![1; 16].into(),
+        };
+        let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
+        for mutate in [
+            |r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
+            |r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
+            |r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
+            |r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
+        ] {
+            let mut changed = activation.clone();
+            mutate(&mut changed);
+            bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
+        }
+        assert_all_distinct(&bodies);
+        let cancellation = MetadataReservationCancelRequest {
+            disk: "disk".into(),
+            token: vec![1; 16].into(),
+        };
+        let mut changed = cancellation.clone();
+        changed.token = vec![2; 16].into();
+        assert_ne!(
+            canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
+            canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
+        );
+        let rename = ReservedRenameDataRequest {
+            rename: Some(RenameDataRequest {
+                dst_path: "object".into(),
+                ..Default::default()
+            }),
+            token: vec![1; 16].into(),
+        };
+        let mut changed_token = rename.clone();
+        changed_token.token = vec![2; 16].into();
+        let mut changed_object = rename.clone();
+        changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
+        assert_all_distinct(&[
+            canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
+            canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
+            canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
+            canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
+                rename: None,
+                token: rename.token,
+            })
+            .expect("missing rename"),
+        ]);
+    }
+
     #[test]
     fn rename_data_canonical_body_binds_every_field() {
         let baseline = RenameDataRequest {

crates/protos/src/node.proto

@@ -339,6 +339,31 @@ message RenameDataResponse {
   bytes rename_data_resp_bin = 4;
 }
 
+message MetadataReservationRequest {
+  string disk = 1;
+  string bucket = 2;
+  string object = 3;
+  uint64 timeout_ms = 4;
+  bytes bucket_incarnation_id = 5;
+}
+
+message MetadataReservationCancelRequest {
+  string disk = 1;
+  bytes token = 2;
+}
+
+message MetadataReservationActivateRequest {
+  string disk = 1;
+  string bucket = 2;
+  string object = 3;
+  bytes token = 4;
+}
+
+message ReservedRenameDataRequest {
+  RenameDataRequest rename = 1;
+  bytes token = 2;
+}
+
 // Fixed-scope native pool.bin recovery. The typed MessagePack payload never
 // accepts an arbitrary volume or object path.
 message PoolMetaNativeRecoveryRequest {
@@ -1314,6 +1339,10 @@ service NodeService {
   rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
   rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
   rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
+  rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
+  rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
+  rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
+  rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
   // The method itself requires source-preserving multipart publication on every peer.
   rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
   rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound

crates/rio/src/checksum.rs

@@ -1081,7 +1081,7 @@ impl ChecksumHasher for Sha512Hasher {
 /// MD5 hasher for the ADDITIONAL checksum (x-amz-checksum-md5). Separate from the
 /// legacy Content-MD5 / ETag machinery — this only serves the flexible-checksum path.
 pub struct Md5Hasher {
-    hasher: rustfs_utils::FastMd5,
+    hasher: md5::Md5,
 }
 
 impl Default for Md5Hasher {
@@ -1092,14 +1092,14 @@ impl Default for Md5Hasher {
 
 impl Md5Hasher {
     pub fn new() -> Self {
-        Self {
-            hasher: rustfs_utils::FastMd5::new(),
-        }
+        use md5::Digest as _;
+        Self { hasher: md5::Md5::new() }
     }
 }
 
 impl Write for Md5Hasher {
     fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        use md5::Digest as _;
         self.hasher.update(buf);
         Ok(buf.len())
     }
@@ -1111,11 +1111,13 @@ impl Write for Md5Hasher {
 
 impl ChecksumHasher for Md5Hasher {
     fn finalize(&mut self) -> Vec<u8> {
-        self.hasher.finalize_bytes().to_vec()
+        use md5::Digest as _;
+        self.hasher.clone().finalize().to_vec()
     }
 
     fn reset(&mut self) {
-        self.hasher.reset();
+        use md5::Digest as _;
+        self.hasher = md5::Md5::new();
     }
 }
 

crates/rio/src/etag_reader.rs

@@ -15,8 +15,8 @@
 use crate::compress_index::{Index, TryGetIndex};
 use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
 use hmac::{Hmac, KeyInit, Mac};
+use md5::{Digest, Md5};
 use pin_project_lite::pin_project;
-use rustfs_utils::FastMd5;
 use sha2::Sha256;
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -27,12 +27,11 @@ pin_project! {
     pub struct  EtagReader<R> {
         #[pin]
         pub inner: R,
-        pub md5: FastMd5,
+        pub md5: Md5,
         pub finished: bool,
         pub checksum: Option<String>,
         resolved_etag: Option<String>,
         encrypted_etag_mac: Option<Hmac<Sha256>>,
-        delegate_inner_etag: bool,
     }
 }
 
@@ -40,12 +39,11 @@ impl<R> EtagReader<R> {
     pub fn new(inner: R, checksum: Option<String>) -> Self {
         Self {
             inner,
-            md5: FastMd5::new(),
+            md5: Md5::new(),
             finished: false,
             checksum,
             resolved_etag: None,
             encrypted_etag_mac: None,
-            delegate_inner_etag: false,
         }
     }
 
@@ -60,17 +58,6 @@ impl<R> EtagReader<R> {
         Ok(reader)
     }
 
-    /// Reuse the inner reader's plaintext ETag when deriving an encrypted ETag,
-    /// avoiding a redundant second MD5 pass over the same plaintext stream.
-    pub fn with_inner_etag_delegation(mut self, delegate: bool) -> Self
-    where
-        R: EtagResolvable,
-    {
-        self.delegate_inner_etag =
-            delegate && self.encrypted_etag_mac.is_some() && self.checksum.is_none() && self.inner.is_etag_reader();
-        self
-    }
-
     /// Get the final md5 value (etag) as a hex string, only compute once.
     /// Can be called multiple times, always returns the same result after finished.
     pub fn get_etag(&mut self) -> String {
@@ -78,8 +65,8 @@ impl<R> EtagReader<R> {
             return etag.clone();
         }
 
-        let digest = self.md5.finalize_bytes();
-        let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+        let etag = self.md5.clone().finalize().to_vec();
+        let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
         self.resolved_etag = Some(etag.clone());
         etag
     }
@@ -100,32 +87,30 @@ where
         if let Poll::Ready(Ok(())) = &poll {
             let filled = &buf.filled()[orig_filled..];
             if !filled.is_empty() {
-                if !*this.delegate_inner_etag {
-                    this.md5.update(filled);
-                }
+                this.md5.update(filled);
             } else {
                 // EOF
                 *this.finished = true;
-                if let Some(checksum) = this.checksum {
-                    let etag = if let Some(etag) = this.resolved_etag.as_ref() {
-                        etag.clone()
-                    } else {
-                        let digest = this.md5.finalize_bytes();
-                        let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
-                        *this.resolved_etag = Some(etag.clone());
-                        etag
-                    };
+                let etag = if let Some(etag) = this.resolved_etag.as_ref() {
+                    etag.clone()
+                } else {
+                    let etag = this.md5.clone().finalize().to_vec();
+                    let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
+                    *this.resolved_etag = Some(etag.clone());
+                    etag
+                };
 
-                    if *checksum != etag {
-                        error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
-                        return Poll::Ready(Err(std::io::Error::new(
-                            std::io::ErrorKind::InvalidData,
-                            BadDigest {
-                                expected_md5: checksum.clone(),
-                                calculated_md5: etag,
-                            },
-                        )));
-                    }
+                if let Some(checksum) = this.checksum
+                    && *checksum != etag
+                {
+                    error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
+                    return Poll::Ready(Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        BadDigest {
+                            expected_md5: checksum.clone(),
+                            calculated_md5: etag,
+                        },
+                    )));
                 }
             }
         }
@@ -133,24 +118,16 @@ where
     }
 }
 
-impl<R> EtagResolvable for EtagReader<R>
-where
-    R: EtagResolvable,
-{
+impl<R> EtagResolvable for EtagReader<R> {
     fn is_etag_reader(&self) -> bool {
-        !self.delegate_inner_etag || self.inner.is_etag_reader()
+        true
     }
     fn try_resolve_etag(&mut self) -> Option<String> {
         if let Some(mut mac) = self.encrypted_etag_mac.clone() {
             if !self.finished {
                 return None;
             }
-            let plaintext_etag = if self.delegate_inner_etag {
-                self.inner.try_resolve_etag()?
-            } else {
-                self.get_etag()
-            };
-            mac.update(plaintext_etag.as_bytes());
+            mac.update(self.get_etag().as_bytes());
             let protected = mac.finalize().into_bytes();
             return Some(hex_simd::encode_to_string(&protected[..16], hex_simd::AsciiCase::Lower));
         }
@@ -190,7 +167,6 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
-    use md5::{Digest, Md5};
     use rand::RngExt;
     use std::io::Cursor;
     use tokio::io::{AsyncReadExt, BufReader};
@@ -364,54 +340,4 @@ mod tests {
         assert_eq!(digest.expected_md5, wrong_checksum);
         assert_eq!(digest.calculated_md5, calculated_md5);
     }
-
-    #[tokio::test]
-    async fn encrypted_etag_reader_with_inner_delegation_matches_standalone() {
-        let data = b"delegated encrypted etag payload across multiple 64-byte MD5 blocks for verification";
-        let key = [0x42; 32];
-        let nonce = [0x24; 12];
-
-        let mut standalone =
-            EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce).expect("standalone encrypted EtagReader");
-        let mut out_standalone = Vec::new();
-        standalone.read_to_end(&mut out_standalone).await.expect("read standalone");
-        let expected_tag = standalone.try_resolve_etag().expect("standalone etag");
-
-        let inner =
-            crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
-                .expect("inner HashReader");
-        assert!(inner.is_etag_reader(), "HashReader with diskable_md5=false must report is_etag_reader");
-        let mut delegated = EtagReader::new_encrypted(inner, key, nonce)
-            .expect("delegated encrypted EtagReader")
-            .with_inner_etag_delegation(true);
-        let mut out_delegated = Vec::new();
-        delegated.read_to_end(&mut out_delegated).await.expect("read delegated");
-        assert_eq!(out_delegated, data);
-        assert_eq!(
-            delegated.try_resolve_etag().as_deref(),
-            Some(expected_tag.as_str()),
-            "delegated encrypted EtagReader must produce identical HMAC ETag without re-hashing plaintext"
-        );
-
-        let truncated = crate::LimitReader::new(
-            crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
-                .expect("inner HashReader for limit"),
-            16,
-        );
-        assert!(
-            !truncated.is_etag_reader(),
-            "truncating LimitReader must not advertise inner ETag delegation"
-        );
-
-        let mut non_etag_inner = EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce)
-            .expect("encrypted EtagReader over Cursor")
-            .with_inner_etag_delegation(true);
-        let mut out_cursor = Vec::new();
-        non_etag_inner.read_to_end(&mut out_cursor).await.expect("read cursor");
-        assert_eq!(
-            non_etag_inner.try_resolve_etag().as_deref(),
-            Some(expected_tag.as_str()),
-            "delegation request on non-EtagResolvable inner must keep outer MD5 hashing active"
-        );
-    }
 }

crates/rio/src/hash_reader.rs

@@ -630,10 +630,6 @@ impl AsyncRead for HashReader {
 }
 
 impl EtagResolvable for HashReader {
-    fn is_etag_reader(&self) -> bool {
-        !self.diskable_md5 && self.inner.is_etag_reader()
-    }
-
     fn try_resolve_etag(&mut self) -> Option<String> {
         if self.diskable_md5 {
             return None;

crates/rio/src/lib.rs

@@ -201,31 +201,10 @@ where
     boxed_reader(WarpReader::new(reader))
 }
 
-impl<T> EtagResolvable for std::io::Cursor<T> {}
-
-impl EtagResolvable for &[u8] {}
-
-impl<R> EtagResolvable for tokio::io::BufReader<R>
-where
-    R: tokio::io::AsyncRead + EtagResolvable,
-{
-    fn is_etag_reader(&self) -> bool {
-        self.get_ref().is_etag_reader()
-    }
-
-    fn try_resolve_etag(&mut self) -> Option<String> {
-        self.get_mut().try_resolve_etag()
-    }
-}
-
 impl<T> EtagResolvable for Box<T>
 where
     T: EtagResolvable + ?Sized,
 {
-    fn is_etag_reader(&self) -> bool {
-        self.as_ref().is_etag_reader()
-    }
-
     fn try_resolve_etag(&mut self) -> Option<String> {
         self.as_mut().try_resolve_etag()
     }

crates/s3-client/Cargo.toml

@@ -25,10 +25,6 @@ keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
 categories = ["web-programming", "development-tools", "network-programming"]
 documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
 
-[features]
-default = []
-fips = ["rustfs-tls-runtime/fips"]
-
 [dependencies]
 rustfs-checksums.workspace = true
 rustfs-config.workspace = true

crates/s3-client/src/transition_api.rs

@@ -81,7 +81,7 @@ use time::Duration;
 use time::OffsetDateTime;
 use tokio::io::BufReader;
 use tokio::io::{AsyncRead, AsyncReadExt};
-use tracing::{error, trace, warn};
+use tracing::{debug, error, trace, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
@@ -448,7 +448,16 @@ impl TransitionClient {
         tier_type: &str,
         timeouts: TransitionClientTimeouts,
     ) -> Result<TransitionClient, std::io::Error> {
-        rustfs_tls_runtime::ensure_crypto_provider().map_err(std::io::Error::other)?;
+        if rustls::crypto::CryptoProvider::get_default().is_none() {
+            // No default provider is set yet; try to install aws-lc-rs.
+            // `install_default` can only fail if another thread races us and installs a provider
+            // between our check and this call, which is still safe to ignore.
+            if rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() {
+                debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install");
+            }
+        } else {
+            debug!("rustls crypto provider already installed, skipping aws-lc-rs install");
+        }
 
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
         let timeouts = timeouts.validate()?;
@@ -1853,36 +1862,21 @@ mod tests {
         assert!(outcome.is_ok(), "TLS config creation should not panic");
     }
 
-    #[cfg(feature = "fips")]
+    /// Installing the rustls crypto provider when one is already set must not panic or return
+    /// an error that surfaces to callers (the race-safe `get_default` check guards the install).
     #[test]
-    fn fips_transition_rejects_incompatible_host_provider() {
-        const CHILD: &str = "RUSTFS_TEST_TRANSITION_FIPS_PROVIDER";
-        if std::env::var_os(CHILD).is_some() {
-            let mut provider = rustfs_tls_runtime::default_crypto_provider();
-            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
-            provider.install_default().expect("install incompatible host provider");
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("test runtime");
-            let error = runtime
-                .block_on(super::TransitionClient::new("127.0.0.1:0", super::Options::default(), ""))
-                .expect_err("transition client must reject the incompatible provider");
-            assert!(error.to_string().contains("process-wide rustls crypto provider"));
-            return;
-        }
-        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
-            .arg("fips_transition_rejects_incompatible_host_provider")
-            .arg("--nocapture")
-            .env(CHILD, "1")
-            .output()
-            .expect("start isolated transition TLS test");
-        assert!(
-            output.status.success(),
-            "stdout={} stderr={}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        );
+    fn provider_install_is_idempotent() {
+        // Install once (may already be set by another test in this binary — that's fine).
+        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+
+        // A second install attempt on an already-set provider must not panic.
+        let outcome = std::panic::catch_unwind(|| {
+            if rustls::crypto::CryptoProvider::get_default().is_none() {
+                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+            }
+            // If a default is already present, the branch above is simply skipped.
+        });
+        assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
     }
 
     #[test]

crates/s3select-api/src/object_store.rs

@@ -640,12 +640,12 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
         GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
         GetRange::Offset(start) => HTTPRangeSpec {
             is_suffix_length: false,
-            start: i64::try_from(*start).unwrap_or(i64::MAX),
+            start: *start as i64,
             end: -1,
         },
         GetRange::Suffix(length) => HTTPRangeSpec {
             is_suffix_length: true,
-            start: i64::try_from(*length).unwrap_or(i64::MAX),
+            start: *length as i64,
             end: -1,
         },
     }
@@ -654,15 +654,15 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
 fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
     HTTPRangeSpec {
         is_suffix_length: false,
-        start: i64::try_from(range.start).unwrap_or(i64::MAX),
-        end: i64::try_from(range.end.saturating_sub(1)).unwrap_or(i64::MAX),
+        start: range.start as i64,
+        end: range.end.saturating_sub(1) as i64,
     }
 }
 
 fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
     HTTPRangeSpec {
         is_suffix_length: false,
-        start: i64::try_from(start).unwrap_or(i64::MAX),
+        start: start as i64,
         end: -1,
     }
 }
@@ -2476,48 +2476,6 @@ mod test {
         assert_eq!(range.end, 19);
     }
 
-    #[test]
-    fn test_get_range_conversion_for_suffix_ranges() {
-        // Normal suffix: last 100 bytes.
-        let range = http_range_spec_from_get_range(&GetRange::Suffix(100));
-        assert!(range.is_suffix_length);
-        assert_eq!(range.start, 100);
-        assert_eq!(range.end, -1);
-
-        // Overflowing suffix: `u64::MAX` previously wrapped to `-1` via `as i64`
-        // and was misread as "last 1 byte". It now clamps to `i64::MAX`, which
-        // `get_length` treats as "the whole object".
-        let range = http_range_spec_from_get_range(&GetRange::Suffix(u64::MAX));
-        assert!(range.is_suffix_length);
-        assert_eq!(range.start, i64::MAX);
-        assert_eq!(range.end, -1);
-    }
-
-    #[test]
-    fn test_get_range_conversion_for_offset_overflow() {
-        // Normal offset.
-        let range = http_range_spec_from_get_range(&GetRange::Offset(42));
-        assert!(!range.is_suffix_length);
-        assert_eq!(range.start, 42);
-        assert_eq!(range.end, -1);
-
-        // Overflowing offset clamps to `i64::MAX`, which is then rejected as
-        // out-of-range (416) by `get_offset_length`.
-        let range = http_range_spec_from_get_range(&GetRange::Offset(u64::MAX));
-        assert!(!range.is_suffix_length);
-        assert_eq!(range.start, i64::MAX);
-        assert_eq!(range.end, -1);
-    }
-
-    #[test]
-    fn test_get_range_conversion_for_bounded_overflow() {
-        // Overflowing bounded range clamps start/end to `i64::MAX`.
-        let range = http_range_spec_from_get_range(&GetRange::Bounded(u64::MAX - 10..u64::MAX));
-        assert!(!range.is_suffix_length);
-        assert_eq!(range.start, i64::MAX);
-        assert_eq!(range.end, i64::MAX);
-    }
-
     #[tokio::test]
     async fn test_scan_range_output_can_convert_field_delimiter() {
         let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);

crates/storage-api/src/lib.rs

@@ -90,9 +90,7 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
 pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
 pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
 pub use error::{StorageErrorCode, StorageResult};
-pub use multipart::MultipartUploadInspection;
 pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
-pub use object::AuthoritativeListOperations;
 pub use object::DeleteAccounting;
 pub use object::ObjectLockDeleteOptions;
 pub use object::ObjectToDeleteIdentity;