fix(obs): add fine-grained object lock histograms (#9177)

Use explicit buckets that preserve millisecond lock acquisitions while retaining visibility into long-held delete and recovery locks. Add an exporter-backed regression test for both histogram instruments.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
+sha256-darwin=a3e166a140f3150218b4a857fedb8749940fad0c8b053d0e3c1b957321bf5beb
-sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+sha256-linux=dce4a8eb634a27b4589bcf6d2ebf248ede97aa59701acc8696391d2812a3177d

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
+sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf

.config/nextest.toml

@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -769,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -849,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -497,6 +497,63 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     # Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1397,6 +1454,7 @@ jobs:
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1424,6 +1482,7 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
+            target/x86_64-unknown-linux-gnu/release/rustfs
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-validation-sweep.yml

@@ -92,6 +92,17 @@ jobs:
           CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
         run: |
           set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
           python3 scripts/resolve_functional_candidate.py
           python3 - <<'PY'
           import json
@@ -99,15 +110,15 @@ jobs:
           from pathlib import Path
           root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
           root.mkdir(parents=True, exist_ok=True)
-          candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
-          manifest = candidate["manifest"]
+          manifest = chain["candidate"]["manifest"]
           (root / "candidate.json").write_text(json.dumps({
               "package_url": manifest["package_url"],
               "package_sha256": manifest["package_sha256"],
               "source_sha": manifest["source_sha"],
           }, sort_keys=True) + "\n")
           with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("testing_sha=" + candidate["testing_sha"] + "\n")
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
               output.write("package_url=" + manifest["package_url"] + "\n")
               output.write("source_sha=" + manifest["source_sha"] + "\n")
           PY
@@ -121,6 +132,26 @@ jobs:
           path: auto-testing
           persist-credentials: false
 
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
       - name: Select sweep cases
         id: select
         env:
@@ -224,7 +255,8 @@ jobs:
             # topology, this purge guarantees the next case starts clean even
             # when the previous one died mid-run.
             for node in ${RUSTFS_NODES}; do
-              ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
                 SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
                 ${SUDO} systemctl stop rustfs 2>/dev/null || true
                 if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
@@ -243,6 +275,7 @@ jobs:
           SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
           PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
           SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
         run: |
           set -euo pipefail
           python3 - <<'PY'
@@ -351,7 +384,10 @@ jobs:
             ${{ runner.temp }}/validation-sweep/sweep-cases.txt
             ${{ runner.temp }}/validation-sweep/candidate.json
             ${{ runner.temp }}/validation-sweep/results/
-          if-no-files-found: error
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
           retention-days: 30
 
       - name: Cleanup environment (after)

.github/workflows/windows-filesystem.yml

@@ -107,32 +107,6 @@ jobs:
         shell: pwsh
         run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
 
-      # Native multipart completion publishes its payload from a different
-      # parent than its metadata, and abort renames under a precondition; both
-      # need their own Windows handle discipline.
-      - name: Test native multipart complete and abort publication
-        shell: pwsh
-        run: >-
-          cargo test -p rustfs-ecstore --lib --
-          native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
-          native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
-          --nocapture
-
-      # Owned-undo reclaim removes a directory it pinned, copy heal persists
-      # fault bytes through a handle, and metadata reads meet a directory.
-      - name: Test native multipart reclaim, copy heal, and directory reads
-        shell: pwsh
-        run: >-
-          cargo test -p rustfs-ecstore --lib --
-          native_multipart_owned_undo
-          native_multipart_copy_heal
-          orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
-          --nocapture
-
-      - name: Test directory removal fallback
-        shell: pwsh
-        run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
-
       - name: Test Windows path conversion
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -212,12 +212,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +235,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +254,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -337,7 +338,7 @@ strum = { version = "0.28.0" }
 sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +357,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +378,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.2", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +428,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

Dockerfile

@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -69,6 +70,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
+
 
 FROM alpine:3.24.2
 
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -124,6 +132,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.97.1-trixie
+FROM rust:1.99.1-trixie
+
+ARG TARGETARCH
+ARG RC_RELEASE=latest
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -21,6 +24,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -36,6 +40,7 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -50,8 +55,12 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
+    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -76,6 +77,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
+
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -131,6 +139,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
+ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97.1-trixie AS builder
+FROM rust:1.99.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
+ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-RUN chmod +x /entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -227,6 +235,7 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -248,8 +257,11 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=builder /usr/local/bin/rc /usr/bin/rc
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -263,5 +275,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -0,0 +1,21 @@
+# RustFS image bootstrap
+
+The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
+
+To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
+
+Optional per-bucket files use the bucket name as the filename without an extension:
+
+| Directory | File format | Applied operation |
+| --- | --- | --- |
+| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
+| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
+| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
+| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
+| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
+
+Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
+
+When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
+
+RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -0,0 +1 @@
+[]

bootstrap/cors/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/lifecycle/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/notifications/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/policies/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/public-access-block.json

@@ -0,0 +1,6 @@
+{
+  "block_public_acls": true,
+  "ignore_public_acls": true,
+  "block_public_policy": true,
+  "restrict_public_buckets": true
+}

bootstrap/versioning/.gitkeep

@@ -0,0 +1 @@
+

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,10 +502,163 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyExists</Code>"
+                    "<Code>BucketAlreadyOwnedByYou</Code>"
                 }));
             }
         }
         Ok(())
     }
+
+    /// A deployment owns a single bucket namespace, so an IAM user that recreates
+    /// an existing bucket is the owner, not another account. In the default region
+    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
+    /// semantics), including buckets created by root and by another IAM user.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
+        let root = env.create_s3_client();
+
+        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(build_test_s3_config(
+            &env.url,
+            "duplicate-iam",
+            "duplicate-iam-secret",
+            None,
+            "duplicate-create",
+        ));
+
+        root.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("root creates the first bucket");
+        iam.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
+
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the IAM user creates its own bucket");
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the same IAM user recreating its bucket gets 200 OK");
+        root.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("root recreating an IAM-created bucket gets 200 OK");
+
+        Ok(())
+    }
+
+    /// Outside the default region the same ownership answer becomes
+    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
+    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
+        let region = "eu-west-1";
+        let root = Client::from_conf(
+            env.create_s3_client()
+                .config()
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-eu-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(
+            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readonly",
+            "duplicate-eu-readonly",
+        )
+        .await?;
+        let readonly = Client::from_conf(
+            build_test_s3_config(
+                &env.url,
+                "duplicate-eu-readonly",
+                "duplicate-eu-readonly-secret",
+                None,
+                "duplicate-create",
+            )
+            .to_builder()
+            .region(Region::new(region))
+            .build(),
+        );
+
+        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
+            client
+                .create_bucket()
+                .bucket(bucket)
+                .send()
+                .await
+                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
+            // Both the creator and the other authenticated identity own the namespace.
+            for creator in [&root, &iam] {
+                let error = creator
+                    .create_bucket()
+                    .bucket(bucket)
+                    .send()
+                    .await
+                    .expect_err("a duplicate create outside us-east-1 must report ownership");
+                assert_eq!(
+                    error.as_service_error().and_then(ProvideErrorMetadata::code),
+                    Some("BucketAlreadyOwnedByYou"),
+                    "{bucket}"
+                );
+                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
+            }
+        }
+
+        let denied = readonly
+            .create_bucket()
+            .bucket("duplicate-eu-root")
+            .send()
+            .await
+            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
+        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+        Ok(())
+    }
 }

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,30 +284,45 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
+                after_lock_contention(&format!("{context} COPY retained"), || {
                     copier
                         .copy_object()
                         .bucket(&bucket)
                         .key(&copy_key)
                         .copy_source(format!("{bucket}/{key}"))
                         .send()
-                    .await?;
+                })
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                after_lock_contention(&format!("{context} DELETE source"), || {
-                let missing = reader
+                    writer.delete_object().bucket(&bucket).key(&key).send()
-                    .head_object()
+                })
-                    .bucket(&bucket)
+                .await??;
-                    .key(&key)
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
-                    .send()
+                    reader.head_object().bucket(&bucket).key(&key).send()
-                    .await
+                })
+                .await?
                 .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/lib.rs

@@ -283,6 +283,9 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
+#[cfg(test)]
+mod multipart_upload_id_test;
+
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -0,0 +1,175 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Multipart upload ID regression tests.
+//!
+//! Every operation that addresses an existing multipart upload must report an
+//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
+//! the ID is malformed or is a well-formed ID of an aborted upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+    const BUCKET: &str = "multipart-upload-id";
+    const KEY: &str = "target.bin";
+    const COPY_SOURCE_KEY: &str = "copy-source.bin";
+    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
+    // the object layer rejects this ID as malformed instead of looking it up.
+    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
+    const PART_NUMBER: i32 = 1;
+    const PART_BODY: &[u8] = b"multipart upload id regression part";
+    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
+    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
+    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
+    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
+
+    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
+    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
+    where
+        E: ProvideErrorMetadata + std::fmt::Debug,
+    {
+        let Err(error) = result else {
+            return Some(format!("{operation}: succeeded"));
+        };
+        let status = error.raw_response().map(|response| response.status().as_u16());
+        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
+            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
+    }
+
+    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(Vec::new()).await?;
+        env.create_test_bucket(BUCKET).await?;
+        let client = env.create_s3_client();
+        client
+            .put_object()
+            .bucket(BUCKET)
+            .key(COPY_SOURCE_KEY)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await?;
+        Ok((env, client))
+    }
+
+    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
+    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
+        let upload_part = client
+            .upload_part()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await;
+        let upload_part_copy = client
+            .upload_part_copy()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
+            .send()
+            .await;
+        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
+                    .build(),
+            )
+            .send()
+            .await;
+        let abort = client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await;
+
+        let mismatches: Vec<String> = [
+            no_such_upload_mismatch("UploadPart", upload_part),
+            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
+            no_such_upload_mismatch("ListParts", list_parts),
+            no_such_upload_mismatch("CompleteMultipartUpload", complete),
+            no_such_upload_mismatch("AbortMultipartUpload", abort),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+        assert!(
+            mismatches.is_empty(),
+            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
+            mismatches.join("\n")
+        );
+    }
+
+    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
+    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
+        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
+        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
+        let status = head
+            .as_ref()
+            .err()
+            .and_then(|error| error.raw_response())
+            .map(|response| response.status().as_u16());
+        assert_eq!(
+            status,
+            Some(OBJECT_NOT_FOUND_STATUS),
+            "rejected operations must not publish {KEY}: {head:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+
+    #[tokio::test]
+    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
+        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
+        client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+}

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
-    VersioningConfiguration,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
+    let wait_for_source_writeback = || async {
-        .delete_object()
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
                         .key(object_key)
                         .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
+            replay_mrf_journal(&storage, None).await;
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -3421,6 +3511,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3580,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3661,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3792,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -4171,6 +4296,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4348,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5528,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6494,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,6 +2809,10 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
                 error!(
                     event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
                     component = LOG_COMPONENT_ECSTORE,
@@ -2820,6 +2824,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                     error = %e,
                     "Replication target operation failed"
                 );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent

crates/ecstore/src/cache_value/metacache_set.rs

@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
 use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, warn};
+use tracing::{Instrument, debug, error, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -363,10 +363,15 @@ async fn list_path_raw_inner(
     let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
 
     let cancel_rx = CancellationToken::new();
+    let walk_options = Arc::new(walk_dir_options(&opts));
 
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
-        let opts_clone = opts.clone();
+        let walk_options = walk_options.clone();
+        #[cfg(test)]
+        let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
+        #[cfg(test)]
+        let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
         let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
@@ -376,17 +381,17 @@ async fn list_path_raw_inner(
         let disk_io_context_clone = disk_io_context.clone();
         let (rd, wr) = tokio::io::duplex(64 * 1024);
         readers.push(MetacacheReader::new(rd));
-        jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
+        let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
             let mut wr = PublishedBytesWriter::new(wr);
             #[cfg(test)]
-            if let Some(observer) = &opts_clone.test_disk_io_context_observed {
+            if let Some(observer) = &test_disk_io_context_observed {
                 observer.store(
                     crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
                     std::sync::atomic::Ordering::Release,
                 );
             }
             #[cfg(test)]
-            let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
+            let test_primary_error = if let Some(behavior) = test_reader_behavior {
                 match behavior {
                     TestReaderBehavior::Eof => return Ok(()),
                     TestReaderBehavior::Entries(entries) => {
@@ -443,8 +448,6 @@ async fn list_path_raw_inner(
                 None
             };
 
-            let wakl_opts = walk_dir_options(&opts_clone);
-
             let mut need_fallback = false;
             let mut last_err = None;
             #[cfg(test)]
@@ -459,7 +462,7 @@ async fn list_path_raw_inner(
 
             if !need_fallback && let Some(disk) = opdisk {
                 let primary_walk_started = std::time::Instant::now();
-                match disk.walk_dir(wakl_opts, &mut wr).await {
+                match disk.walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_res) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_primary",
@@ -483,8 +486,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_missing_path",
                                 error = ?err,
@@ -495,8 +498,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_failed",
                                 error = ?err,
@@ -598,8 +601,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
+                            bucket = %walk_options.bucket,
-                            path = %opts_clone.path,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing_for_path",
                             error = ?err,
@@ -610,8 +613,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
+                            bucket = %walk_options.bucket,
-                            path = %opts_clone.path,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing",
                             error = ?err,
@@ -626,7 +629,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
+                match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -652,8 +655,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_missing_path",
                                 error = ?err,
@@ -664,8 +667,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_failed",
                                 error = ?err,
@@ -687,12 +690,13 @@ async fn list_path_raw_inner(
 
             // warn!("list_path_raw: while need_fallback done");
             Ok(())
-        }));
+        });
+        jobs.spawn(producer.in_current_span());
     }
 
     let revjob_rx = rx.clone();
     let mut revjobs = JoinSet::new();
-    revjobs.spawn(async move {
+    let merge = async move {
         // Consumer-side peek timeout, and a caveat worth understanding
         // (rustfs/backlog#1217).
         //
@@ -1035,7 +1039,8 @@ async fn list_path_raw_inner(
             }
         }
         Ok(ListPathRawOutcome { limit_reached })
-    });
+    };
+    revjobs.spawn(merge.in_current_span());
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
@@ -1249,6 +1254,163 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn list_path_raw_fanout_retains_only_linear_disk_references() {
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let baseline = Arc::strong_count(&disk);
+
+        for primary_count in [1, 4, 16] {
+            for fallback_count in [0, 2] {
+                let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let behaviors = entered
+                    .iter()
+                    .zip(&stopped)
+                    .map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
+                        entered: entered.clone(),
+                        stopped: stopped.clone(),
+                    })
+                    .collect();
+                let cancel = CancellationToken::new();
+                let listing = tokio::spawn(list_path_raw(
+                    cancel.clone(),
+                    ListPathRawOptions {
+                        disks: vec![Some(disk.clone()); primary_count],
+                        fallback_disks: vec![Some(disk.clone()); fallback_count],
+                        min_disks: primary_count,
+                        test_reader_behaviors: behaviors,
+                        peek_timeout: Some(Duration::from_secs(60)),
+                        ..Default::default()
+                    },
+                ));
+                for signal in &entered {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("every producer must enter the real fanout before counting references");
+                }
+
+                // The merge owns the configured primary/fallback slots, each producer
+                // owns its primary, and the shared fallback queue owns each fallback.
+                let expected = baseline + 2 * (primary_count + fallback_count);
+                let retained = Arc::strong_count(&disk);
+                cancel.cancel();
+                timeout(Duration::from_secs(2), listing)
+                    .await
+                    .expect("cancellation must stop the listing")
+                    .expect("listing task must join")
+                    .expect("external cancellation must finish cleanly");
+                for signal in &stopped {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("cancellation must drop every producer");
+                }
+                assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
+                assert_eq!(
+                    retained, expected,
+                    "fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn list_path_raw_producer_and_merge_preserve_request_span() {
+        use tracing_subscriber::{Layer, layer::SubscriberExt};
+
+        struct ProducerSpan(Arc<AtomicBool>);
+        #[derive(Default)]
+        struct StateField(bool);
+        impl tracing::field::Visit for StateField {
+            fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
+
+            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+                self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
+            }
+        }
+        impl<S> Layer<S> for ProducerSpan
+        where
+            S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+        {
+            fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
+                let mut state = StateField::default();
+                event.record(&mut state);
+                if state.0
+                    && context
+                        .lookup_current()
+                        .is_some_and(|span| span.name() == "flink_list_request")
+                {
+                    self.0.store(true, std::sync::atomic::Ordering::Release);
+                }
+            }
+        }
+
+        // The subscriber is thread-local; keep the spawned tasks on this runtime thread.
+        let producer_observed = Arc::new(AtomicBool::new(false));
+        let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let error = list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![Some(disk)],
+                bucket: "missing-bucket".to_owned(),
+                min_disks: 1,
+                ..Default::default()
+            },
+        )
+        .instrument(request.clone())
+        .await
+        .expect_err("missing bucket must fail the real disk producer");
+        assert_eq!(error, DiskError::VolumeNotFound);
+        assert!(
+            producer_observed.load(std::sync::atomic::Ordering::Acquire),
+            "disk error must retain the request span"
+        );
+
+        let merge_observed = Arc::new(AtomicBool::new(false));
+        let merge_sink = merge_observed.clone();
+        let request_id = request.id();
+        list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
+                agreed: Some(Box::new(move |_| {
+                    merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
+                    Box::pin(async { Ok(()) })
+                })),
+                ..Default::default()
+            },
+        )
+        .instrument(request)
+        .await
+        .expect("merge should deliver the test entry");
+        assert!(
+            merge_observed.load(std::sync::atomic::Ordering::Acquire),
+            "merge callback must retain the request span"
+        );
+    }
+
     #[tokio::test]
     async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
         let root = tempfile::tempdir().expect("local disk root should be created");

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1637,6 +1637,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1709,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1780,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2012,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2027,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -18,6 +18,7 @@ use crate::cluster::rpc::client::{
 use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
+use futures::{FutureExt, future::BoxFuture};
 use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
@@ -32,16 +33,17 @@ use rustfs_protos::{
 };
 use std::collections::HashMap;
 use std::future::Future;
+use std::panic::AssertUnwindSafe;
 use std::pin::Pin;
 use std::sync::{Mutex, OnceLock};
 use std::time::Duration;
-use tokio::task::JoinHandle;
 use tokio::time::{Instant, timeout};
 use tonic::service::interceptor::InterceptedService;
 use tonic::{Request, Response};
 use tracing::{debug, info, warn};
 
 const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
+const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
 
@@ -49,18 +51,38 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
     set_tonic_rolling_mutation_body_digest(request)
 }
 
-/// Work to run if an RPC that already timed out for its caller completes later.
+/// Work to run if an RPC completes after its caller stops waiting.
 type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
 type LateFailure = LateCompletion<()>;
+type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
 
-struct RpcCleanup<T> {
+/// Own admission before connecting as well as while an RPC is in flight.
-    late: LateCompletion<T>,
+struct LockRequestGuard<'a> {
-    late_failure: LateFailure,
+    client: &'a RemoteClient,
+    op: &'static str,
     admission: Option<LockRequestAdmission>,
 }
 
-impl<T> RpcCleanup<T> {
+impl LockRequestGuard<'_> {
-    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
+    fn into_admission(mut self) -> Option<LockRequestAdmission> {
+        self.admission.take()
+    }
+}
+
+impl Drop for LockRequestGuard<'_> {
+    fn drop(&mut self) {
+        self.client.release_lock_request(self.op, self.admission.take());
+    }
+}
+
+struct RpcCleanup<'a, T> {
+    late: LateCompletion<T>,
+    late_failure: LateFailure,
+    admission: Option<LockRequestGuard<'a>>,
+}
+
+impl<'a, T> RpcCleanup<'a, T> {
+    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
         Self {
             late,
             late_failure,
@@ -69,6 +91,25 @@ impl<T> RpcCleanup<T> {
     }
 }
 
+/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
+struct InFlightRpc<'a, T: Send + 'static> {
+    client: &'a RemoteClient,
+    op: &'static str,
+    resource_summary: &'a str,
+    future: Option<RpcFuture<T>>,
+    cleanup: Option<RpcCleanup<'a, T>>,
+}
+
+impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
+    fn drop(&mut self) {
+        if let Some(future) = self.future.take()
+            && let Some(cleanup) = self.cleanup.take()
+        {
+            self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
+        }
+    }
+}
+
 /// The liveness window is this many RPC deadlines: a peer that completed a
 /// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
 const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
@@ -137,6 +178,9 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
     let mut peers = lock_peer_channel_health()
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if let Some(health) = peers.get_mut(addr) {
+        return update(health);
+    }
     update(peers.entry(addr.to_string()).or_default())
 }
 
@@ -158,6 +202,15 @@ fn reset_lock_peer_health_for_test(addr: &str) {
         .remove(addr);
 }
 
+#[cfg(test)]
+type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
+
+#[cfg(test)]
+fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
+    static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
+    PAUSES.get_or_init(Mutex::default)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum EvictionTrigger {
     /// The caller's deadline expired while the stream was still open.
@@ -445,6 +498,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -502,6 +559,14 @@ impl RemoteClient {
         admission
     }
 
+    fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
+        self.admit_lock_request(op).map(|admission| LockRequestGuard {
+            client: self,
+            op,
+            admission: Some(admission),
+        })
+    }
+
     fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
         if is_request_breaker_operation(op) && admission.is_some() {
             with_lock_peer_health(&self.addr, |health| {
@@ -583,6 +648,17 @@ impl RemoteClient {
         trigger: EvictionTrigger,
         deadline: Duration,
     ) {
+        #[cfg(test)]
+        let pause = eviction_pauses_for_test()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .remove(&self.addr);
+        #[cfg(test)]
+        if let Some((entered, resume)) = pause {
+            let _ = entered.send(());
+            let _ = resume.await;
+        }
+
         let now = Instant::now();
         let cooldown = Self::eviction_cooldown();
         let liveness_window = Self::liveness_window(deadline);
@@ -613,22 +689,26 @@ impl RemoteClient {
         );
     }
 
-    /// Keep a timed-out RPC running instead of cancelling its stream.
+    /// Keep an abandoned RPC running instead of cancelling its stream.
     ///
     /// Dropping the future sends `RST_STREAM`; under load those resets pile up
     /// in the server's pending-accept queue until it answers `GOAWAY
     /// too_many_resets` and kills every stream on the connection. A detached
     /// stream ends on its own within the internode RPC timeout, the number per
     /// peer is bounded, and a lock granted after its caller gave up is released.
-    fn detach_timed_out_rpc<T: Send + 'static>(
+    fn detach_rpc<T: Send + 'static>(
         &self,
         op: &'static str,
         resource_summary: &str,
-        handle: JoinHandle<std::result::Result<T, tonic::Status>>,
+        future: RpcFuture<T>,
-        late: LateCompletion<T>,
+        cleanup: RpcCleanup<'_, T>,
-        late_failure: LateFailure,
-        admission: Option<LockRequestAdmission>,
     ) {
+        let RpcCleanup {
+            late,
+            late_failure,
+            admission,
+        } = cleanup;
+        let admission = admission.and_then(LockRequestGuard::into_admission);
         let limit = Self::detached_rpc_limit();
         let admitted = with_lock_peer_health(&self.addr, |health| {
             if health.detached_rpcs >= limit {
@@ -639,25 +719,29 @@ impl RemoteClient {
             }
         });
         if !admitted {
-            handle.abort();
+            drop(future);
             self.release_lock_request(op, admission);
             if let Some(late_failure) = late_failure {
                 drop(tokio::spawn(late_failure(())));
             }
             rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
             debug!(
+                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                state = "detached_budget_exhausted",
                 addr = %self.addr,
                 op,
                 resource_summary,
                 limit,
-                "Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
+                "Abandoned remote lock RPC cancelled"
             );
             return;
         }
         rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
         let addr = self.addr.clone();
         tokio::spawn(async move {
-            let outcome = handle.await;
+            let outcome = future.await;
             with_lock_peer_health(&addr, |health| {
                 health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
                 if is_request_breaker_operation(op) && admission.is_some() {
@@ -672,6 +756,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;
@@ -683,19 +768,31 @@ impl RemoteClient {
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
                     debug!(
+                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                        state = "late_rpc_failed",
                         addr = %addr,
                         op,
                         tonic_code = ?status.code(),
                         tonic_message = status.message(),
-                        "Detached remote lock RPC failed after its caller timed out"
+                        "Abandoned remote lock RPC failed"
                     );
                 }
-                Err(join_error) => {
+                Err(_) => {
                     if let Some(late_failure) = late_failure {
                         late_failure(()).await;
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
-                    debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
+                    debug!(
+                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                        state = "late_rpc_panicked",
+                        addr = %addr,
+                        op,
+                        "Abandoned remote lock RPC panicked"
+                    );
                 }
             }
         });
@@ -783,16 +880,28 @@ impl RemoteClient {
         self.release_late_release();
         rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
         if outcome == "released" {
-            debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
+            debug!(
+                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                state = "late_grants_released",
+                addr = %self.addr,
+                count = original_count,
+                "Released remote locks granted after their caller stopped waiting"
+            );
         } else {
             let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
             if log_warning {
                 warn!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "late_grants_unreleased",
                     addr = %self.addr,
                     count = original_count,
                     outcome,
                     suppressed_timeouts,
-                    "Could not release every remote lock granted after its caller timed out; the server lease will expire it"
+                    "Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
                 );
             } else {
                 debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
@@ -806,25 +915,47 @@ impl RemoteClient {
         resource_summary: &str,
         deadline: Duration,
         future: Fut,
-        cleanup: RpcCleanup<T>,
+        cleanup: RpcCleanup<'_, T>,
     ) -> std::result::Result<T, LockError>
     where
         Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
         T: Send + 'static,
     {
-        let RpcCleanup {
+        let mut rpc = InFlightRpc {
-            late,
+            client: self,
+            op,
+            resource_summary,
+            future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
+            cleanup: Some(cleanup),
+        };
+        let outcome = match rpc.future.as_mut() {
+            Some(future) => tokio::select! {
+                // A response becoming ready on the deadline tick must not turn an
+                // expired caller budget into a transport error and channel eviction.
+                biased;
+                _ = tokio::time::sleep(deadline) => None,
+                outcome = future => Some(outcome),
+            },
+            None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
+        };
+        if outcome.is_some() {
+            rpc.future.take();
+        }
+        let Some(RpcCleanup {
+            late: _,
             late_failure,
             admission,
-        } = cleanup;
+        }) = rpc.cleanup.as_mut()
-        let mut handle = tokio::spawn(future);
+        else {
-        match timeout(deadline, &mut handle).await {
+            return Err(LockError::internal("remote lock RPC has no cleanup owner"));
-            Ok(Ok(Ok(response))) => {
+        };
-                self.record_rpc_success(op, admission);
+        let token = admission.as_ref().and_then(|guard| guard.admission);
-                self.release_lock_request(op, admission);
+        let result = match outcome {
+            Some(Ok(Ok(response))) => {
+                self.record_rpc_success(op, token);
                 Ok(response)
             }
-            Ok(Ok(Err(err))) => {
+            Some(Ok(Err(err))) => {
                 let reason = err.to_string();
                 if err.code() == tonic::Code::DeadlineExceeded {
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
@@ -857,12 +988,11 @@ impl RemoteClient {
                             "Suppressed repeated remote lock RPC deadline"
                         );
                     }
-                    self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
+                    if let Some(late_failure) = late_failure.take() {
-                        .await;
-                    self.release_lock_request(op, admission);
-                    if let Some(late_failure) = late_failure {
                         drop(tokio::spawn(late_failure(())));
                     }
+                    self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
+                        .await;
                     return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
                 }
                 // Only evict (and re-dial) the cached channel when the failure is a genuine
@@ -923,30 +1053,32 @@ impl RemoteClient {
                     );
                 }
                 if transport_failure {
+                    // Ambiguous-grant cleanup owns its work before channel eviction can suspend.
+                    if let Some(late_failure) = late_failure.take() {
+                        drop(tokio::spawn(late_failure(())));
+                    }
                     self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
                         .await;
                 }
-                self.release_lock_request(op, admission);
-                if transport_failure && let Some(late_failure) = late_failure {
-                    drop(tokio::spawn(late_failure(())));
-                }
                 Err(LockError::internal(format!("{op} RPC failed: {reason}")))
             }
-            Ok(Err(join_error)) => {
+            Some(Err(_)) => {
                 warn!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "rpc_panicked",
                     addr = %self.addr,
                     op,
                     resource_summary,
-                    error = %join_error,
+                    "Remote lock RPC panicked"
-                    "Remote lock RPC task ended abnormally"
                 );
-                self.release_lock_request(op, admission);
+                if let Some(late_failure) = late_failure.take() {
-                if let Some(late_failure) = late_failure {
                     drop(tokio::spawn(late_failure(())));
                 }
-                Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
+                Err(LockError::internal(format!("{op} RPC future panicked")))
             }
-            Err(_) => {
+            None => {
                 let reason = format!("RPC timed out after {deadline:?}");
                 rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
                 self.record_request_timeout(op, deadline);
@@ -978,12 +1110,17 @@ impl RemoteClient {
                         "Suppressed repeated remote lock RPC timeout"
                     );
                 }
+                // Continue draining the abandoned stream while channel eviction waits.
+                drop(rpc);
                 self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
                     .await;
-                self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
+                return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
-                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
             }
+        };
+        if rpc.future.is_none() {
+            rpc.cleanup.take();
         }
+        result
     }
 
     fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
@@ -1074,7 +1211,7 @@ impl RemoteClient {
 #[async_trait]
 impl LockClient for RemoteClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
-        let admission = match self.admit_lock_request("lock") {
+        let admission = match self.admit_lock_request_guard("lock") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_response(request)),
         };
@@ -1085,7 +1222,6 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock", rpc_timeout);
-                self.release_lock_request("lock", Some(admission));
                 return Err(err);
             }
         };
@@ -1093,14 +1229,12 @@ impl LockClient for RemoteClient {
         let args = match serde_json::to_string(&bounded_request) {
             Ok(args) => args,
             Err(err) => {
-                self.release_lock_request("lock", Some(admission));
                 return Err(LockError::internal(format!("Failed to serialize request: {err}")));
             }
         };
         let mut req = Request::new(GenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
-            self.release_lock_request("lock", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_hook(request.lock_id.clone());
@@ -1142,7 +1276,7 @@ impl LockClient for RemoteClient {
             return Ok(Vec::new());
         }
 
-        let admission = match self.admit_lock_request("lock_batch") {
+        let admission = match self.admit_lock_request_guard("lock_batch") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_batch(requests)),
         };
@@ -1156,7 +1290,6 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock_batch", rpc_timeout);
-                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
@@ -1170,14 +1303,12 @@ impl LockClient for RemoteClient {
         {
             Ok(args) => args,
             Err(err) => {
-                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
         let mut req = Request::new(BatchGenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
-            self.release_lock_request("lock_batch", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
@@ -1705,6 +1836,351 @@ mod tests {
         assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
     }
 
+    #[tokio::test]
+    async fn test_remote_client_cancelled_connection_releases_request_admission() {
+        ensure_test_rpc_secret();
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
+        let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
+        let client = RemoteClient::new(addr.clone());
+        let request = test_lock_request(Duration::from_secs(30));
+
+        for batch in [false, true] {
+            reset_lock_peer_health_for_test(&addr);
+            with_lock_peer_health(&addr, |health| {
+                health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
+            });
+            let mut acquisition = Box::pin(async {
+                if batch {
+                    client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
+                } else {
+                    client.acquire_lock(&request).await.map(|_| ())
+                }
+            });
+            std::future::poll_fn(|cx| {
+                assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
+                std::task::Poll::Ready(())
+            })
+            .await;
+            let pending = lock_peer_health_for_test(&addr);
+            assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
+            assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
+
+            drop(acquisition);
+            let cancelled = lock_peer_health_for_test(&addr);
+            assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
+            assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
+            let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
+            client.release_lock_request("lock", Some(probe));
+        }
+        reset_lock_peer_health_for_test(&addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
+        let addr = "http://cancelled-rpc-late-grant";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
+                Box::pin(async move {
+                    released_tx.send(granted).expect("late grant observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_secs(30),
+                    async move {
+                        started_tx.send(()).expect("caller observes RPC dispatch");
+                        Ok(grant_rx.await.expect("test decides when peer grants lock"))
+                    },
+                    RpcCleanup::new(late, None, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), started_rx)
+            .await
+            .expect("RPC must dispatch")
+            .expect("dispatch signal");
+        caller.abort();
+        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            1,
+            "cancelled RPC uses bounded detached ownership"
+        );
+        grant_tx.send(true).expect("cancellation must preserve the running RPC");
+        assert!(
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("late grant must be cleaned up")
+                .expect("cleanup result")
+        );
+        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
+        let addr = "http://cancelled-rpc-late-failure";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late_failure: LateFailure = Some(Box::new(move |()| {
+                Box::pin(async move {
+                    released_tx.send(()).expect("ambiguous cleanup observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock_batch",
+                    "bucket/object",
+                    Duration::from_secs(30),
+                    async move {
+                        started_tx.send(()).expect("caller observes RPC dispatch");
+                        failure_rx.await.expect("test decides when transport fails");
+                        Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
+                    },
+                    RpcCleanup::new(None, late_failure, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), started_rx)
+            .await
+            .expect("RPC must dispatch")
+            .expect("dispatch signal");
+        caller.abort();
+        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
+        failure_tx.send(()).expect("cancellation must preserve the running RPC");
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("ambiguous grant must be cleaned up")
+            .expect("cleanup signal");
+        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
+        for (label, status) in [
+            ("transport", tonic::Status::cancelled("ambiguous transport failure")),
+            ("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
+        ] {
+            let addr = format!("http://cancelled-completed-{label}");
+            reset_lock_peer_health_for_test(&addr);
+            let client = RemoteClient::new(addr.clone());
+            let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
+            let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
+            eviction_pauses_for_test()
+                .lock()
+                .expect("eviction pause registry")
+                .insert(addr.clone(), (evicting_tx, resume_rx));
+            let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+            let caller = tokio::spawn(async move {
+                let late_failure: LateFailure = Some(Box::new(move |()| {
+                    Box::pin(async move {
+                        released_tx
+                            .send(())
+                            .expect("completed-failure cleanup observer remains alive");
+                    })
+                }));
+                client
+                    .execute_rpc(
+                        "lock",
+                        "bucket/object",
+                        Duration::from_secs(30),
+                        async move { Err::<(), _>(status) },
+                        RpcCleanup::new(None, late_failure, None),
+                    )
+                    .await
+            });
+            timeout(Duration::from_secs(1), evicting_rx)
+                .await
+                .expect("completed error must reach channel eviction")
+                .expect("eviction signal");
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("cleanup must transfer before channel eviction suspends")
+                .expect("cleanup signal");
+            caller.abort();
+            assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
+            assert_eq!(
+                lock_peer_health_for_test(&addr).detached_rpcs,
+                0,
+                "completed RPC cannot consume a detached slot"
+            );
+            reset_lock_peer_health_for_test(&addr);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
+        let addr = "http://same-tick-rpc-deadline";
+        reset_lock_peer_health_for_test(addr);
+        with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
+        let client = RemoteClient::new(addr.to_string());
+        let budget = Duration::from_millis(50);
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let late_failure: LateFailure = Some(Box::new(move |()| {
+            Box::pin(async move {
+                released_tx.send(()).expect("same-tick cleanup observer remains alive");
+            })
+        }));
+        let result = client
+            .execute_rpc(
+                "lock",
+                "bucket/object",
+                budget,
+                async move {
+                    tokio::time::sleep(budget).await;
+                    Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
+                },
+                RpcCleanup::new(None, late_failure, None),
+            )
+            .await;
+        assert!(
+            matches!(result, Err(LockError::Timeout { .. })),
+            "expired caller budget must retain timeout classification"
+        );
+        let health = lock_peer_health_for_test(addr);
+        assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
+        assert!(
+            health.last_eviction.is_none(),
+            "a recently serving peer keeps its channel after a caller timeout"
+        );
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("same-tick ambiguous completion must trigger cleanup")
+            .expect("cleanup signal");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            0,
+            "same-tick completion returns detached admission"
+        );
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
+        let addr = "http://timeout-late-grant-during-eviction";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
+        let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
+        eviction_pauses_for_test()
+            .lock()
+            .expect("eviction pause registry")
+            .insert(addr.to_string(), (evicting_tx, resume_rx));
+        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
+                Box::pin(async move {
+                    released_tx.send(granted).expect("late grant observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_millis(25),
+                    async move { Ok(grant_rx.await.expect("test controls the late grant")) },
+                    RpcCleanup::new(late, None, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), evicting_rx)
+            .await
+            .expect("timed-out RPC must reach channel eviction")
+            .expect("eviction signal");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            1,
+            "eviction waits with a detached RPC owner"
+        );
+        grant_tx
+            .send(true)
+            .expect("timed-out stream remains available for a late grant");
+        assert!(
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("late grant must be cleaned up before eviction resumes")
+                .expect("late cleanup result")
+        );
+        assert!(!caller.is_finished(), "channel eviction is still paused");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            0,
+            "late completion returns detached admission"
+        );
+        resume_tx.send(()).expect("resume channel eviction");
+        let result = caller.await.expect("timed-out caller completes without panicking");
+        assert!(
+            matches!(result, Err(LockError::Timeout { .. })),
+            "caller preserves its RPC timeout result"
+        );
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_success_polls_rpc_on_callers_task() {
+        tokio::spawn(async {
+            let client = RemoteClient::new("http://direct-poll-success".to_string());
+            let caller_id = tokio::task::id();
+            let rpc_id = client
+                .execute_rpc(
+                    "ping",
+                    "health-lock-online",
+                    Duration::from_secs(1),
+                    async { Ok(tokio::task::id()) },
+                    RpcCleanup::new(None, None, None),
+                )
+                .await
+                .expect("successful RPC must return its result");
+            assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
+        })
+        .await
+        .expect("caller task must complete");
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let client = RemoteClient::new("http://direct-poll-panic".to_string());
+            let late_failure: LateFailure = Some(Box::new(move |()| {
+                Box::pin(async move {
+                    released_tx.send(()).expect("panic cleanup observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc::<(), _>(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_secs(1),
+                    async { panic!("test peer RPC panic") },
+                    RpcCleanup::new(None, late_failure, None),
+                )
+                .await
+        });
+        let result = caller.await.expect("RPC panic must not escape to its caller");
+        assert!(
+            matches!(result, Err(LockError::Internal { .. })),
+            "RPC panic remains an internal lock failure"
+        );
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("panic must trigger ambiguous-grant cleanup")
+            .expect("cleanup signal");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -6839,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7234,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7288,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7301,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7314,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -14546,6 +14606,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14556,10 +14633,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -29482,6 +29559,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();

crates/ecstore/src/disk/disk_store.rs

@@ -853,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +882,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1577,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1932,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2638,6 +2673,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -6044,6 +6086,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/fs.rs

@@ -148,58 +148,47 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
     native_io(move || create_dir_all_std(&path)).await
 }
 
-fn is_dir_error(path: &Path, e: &io::Error) -> bool {
+fn is_dir_error(e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
         // macOS: remove_file on a directory returns EPERM
         || (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
-        || is_windows_dir_access_denied(path, e)
-}
-
-/// Windows reports `remove_file` on a directory (or directory symlink) as
-/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
-/// failure without inspecting the entry itself.
-#[cfg(windows)]
-fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
-    use std::os::windows::fs::FileTypeExt;
-
-    e.kind() == io::ErrorKind::PermissionDenied
-        && std::fs::symlink_metadata(path).is_ok_and(|metadata| {
-            let file_type = metadata.file_type();
-            file_type.is_dir() || file_type.is_symlink_dir()
-        })
-}
-
-#[cfg(not(windows))]
-fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
-    false
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
+    // Try remove_file first; fall back to remove_dir if it's a directory
+    let file_path = path.as_ref().to_path_buf();
+    match native_io(move || std::fs::remove_file(file_path)).await {
+        Ok(()) => Ok(()),
+        Err(e) if is_dir_error(&e) => {
             let path = path.as_ref().to_path_buf();
-    native_io(move || remove_std(path)).await
+            native_io(move || std::fs::remove_dir(path)).await
+        }
+        Err(e) => Err(e),
+    }
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir_all if it's a directory
-    let path = path.as_ref().to_path_buf();
+    let file_path = path.as_ref().to_path_buf();
-    native_io(move || match std::fs::remove_file(&path) {
+    match native_io(move || std::fs::remove_file(file_path)).await {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
+        Err(e) if is_dir_error(&e) => {
+            let path = path.as_ref().to_path_buf();
+            native_io(move || std::fs::remove_dir_all(path)).await
+        }
         Err(e) => Err(e),
-    })
+    }
-    .await
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir if it's a directory
-    let path = path.as_ref();
+    match std::fs::remove_file(path.as_ref()) {
-    match std::fs::remove_file(path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
+        Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
         Err(e) => Err(e),
     }
 }

crates/ecstore/src/disk/local/commit.rs

@@ -198,12 +198,7 @@ pub(super) fn create_local_inline_rollback_backup(
             return Err(err);
         }
     } else if sync && !old_metadata.is_empty() {
-        // Windows only flushes a handle opened for writing.
+        std::fs::File::open(&backup_path)?.sync_data()?;
-        #[cfg(windows)]
-        let backup = std::fs::OpenOptions::new().write(true).open(&backup_path)?;
-        #[cfg(not(windows))]
-        let backup = std::fs::File::open(&backup_path)?;
-        backup.sync_data()?;
     }
     Ok(backup_path)
 }
@@ -1599,13 +1594,12 @@ impl LocalDisk {
                 );
             }
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
-                #[cfg(any(target_os = "linux", windows))]
+                #[cfg(target_os = "linux")]
                 let payload_guard = if multipart_publication.is_some() {
                     Some(
                         os::multipart_payload_commit_guard(
                             &commit.directory_guard,
                             src_data_path.parent().ok_or(DiskError::InvalidPath)?,
-                            commit.publication_root,
                             commit.mutation_lease.clone(),
                         )
                         .await
@@ -1614,9 +1608,9 @@ impl LocalDisk {
                 } else {
                     None
                 };
-                #[cfg(any(target_os = "linux", windows))]
+                #[cfg(target_os = "linux")]
                 let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
-                #[cfg(not(any(target_os = "linux", windows)))]
+                #[cfg(not(target_os = "linux"))]
                 let directory_guard = &commit.directory_guard;
                 if let Err(err) = os::rename_all_with_commit_guard(
                     src_data_path,

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -1002,21 +1002,17 @@ pub(super) fn inspection_with_markers(
 
 type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
 
-static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
+const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
-    std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
 
-fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
+static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
-    let mut map = PREPARED_DELETE_RECEIPTS.lock();
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
-    if map.len() >= 4096
+
-        && let Some(evict_key) = map.keys().next().cloned()
+fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
-    {
+    PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
-        map.remove(&evict_key);
-    }
-    map.insert(rollback, entry);
 }
 
 fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
-    PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
+    PREPARED_DELETE_RECEIPTS.remove(rollback)
 }
 
 impl LocalDisk {
@@ -1291,7 +1287,13 @@ impl LocalDisk {
         let volume = volume.to_owned();
         let object = object.to_owned();
         let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
-        os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
+        // Journal inspections ride the durable-mutation lane (the fused delete
+        // prepare already does): a bulk delete publishes one transaction per
+        // key, and the one-worker foreground namespace lane would serialize
+        // those per-drive publishes behind every other mutation.
+        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
+            Ok(inspection_with_markers(&root, &volume, &object, transaction))
+        })
         .await
         .map_err(to_file_error)?
     }
@@ -1689,7 +1691,10 @@ impl LocalDisk {
             let plan_root = root.clone();
             let missing = observed.current_digest.is_none();
             let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
-            let plan = os::run_blocking_namespace_operation(lease, move || {
+            // Same durable-mutation lane as the journal inspection above: this
+            // plan closure runs inside every settle on the bulk-delete publish
+            // path.
+            let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
                 Ok((|| {
                     let before =
                         regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -270,21 +270,17 @@ struct ActiveMultipartSourceEntry {
     parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
 }
 
-static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
-    parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
-> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
 
-pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
     let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
         return;
     };
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
-    if map.len() >= 4096
-        && let Some(evict_key) = map.keys().next().cloned()
-    {
-        map.remove(&evict_key);
-    }
-    map.insert(
         source,
         ActiveMultipartSourceEntry {
             raw,
@@ -304,56 +300,50 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
 }
 
 pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
-    let (raw, fi, head_stamp) = {
+    let (raw, fi, head_stamp) =
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
-        let entry = map.get(source)?;
-        (entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
-    };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     Some((raw, fi))
 }
 
 pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
-    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     true
 }
 
 pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
-    let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
+    let Some((head_stamp, clean)) =
-        .lock()
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
-        .get(source)
-        .map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
     else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     clean
 }
 
 pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.unsettled_parts.insert(part_number);
         entry.parts.remove(&part_number);
-    }
+    });
 }
 
 pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
     let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .lock()
+        .with_get(source, |entry| entry.fi.data_dir)
-        .get(source)
+        .flatten()
-        .and_then(|entry| entry.fi.data_dir)
         .filter(|dir| !dir.is_nil())
     else {
         return;
@@ -362,9 +352,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
     let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
         return;
     };
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.parts.insert(part.number, (part_stamp, part));
-    }
+    });
 }
 
 pub(super) fn finish_active_multipart_part_transaction(
@@ -373,34 +363,29 @@ pub(super) fn finish_active_multipart_part_transaction(
     committed: bool,
     retired_existing_txn: bool,
 ) {
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
     let Some(part_number) = part_number.filter(|_| committed) else {
-        map.remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return;
     };
     if !retired_existing_txn {
         return;
     }
-    if let Some(entry) = map.get_mut(source)
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
-        && !entry.unsettled_parts.remove(&part_number)
-    {
-        map.remove(source);
-    }
 }
 
 pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
-    let (head_stamp, data_dir, expected_stamp, part) = {
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+        .with_get(source, |entry| {
-        let entry = map.get(source)?;
             if !entry.unsettled_parts.is_empty() {
                 return None;
             }
             let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
             let (expected_stamp, part) = entry.parts.get(&part_number)?;
-        (entry.head_stamp, data_dir, *expected_stamp, part.clone())
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
-    };
+        })
+        .flatten()?;
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -411,11 +396,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
 }
 
 pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
 }
 
 pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
 }
 
 /// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -1527,7 +1512,7 @@ impl LocalDisk {
                     }
                     Err(error) => return Err(error),
                 };
-                let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
+                let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
                 if !parts.is_empty() {
                     let anchor_path = directory.join(RETIREMENT_ANCHOR);
                     let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
@@ -1560,13 +1545,7 @@ impl LocalDisk {
                 }
                 // An anchor-less empty DD is only rmdir'ed. A new child makes
                 // this fail closed; no recursive removal can consume its bytes.
-                // A Windows pin refuses DELETE sharing, so the leaf pin is
-                // released first; the object pin still holds the parent.
-                #[cfg(windows)]
-                drop(directory_pin);
                 std::fs::remove_dir(&directory)?;
-                #[cfg(not(windows))]
-                drop(directory_pin);
                 if sync {
                     os::fsync_dir_std(&object_path)?;
                 }
@@ -1942,6 +1921,7 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
     pub(super) fn prepare_and_install_head_on_thread(
         &self,
         new_metadata: &[u8],

crates/ecstore/src/services/rebalance/entry.rs

@@ -1899,10 +1899,6 @@ mod tests {
                 &ObjectOptions {
                     version_id: Some(version_id.to_string()),
                     mod_time: earlier(version.mod_time),
-                    // Without the object lock the rename waits for every
-                    // drive, so no early-ACK tail is still writing the copies
-                    // removed below.
-                    no_lock: true,
                     ..versioned
                 },
             ))

crates/ecstore/src/services/rebalance/worker.rs

@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     }
 }
 
+fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
+    match rebalance_error_source(err) {
+        Error::Lock(err) => is_rebalance_transient_lock_error(err),
+        // No metadata was written when lock admission failed. Each attempt
+        // must acquire its quorum again before loading or saving this run.
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket,
+            object,
+            ..
+        } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
+        _ => false,
+    }
+}
+
 fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return true;
@@ -385,9 +400,7 @@ where
         match result {
             Ok(value) => return Ok(value),
             Err(err) => {
-                if attempt.saturating_add(1) >= max_attempts.max(1)
+                if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
-                    || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
-                {
                     return Err(err);
                 }
                 #[cfg(test)]
@@ -719,6 +732,82 @@ mod error_source_tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
+        let mut attempts = 0;
+        retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(if attempts == 1 {
+                resolve_rebalance_meta_save_result(
+                    Err(rebalance_meta_lock_error(
+                        rustfs_lock::LockError::QuorumNotReached {
+                            required: 1,
+                            achieved: 0,
+                        },
+                        "write",
+                    )),
+                    "stop_rebalance",
+                )
+            } else {
+                Ok(())
+            })
+        })
+        .await
+        .expect("a later quorum-protected admission must be allowed to succeed");
+        assert_eq!(attempts, 2);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
+        for max_attempts in [0, 1, 3] {
+            let mut attempts = 0;
+            let result = retry_rebalance_metadata_access(None, max_attempts, || {
+                attempts += 1;
+                std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )))
+            })
+            .await;
+            let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
+            assert_eq!(attempts, max_attempts.max(1));
+            assert!(matches!(
+                rebalance_error_source(&error),
+                Error::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    required: 1,
+                    achieved: 0,
+                    ..
+                }
+            ));
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
+        let mut attempts = 0;
+        let result = retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(Err::<(), _>(if attempts == 1 {
+                rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )
+            } else {
+                Error::OperationCanceled
+            }))
+        })
+        .await;
+        assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
+        assert!(matches!(result, Err(Error::OperationCanceled)));
+    }
+
     #[tokio::test]
     async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
         for err in [
@@ -731,6 +820,20 @@ mod error_source_tests {
                 required: 3,
                 achieved: 2,
             },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "data-bucket".to_string(),
+                object: REBAL_META_NAME.to_string(),
+                required: 1,
+                achieved: 0,
+            },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
+                object: "other-metadata.bin".to_string(),
+                required: 1,
+                achieved: 0,
+            },
             Error::other("stale rebalance run rejected: lock acquisition timed out"),
             Error::other("rebalance distributed run fence lost"),
             Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),

crates/ecstore/src/set_disk/mod.rs

@@ -4696,6 +4696,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
             .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
 }
 
+// Reusing a held writer additionally requires the same quorum, so repeated
+// clients must not give different weight to an otherwise identical domain.
+fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    !left.is_empty()
+        && left.len() == right.len()
+        && left.iter().all(|client| {
+            left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+                && right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+        })
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5645,9 +5656,9 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            (true, true) => {
+            // Distributed lock clients receive the raw ObjectKey; the pool
-                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
+            // namespace is not part of the key stored by their backends.
-            }
+            (true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
             _ => false,
         }
     }
@@ -9882,7 +9893,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn namespace_lock_domain_includes_pool_namespace() {
+    async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
 
         let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9921,11 +9932,25 @@ mod tests {
             "sets in the same pool share the object namespace lock domain"
         );
         assert!(
-            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
+            same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pool namespaces must not be deduplicated solely by identical clients"
+            "different pools using identical backends lock the same raw object key"
         );
     }
 
+    #[test]
+    fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
+        let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
+        assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
+        assert!(!same_distributed_lock_quorum(
+            &[a.clone(), a.clone(), b.clone()],
+            &[a.clone(), b.clone(), b]
+        ));
+        assert!(!same_distributed_lock_quorum(&[], &[]));
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {

crates/ecstore/src/set_disk/ops/locking.rs

@@ -95,13 +95,8 @@ impl SetDisks {
         bucket: &str,
         object: &str,
     ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
-        let resource = ObjectKey::with_version(
-            bucket,
-            object_name_disk_identity(object).as_ref(),
-            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
-        );
         let guard = self
-            .namespace_lock_for_key(resource)
+            .namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
             .await
             .get_write_lock(super::super::get_lock_acquire_timeout())
             .await
@@ -111,6 +106,94 @@ impl SetDisks {
         Ok(Arc::new(guard))
     }
 
+    fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
+        ObjectKey::with_version(
+            bucket,
+            object_name_disk_identity(object).as_ref(),
+            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
+        )
+    }
+
+    /// Batched write-lock acquisition for the whole key set of one bulk
+    /// mutation: one lock round instead of one quorum round-trip per key.
+    ///
+    /// Bulk delete is the only multi-key write-lock holder. The batched round
+    /// is deadlock-free by construction (a single `lock_batch` RPC per lock
+    /// client, and every lock server orders concurrent batches by the same
+    /// global (shard, key) order), so unlike an optimistic probe wave it
+    /// needs no canonical serial fallback. Outcomes are per key and use the
+    /// same error shape as the single-key paths.
+    async fn namespace_write_locks_batch(
+        &self,
+        bucket: &str,
+        resources: &[ObjectKey],
+        timeout: std::time::Duration,
+        mode: &'static str,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        if resources.is_empty() {
+            return Vec::new();
+        }
+        let lock = self.namespace_lock_for_key(resources[0].clone()).await;
+        lock.get_write_locks_batch(resources, timeout)
+            .await
+            .into_iter()
+            .zip(resources)
+            .map(|(result, resource)| {
+                result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
+            })
+            .collect()
+    }
+
+    /// Batched raw namespace write locks for a bulk delete's key set. The
+    /// resource keys use the same disk-identity folding as `new_ns_lock`, so
+    /// aliases (`a\b` vs `a/b`) still share one lock.
+    pub(crate) async fn acquire_object_write_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| ObjectKey {
+                bucket: Arc::from(bucket),
+                object: Arc::from(object_name_disk_identity(object).as_ref()),
+                version: None,
+            })
+            .collect();
+        self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
+    }
+
+    /// Batched delete-transaction locks for a bulk delete's key set; see
+    /// [`Self::acquire_delete_transaction_lock`] for the resource shape.
+    pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| self.delete_transaction_resource(bucket, object))
+            .collect();
+        let mut guards = Vec::with_capacity(resources.len());
+        for (object, result) in objects.iter().zip(
+            self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
+                .await,
+        ) {
+            match result {
+                Ok(guard) => {
+                    #[cfg(test)]
+                    super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
+                    let _ = object;
+                    guards.push(Ok(Arc::new(guard)));
+                }
+                Err(error) => guards.push(Err(error)),
+            }
+        }
+        guards
+    }
+
     #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
     pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
         match err {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -57,7 +57,6 @@ use super::object::{
     object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
 };
 use crate::api::config::storageclass;
-#[cfg(test)]
 use crate::bucket::metadata_sys::ObjectLockConfigState;
 use crate::bucket::quota::reservation;
 use crate::bucket::replication::ReplicationStatusType;
@@ -4114,6 +4113,40 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         )
         .await?;
 
+        if !opts.versioned
+            && !crate::bucket::utils::is_meta_bucketname(bucket)
+            && let Some(snapshot) = opts.object_lock_config_snapshot.as_deref()
+            && !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)
+        {
+            // Completion replaces the null version, not necessarily the latest
+            // version inspected by the request-level Object Lock check.
+            let current = self
+                .get_object_info(
+                    bucket,
+                    object,
+                    &ObjectOptions {
+                        version_id: Some(Uuid::nil().to_string()),
+                        no_lock: true,
+                        metadata_cache_safe: false,
+                        versioned: opts.versioned,
+                        version_suspended: opts.version_suspended,
+                        ..Default::default()
+                    },
+                )
+                .await;
+            match current {
+                Ok(existing) => {
+                    if check_object_lock_for_deletion_with_state(snapshot.state(), &existing, false)?.is_some() {
+                        return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
+                    }
+                }
+                // A null delete marker has no protected payload to replace.
+                Err(StorageError::MethodNotAllowed) => {}
+                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
+                Err(err) => return Err(err),
+            }
+        }
+
         if opts.data_movement
             && opts.http_preconditions.is_some()
             && let Some(err) = self.check_write_precondition(bucket, object, opts).await
@@ -10060,7 +10093,7 @@ mod tests {
 
         // Real disk callbacks and health probes must share wall-clock deadlines.
         // Advancing Tokio time can expire unrelated healthy-drive probes.
-        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
             let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
             let bucket = "mpu-quorum-success-physical";
@@ -10087,6 +10120,8 @@ mod tests {
                 local.observe_multipart_cleanup_for_test(&upload_path)
             }).collect::<Vec<_>>();
             let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
+            // Fault only the held leg; healthy disks keep their normal I/O budgets.
+            let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
             let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
             let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
             let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10243,7 +10278,15 @@ mod tests {
                 let size = copy_info.get_actual_size().expect("actual source size");
                 copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
                 tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
-                    .await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
+                    .await.expect("cross-key Copy cannot wait on the old successful logical owner")
+                    .unwrap_or_else(|error| {
+                        let health = disks.iter().enumerate().map(|(slot, disk)| {
+                            let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
+                            let metrics = local.metrics_snapshot();
+                            (slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
+                        }).collect::<Vec<_>>();
+                        panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
+                    });
                 drop(copy_info);
                 let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
                 let mut actual = Vec::new();

crates/ecstore/src/set_disk/ops/object.rs

@@ -4958,7 +4958,20 @@ impl SetDisks {
                 );
             }
 
-            if let Some(version_id) = opts.version_id.as_deref()
+            // A write without a version id to a key that is not versioning-Enabled
+            // replaces the key's null version in place, so the WORM gate must judge
+            // that version too. The request-layer pre-check reads the latest
+            // version, which is not the one replaced when a UUID version or a
+            // delete marker sits above a retained null version.
+            let worm_gate_version_id = opts.version_id.clone().or_else(|| {
+                (!opts.versioned
+                    && opts
+                        .object_lock_config_snapshot
+                        .as_ref()
+                        .is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
+                .then(|| Uuid::nil().to_string())
+            });
+            if let Some(version_id) = worm_gate_version_id.as_deref()
                 && !is_meta_bucketname(bucket)
             {
                 let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
@@ -4978,9 +4991,10 @@ impl SetDisks {
                     .await;
                 match current {
                     Ok(existing) => {
-                        let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
+                        let object_lock_config = opts
-                            Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
+                            .object_lock_config_snapshot
-                        })?;
+                            .as_deref()
+                            .ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
                         // The WORM gate protects the locked version from local
                         // overwrites; an authorized replication write passes it
                         // only when the LWW merge below will judge every
@@ -5011,6 +5025,8 @@ impl SetDisks {
                             }
                         }
                     }
+                    // A null delete marker has no protected payload to replace.
+                    Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
                 }
@@ -9304,6 +9320,60 @@ impl SetDisks {
 }
 
 impl SetDisks {
+    /// Judge the version a versionless destructive delete removes.
+    ///
+    /// For a key that is not versioning-`Enabled`, a delete without a version
+    /// id removes the `null` version, while the preceding Object Lock check
+    /// judged the key's latest entry. When a UUID version or delete marker
+    /// sits above a retained `null` version, the two differ and the retained
+    /// version must be judged on its own. The caller holds the object's write
+    /// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
+    /// lookup: their latest entry is the version removed.
+    ///
+    /// The lookup future is boxed from a synchronous wrapper so it leaves the
+    /// delete futures' frames: `delete_object` already nests the boxed batch
+    /// path, and inlining a second `get_object_info` overflowed the 4 MiB test
+    /// stack on Linux.
+    fn check_null_version_object_lock_delete<'a>(
+        &'a self,
+        bucket: &'a str,
+        object: &'a str,
+        latest: &'a ObjectInfo,
+        opts: &'a ObjectOptions,
+    ) -> impl Future<Output = Result<()>> + Send + 'a {
+        Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
+    }
+
+    async fn check_null_version_object_lock_delete_inner(
+        &self,
+        bucket: &str,
+        object: &str,
+        latest: &ObjectInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()> {
+        if is_meta_bucketname(bucket)
+            || opts.version_id.is_some()
+            || set_disk_delete_creates_delete_marker(opts)
+            || opts.transition.expire_restored
+            || latest.version_id.is_none_or(|version| version.is_nil())
+        {
+            return Ok(());
+        }
+        let null_opts = ObjectOptions {
+            version_id: Some(Uuid::nil().to_string()),
+            no_lock: true,
+            metadata_cache_safe: false,
+            ..opts.clone()
+        };
+        match self.get_object_info(bucket, object, &null_opts).await {
+            Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
+            // A null delete marker has no protected payload to remove.
+            Err(StorageError::MethodNotAllowed) => Ok(()),
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
+            Err(err) => Err(err),
+        }
+    }
+
     /// Read current metadata without cache admission or read-side repair. The
     /// outer batch owner already holds the sorted namespace write-lock set.
     pub(crate) async fn conditional_delete_current_snapshot(
@@ -9338,6 +9408,76 @@ where
     futures::future::try_join_all(probes).await.map(|_| ()).map_err(Error::from)
 }
 
+/// Per-key source-read inputs for the bulk-delete admission loop. The bounded
+/// metadata pre-fetch and the loop itself must resolve these identically, so
+/// both sides derive them here.
+struct BatchDeleteSourceInputs {
+    replication_object_name: String,
+    explicit_null_version: bool,
+    version_id: Option<Uuid>,
+    versioned: bool,
+    version_suspended: bool,
+    check_opts: ObjectOptions,
+    object_lock_check_required: bool,
+    replicate_delete: bool,
+    needs_source_read: bool,
+}
+
+fn batch_delete_source_inputs(
+    bucket: &str,
+    dobj: &ObjectToDelete,
+    opts: &ObjectOptions,
+    delete_config_snapshot: &DeleteReplicationConfigSnapshot,
+    object_lock_config: &Arc<ObjectLockConfigSnapshot>,
+    guarded: bool,
+) -> BatchDeleteSourceInputs {
+    let replication_object_name = decode_dir_object(&dobj.object_name);
+    let explicit_null_version = is_explicit_null_version(dobj.version_id);
+    let version_id = delete_file_info_version_id(dobj.version_id);
+    let (versioned, version_suspended) = delete_config_snapshot
+        .versioning_config()
+        .delete_state(replication_object_name.as_str());
+    let check_opts = ObjectOptions {
+        version_id: dobj.version_id.map(|version_id| version_id.to_string()),
+        versioned,
+        version_suspended,
+        object_lock_delete: opts.object_lock_delete.clone(),
+        object_lock_config_snapshot: Some(Arc::clone(object_lock_config)),
+        no_lock: true,
+        suppress_read_repair: guarded,
+        ..Default::default()
+    };
+    // A missing bucket Object Lock configuration does not prove that
+    // persisted object metadata is clean: migrated or corrupt data may
+    // still carry an explicit retention or legal hold. Only a real
+    // delete-marker creation can skip the object-level WORM check.
+    let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
+    let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
+    let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
+    let replication_needs_source = replicate_delete
+        && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
+    // Guarded explicit-version deletes are rejected before any read and must
+    // not join the pre-fetch wave.
+    let admission_error = guarded && dobj.version_id.is_some();
+    let needs_source_read = !admission_error
+        && (object_lock_check_required
+            || replication_needs_source
+            || opts.tier_delete_journal_api.is_some()
+            || dobj.expected_identity.is_some()
+            || dobj.version_id.is_some());
+    BatchDeleteSourceInputs {
+        replication_object_name,
+        explicit_null_version,
+        version_id,
+        versioned,
+        version_suspended,
+        check_opts,
+        object_lock_check_required,
+        replicate_delete,
+        needs_source_read,
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
     type Error = Error;
@@ -9736,7 +9876,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let unique_lock_count = batch.requests.len();
 
         let mut failed_map = HashMap::new();
-        let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
+        let mut local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
         let mut locked_objects = HashSet::new();
 
         // Instance-scoped, not the ambient facade (backlog#1052) — see
@@ -9753,7 +9893,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             (failed_map, locked_objects, dist_batch_lock_ids) = self.acquire_dist_delete_object_locks_batch(&batch).await;
         } else {
             let batch_result = self.local_lock_manager.acquire_locks_batch(batch).await;
-            _local_batch_guards = batch_result.guards;
+            local_batch_guards = batch_result.guards;
 
             for key in batch_result.successful_locks {
                 locked_objects.insert(key.object.as_ref().to_string());
@@ -9795,8 +9935,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // Every raw batch lock is acquired first. Canonical sorting also
             // gives aliases of one disk key exactly one transaction owner.
             let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
-            for key in keys {
+            // Bulk delete is the only multi-key transaction-lock holder; one
-                match self.acquire_delete_transaction_lock(bucket, &key).await {
+            // batched lock round replaces one quorum round-trip per key and is
+            // deadlock-free by construction (a single `lock_batch` RPC per lock
+            // client, global (shard, key) order at each server). Per-key
+            // failures keep the set layer's per-key error reporting.
+            let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
+            let acquired = self
+                .acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
+                .await;
+            for (key, result) in keys.into_iter().zip(acquired) {
+                match result {
                     Ok(guard) => {
                         opts.add_namespace_lock_guard(&guard);
                         transaction_guards.insert(key, guard);
@@ -9832,6 +9981,54 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 },
             }
         };
+        // Per-key read inputs are resolved exactly once; the pre-fetch and the
+        // admission loop below consume this table so the two cannot disagree
+        // on eligibility or read options.
+        let mut source_inputs: Vec<Option<BatchDeleteSourceInputs>> = Vec::with_capacity(objects.len());
+        for dobj in &objects {
+            let guarded = current_generations.is_some_and(|generations| generations.contains_key(&dobj.object_name));
+            source_inputs.push(Some(batch_delete_source_inputs(
+                bucket,
+                dobj,
+                &opts,
+                &delete_config_snapshot,
+                &object_lock_config,
+                guarded,
+            )));
+        }
+        // Bounded-parallel pre-fetch of the source metadata each admitted key
+        // needs below. These reads are pure quorum lookups under the locks
+        // already held; sequencing one metadata fan-out per key made a
+        // 1000-key batch pay a full round of drive reads at a time.
+        const DELETE_OBJECTS_SOURCE_READ_CONCURRENCY: usize = 16;
+        let mut source_reads: HashMap<usize, (ObjectInfo, FileInfo, Option<Error>)> = HashMap::with_capacity(objects.len());
+        {
+            let mut pending = Vec::new();
+            for (i, source) in source_inputs.iter().enumerate() {
+                if del_errs[i].is_some() {
+                    continue;
+                }
+                let Some(source) = source else {
+                    continue;
+                };
+                if !source.needs_source_read {
+                    continue;
+                }
+                let object_name = objects[i].object_name.clone();
+                let check_opts = &source.check_opts;
+                pending.push(async move {
+                    let (goi, file_info, _write_quorum, gerr) = self
+                        .get_object_info_fileinfo_and_quorum(bucket, &object_name, check_opts)
+                        .await;
+                    (i, (goi, file_info, gerr))
+                });
+            }
+            let reads = futures::stream::iter(pending)
+                .buffer_unordered(DELETE_OBJECTS_SOURCE_READ_CONCURRENCY)
+                .collect::<Vec<_>>()
+                .await;
+            source_reads.extend(reads);
+        }
         let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
         let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
         let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
@@ -9853,41 +10050,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 ));
                 continue;
             }
-            let replication_object_name = decode_dir_object(&dobj.object_name);
+            let Some(source) = source_inputs[i].take() else {
-            let explicit_null_version = is_explicit_null_version(dobj.version_id);
+                continue;
-            let version_id = delete_file_info_version_id(dobj.version_id);
+            };
-            let (versioned, version_suspended) = delete_config_snapshot
+            let BatchDeleteSourceInputs {
-                .versioning_config()
+                replication_object_name,
-                .delete_state(replication_object_name.as_str());
+                explicit_null_version,
-            let check_opts = ObjectOptions {
+                version_id,
-                version_id: dobj.version_id.map(|version_id| version_id.to_string()),
                 versioned,
                 version_suspended,
-                object_lock_delete: opts.object_lock_delete.clone(),
+                check_opts,
-                object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)),
+                object_lock_check_required,
-                no_lock: true,
+                replicate_delete,
-                suppress_read_repair: guarded,
+                needs_source_read,
-                ..Default::default()
+            } = source;
-            };
+            let (mut goi, mut authoritative_file_info, mut gerr) = if needs_source_read {
-            // A missing bucket Object Lock configuration does not prove that
+                match source_reads.remove(&i) {
-            // persisted object metadata is clean: migrated or corrupt data may
+                    Some(read) => read,
-            // still carry an explicit retention or legal hold. Only a real
+                    None => {
-            // delete-marker creation can skip the object-level WORM check.
-            let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
-            let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
-            let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
-            let replication_needs_source = replicate_delete
-                && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
-            let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required
-                || replication_needs_source
-                || opts.tier_delete_journal_api.is_some()
-                || dobj.expected_identity.is_some()
-                || dobj.version_id.is_some()
-            {
                         let (goi, file_info, _write_quorum, gerr) = self
                             .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
                             .await;
                         (goi, file_info, gerr)
+                    }
+                }
             } else {
                 (ObjectInfo::default(), FileInfo::default(), None)
             };
@@ -9904,6 +10090,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 && !guarded
                 && goi.name.is_empty()
             {
+                // Recovery can restore or retire versions of this key
+                // (rolling back an interrupted prepared delete brings a
+                // retained version back). A pre-fetched read for a later
+                // duplicate of the same name would still report the stale
+                // miss and skip its object-lock check — bypassing retention —
+                // so evict those entries and let the loop re-read them live.
+                for (other_index, other) in objects.iter().enumerate() {
+                    if other_index != i && other.object_name == dobj.object_name {
+                        source_reads.remove(&other_index);
+                    }
+                }
                 let recovery = self
                     .recover_delete_key(
                         bucket,
@@ -10022,6 +10219,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 del_errs[i] = Some(err);
                 continue;
             }
+            // Not gated on `source_missing`: with a delete marker on top the
+            // latest-version lookup reports the key as missing while the null
+            // version below it may still be retained.
+            if object_lock_check_required
+                && let Err(err) = self
+                    .check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
+                    .await
+            {
+                del_errs[i] = Some(err);
+                continue;
+            }
 
             if opts.tier_delete_journal_api.is_some()
                 && !source_missing
@@ -10477,6 +10685,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // response can hide a persisted decision and must never roll back.
             let groups = groups.collect::<Vec<_>>();
             published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
+            // Journal publish fans out per object per drive; keep enough
+            // publications in flight to feed the drives' durable-mutation
+            // lanes (8 workers each) across a 16-drive set.
+            const DELETE_JOURNAL_PUBLISH_CONCURRENCY: usize = 32;
             let publications = futures::stream::iter(groups.into_iter().map(|index: usize| {
                 let group = &vers[index];
                 let recovery_ticket = recovery_tickets[index].clone();
@@ -10512,7 +10724,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     (index, result)
                 }
             }))
-            .buffer_unordered(16)
+            .buffer_unordered(DELETE_JOURNAL_PUBLISH_CONCURRENCY)
             .collect::<Vec<_>>()
             .await;
             for (index, result) in publications {
@@ -10652,9 +10864,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         // Deferred recovery takes fresh raw and transaction locks. Discard the
         // completed leases' deadlines while retaining every caller-owned fence.
         opts.namespace_lock_fence = outer_namespace_lock_fence;
-        // Release inner owners before reacquiring any raw key lock.
+        // Release inner owners before reacquiring any raw key lock. The raw
+        // batch guards must go too: deferred recovery takes a fresh write lock
+        // on the same keys below, and a still-held local batch guard would
+        // deadlock it against itself (the distributed path releases its batch
+        // ids in the same region).
         drop(namespace_owner);
         drop(transaction_guards);
+        drop(local_batch_guards);
 
         // TODO(backlog): support partial object deletion for multi-part objects
 
@@ -11094,6 +11311,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             opts.precondition_check(&goi)?;
             check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
         }
+        self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
+            .await?;
         let delete_marker_purge_candidate = opts
             .version_id
             .as_deref()
@@ -24992,6 +25211,131 @@ mod delete_objects_lock_gating_tests {
             .expect("plain object should be written");
     }
 
+    /// Regression for the PR9155 review finding: a pre-fetched source-missing
+    /// read of a later duplicate must not skip the object-lock check after an
+    /// earlier entry's recovery rolls back an interrupted prepared delete and
+    /// restores a GOVERNANCE-retained version. Without the eviction the later
+    /// duplicate deletes the restored version, bypassing retention.
+    #[tokio::test]
+    async fn duplicate_entries_recheck_object_lock_after_recovery_restores_retained_version() {
+        let (_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "recovery-retention-dup";
+        let object = "retained.bin";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+
+        // A versioned put, so the batch can address the version explicitly.
+        let mut reader = PutObjReader::from_vec(vec![7u8; 256]);
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("versioned object should be written");
+        let version = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read back the version identity")
+            .version_id
+            .expect("versioned put assigns a version id");
+
+        // GOVERNANCE retention on that version, persisted the way
+        // PutObjectRetention persists it.
+        let retain_until = OffsetDateTime::now_utc() + time::Duration::days(60);
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    version_id: Some(version.to_string()),
+                    eval_metadata: Some(HashMap::from([
+                        (metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
+                        (
+                            metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
+                            retain_until
+                                .format(&time::format_description::well_known::Rfc3339)
+                                .expect("retention date formats"),
+                        ),
+                    ])),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("governance retention should persist");
+
+        // Interrupted authorized delete: the recoverable per-disk delete
+        // applies the version removal and leaves the prepared journal
+        // undecided — exactly a crash between apply and publish.
+        let transaction = Uuid::new_v4();
+        for disk in &disk_stores {
+            let fi = disk
+                .read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("version exists before the interrupted delete");
+            let errors = disk
+                .delete_versions_recoverable(
+                    bucket,
+                    vec![FileInfoVersions {
+                        name: object.to_string(),
+                        versions: vec![fi],
+                        ..Default::default()
+                    }],
+                    DeleteOptions {
+                        recoverable_delete: true,
+                        old_data_dir: Some(transaction),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(errors.iter().all(Option::is_none), "the plant must apply: {errors:?}");
+        }
+
+        // A no-bypass batch naming the retained version twice: the first
+        // entry rolls the prepared delete back (restoring the retained
+        // version) and must be denied; the second must re-read and stay
+        // denied too instead of reusing the stale pre-fetched miss.
+        let victim = ObjectToDelete {
+            object_name: object.to_string(),
+            version_id: Some(version),
+            ..Default::default()
+        };
+        let (_deleted, errors) = set_disks
+            .delete_objects(
+                bucket,
+                vec![victim.clone(), victim],
+                ObjectOptions {
+                    versioned: true,
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            errors.iter().all(|error| error
+                .as_ref()
+                .is_some_and(|error| matches!(error, StorageError::PrefixAccessDenied(_, _)))),
+            "both duplicates must be denied by the restored retention: {errors:?}"
+        );
+
+        // The retained version survives the batch.
+        for disk in &disk_stores {
+            disk.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("the retained version must survive the batch");
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial(shard_integrity_rollout)]
     async fn conditional_delete_snapshot_preserves_multipart_etag() {

crates/ecstore/src/store/heal.rs

@@ -3692,6 +3692,177 @@ mod tests {
         );
     }
 
+    // The RPC lock service uses the raw ObjectKey and one backend per node,
+    // even when the requesting sets carry different pool namespace strings.
+    async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
+        use rustfs_lock::client::{LockClient, local::LocalClient};
+
+        let clients = || {
+            (0..2)
+                .map(|_| {
+                    Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
+                })
+                .collect::<Vec<_>>()
+        };
+        let first = clients();
+        let second = if shared { first.clone() } else { clients() };
+        let mut pools = source.pools.clone();
+        for (pool, lockers) in pools.iter_mut().zip([first, second]) {
+            for set in &mut Arc::make_mut(pool).disk_set {
+                Arc::make_mut(set).lockers = lockers.clone();
+            }
+        }
+        let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
+        let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
+        Arc::new(ECStore {
+            id: Uuid::new_v4(),
+            disk_map: Default::default(),
+            pools,
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
+            pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
+            rebalance_meta: tokio::sync::RwLock::new(None),
+            decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(state),
+            ctx: Arc::clone(&source.ctx),
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
+        let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let first_lock = first
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("first pool lock");
+        let second_lock = second
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("second pool lock");
+        let guard = first_lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("first writer");
+        assert!(
+            second_lock
+                .get_write_lock(std::time::Duration::from_millis(150))
+                .await
+                .is_err(),
+            "different pool namespaces still conflict on the same backend and object key"
+        );
+        drop(guard);
+        second_lock
+            .get_write_lock(std::time::Duration::from_secs(3))
+            .await
+            .expect("second writer should acquire after the first releases");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    dry_run: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
+        .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let missing = remove_pool_meta_shard(&store, 1).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(10),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("cross-pool repair must not reenter the shared writer")
+        .expect("pool metadata repair should return a result");
+        assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+        assert!(
+            missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "the selected missing shard must be restored"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
+        let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let lock = target
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("target lock");
+        let guard = lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("hold target lock");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    POOL_META_NAME,
+                    "",
+                    &HealOpts {
+                        pool: Some(1),
+                        dry_run: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            barrier.wait_until_external_heal_target_lock_attempted(),
+        )
+        .await
+        .expect("inspection must attempt the independent target lock");
+        assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
+        drop(guard);
+        let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
+            .await
+            .expect("inspection should progress after target lock release")
+            .expect("heal task should not panic")
+            .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {

crates/ecstore/src/store/init.rs

@@ -27352,4 +27352,282 @@ mod tests {
             .expect("recovery must preserve the local source object");
         assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
     }
+
+    #[cfg(feature = "test-util")]
+    async fn seed_retained_null_version(
+        store: &Arc<crate::store::ECStore>,
+        bucket: &str,
+        object: &str,
+        protection: &str,
+        latest_is_marker: bool,
+    ) {
+        let metadata = if protection == "legal-hold" {
+            HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
+        } else {
+            HashMap::from([
+                ("x-amz-object-lock-mode".to_string(), protection.to_string()),
+                ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
+            ])
+        };
+        let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut original,
+                &ObjectOptions {
+                    user_defined: metadata,
+                    mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed retained null version");
+        let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut latest,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed newer unprotected UUID version");
+        if latest_is_marker {
+            store
+                .delete_object(
+                    bucket,
+                    object,
+                    ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create a UUID delete marker above the retained null version");
+        }
+    }
+
+    #[cfg(feature = "test-util")]
+    async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
+        let mut reader = store
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(Uuid::nil().to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("retained null version must remain readable");
+        let mut body = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut body)
+            .await
+            .expect("read retained null version");
+        assert_eq!(body, b"protected null version");
+    }
+
+    /// A versionless delete removes the null version, so the Object Lock check
+    /// must judge that version even when a UUID version or delete marker is
+    /// the key's latest entry (rustfs/backlog#3017).
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_delete_checks_actual_target() {
+        use crate::bucket::replication::DeleteReplicationConfigSnapshot;
+        use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
+
+        let temp = tempfile::tempdir().expect("create null-version delete store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        let configs = [
+            VersioningConfiguration::default(),
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+                ..Default::default()
+            },
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                excluded_prefixes: Some(vec![ExcludedPrefix {
+                    prefix: Some("archive/".to_string()),
+                }]),
+                ..Default::default()
+            },
+        ];
+        for (case, config) in configs.into_iter().enumerate() {
+            let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
+            for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+                for latest_is_marker in [false, true] {
+                    let object = format!("archive/{case}-{protection}-{latest_is_marker}");
+                    seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                    let (_, errors) = store
+                        .delete_objects(
+                            &bucket,
+                            vec![ObjectToDelete {
+                                object_name: object.clone(),
+                                ..Default::default()
+                            }],
+                            ObjectOptions {
+                                delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                ..Default::default()
+                            },
+                        )
+                        .await;
+                    assert!(
+                        matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
+                        "batch delete must judge the retained null version: {object}: {errors:?}"
+                    );
+                    let error = store
+                        .delete_object(
+                            &bucket,
+                            &object,
+                            ObjectOptions {
+                                version_suspended: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect_err("suspended single delete must also judge the retained null version");
+                    assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                    assert_retained_null_version_body(&store, &bucket, &object).await;
+                    if protection == "GOVERNANCE" {
+                        let (_, errors) = store
+                            .delete_objects(
+                                &bucket,
+                                vec![ObjectToDelete {
+                                    object_name: object.clone(),
+                                    ..Default::default()
+                                }],
+                                ObjectOptions {
+                                    delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                    object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
+                                        bypass_governance: true,
+                                    }),
+                                    ..Default::default()
+                                },
+                            )
+                            .await;
+                        assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
+                        assert!(
+                            store
+                                .get_object_reader(
+                                    &bucket,
+                                    &object,
+                                    None,
+                                    HeaderMap::new(),
+                                    &ObjectOptions {
+                                        version_id: Some(Uuid::nil().to_string()),
+                                        ..Default::default()
+                                    }
+                                )
+                                .await
+                                .is_err(),
+                            "authorized deletion must make the retained null payload unreadable"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    /// A versionless PUT or multipart completion replaces the null version in
+    /// place, so the commit-time WORM gate must judge that version even when a
+    /// UUID version or delete marker is the key's latest entry
+    /// (rustfs/backlog#3019).
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
+        let temp = tempfile::tempdir().expect("create null-version overwrite store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+            for latest_is_marker in [false, true] {
+                let object = format!("{protection}-{latest_is_marker}");
+                seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
+                let error = store
+                    .put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
+                    .await
+                    .expect_err("PUT must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+
+                let upload = store
+                    .clone()
+                    .new_multipart_upload(&bucket, &object, &ObjectOptions::default())
+                    .await
+                    .expect("initiate multipart upload");
+                let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
+                let part = store
+                    .clone()
+                    .put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
+                    .await
+                    .expect("stage multipart replacement");
+                let error = store
+                    .clone()
+                    .complete_multipart_upload(
+                        &bucket,
+                        &object,
+                        &upload.upload_id,
+                        vec![crate::storage_api_contracts::multipart::CompletePart {
+                            part_num: part.part_num,
+                            etag: part.etag,
+                            ..Default::default()
+                        }],
+                        &ObjectOptions::default(),
+                    )
+                    .await
+                    .expect_err("multipart completion must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+
+                let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
+                store
+                    .put_object(
+                        &bucket,
+                        &object,
+                        &mut next_version,
+                        &ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("a new UUID version must remain allowed");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+            }
+        }
+    }
 }

crates/ecstore/src/store/init/multipart_copy_heal_tests.rs

@@ -318,10 +318,7 @@ mod multipart_copy_heal_tests {
             let mut corrupt = original.clone();
             corrupt[0] ^= 0x80;
             std::fs::write(&shard, &corrupt).expect("actual bitrot corruption in shared physical shard");
-            // Windows only flushes a handle opened for writing.
+            std::fs::File::open(&shard)
-            std::fs::OpenOptions::new()
-                .write(true)
-                .open(&shard)
                 .expect("open corrupted actual shard")
                 .sync_all()
                 .expect("persist actual fault bytes");

crates/ecstore/src/store/multipart.rs

@@ -1210,7 +1210,34 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<ObjectInfo> {
         check_complete_multipart_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
+            // The set-level commit judges the null version this completion
+            // replaces; give it the same authoritative snapshot a PUT carries.
+            let expected_incarnation_id = opts
+                .expected_bucket_incarnation_id
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
+            let lifecycle_fence = opts
+                .bucket_lifecycle_lock_fence
+                .as_ref()
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
+            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
+                Some(snapshot) => Arc::clone(snapshot),
+                None => {
+                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
+                        .await?
+                }
+            };
+            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
+                return Err(Error::other(
+                    "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
+                ));
+            }
+            snapshot.add_lock_fences(&mut opts);
+            opts.object_lock_config_snapshot = Some(snapshot);
+        }
         if !self.single_pool() {
             opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
         }

crates/ecstore/src/store/object.rs

@@ -1352,8 +1352,8 @@ impl RestoreWorkerGuard {
     }
 }
 
-impl Drop for ObjectLockDiagGuard {
+impl ObjectLockDiagGuard {
-    fn drop(&mut self) {
+    fn record_hold_diag(&self) {
         if !self.enabled || self.guard.as_ref().is_released() {
             return;
         }
@@ -1378,6 +1378,12 @@ impl Drop for ObjectLockDiagGuard {
     }
 }
 
+impl Drop for ObjectLockDiagGuard {
+    fn drop(&mut self) {
+        self.record_hold_diag();
+    }
+}
+
 /// A failure to preserve one object generation for a SelectObjectContent read.
 #[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
@@ -3467,22 +3473,53 @@ impl ECStore {
         }
     }
 
+    /// Acquire the per-object write locks a bulk delete needs before any
+    /// mutation, one batched lock round for the whole key set. Bulk delete is
+    /// the only multi-key write-lock holder, and the batched round is
+    /// deadlock-free by construction (one `lock_batch` RPC per lock client,
+    /// global key order at each server), so no optimistic wave or serial
+    /// fallback is needed. Outcomes are per key: a locked-out key fails alone
+    /// and the rest of the batch proceeds.
     async fn acquire_delete_objects_write_locks(
         &self,
         bucket: &str,
         objects: &[ObjectToDelete],
         opts: &mut ObjectOptions,
-    ) -> Result<Vec<ObjectLockDiagGuard>> {
+    ) -> Result<(Vec<ObjectLockDiagGuard>, HashMap<String, StorageError>)> {
         if opts.no_lock || objects.is_empty() {
-            return Ok(Vec::new());
+            return Ok((Vec::new(), HashMap::new()));
         }
 
         let object_names = sorted_unique_delete_object_names(objects);
-        // Lock order: encoded object names are acquired in ascending order, then
+        let acquired = self.pools[0].disk_set[0]
-        // the set-layer calls receive no_lock so they do not reacquire them.
+            .acquire_object_write_locks_batch(bucket, &object_names, get_lock_acquire_timeout())
+            .await;
+
+        let diag_enabled = is_object_lock_diag_enabled();
         let mut guards = Vec::with_capacity(object_names.len());
-        for object in object_names {
+        let mut failures = HashMap::new();
-            guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
+        for (object, result) in object_names.iter().zip(acquired) {
+            match result {
+                Ok(guard) => {
+                    #[cfg(any(test, feature = "test-util"))]
+                    notify_delete_namespace_pending(bucket);
+                    guards.push(
+                        ObjectLockDiagGuard::new(
+                            guard,
+                            diag_enabled,
+                            "delete_objects",
+                            diag_enabled.then(|| bucket.to_string()),
+                            diag_enabled.then(|| (*object).to_string()),
+                            diag_enabled.then(|| self.pools[0].disk_set[0].locker_owner.clone()),
+                            ObjectLockDiagMode::Write,
+                        )
+                        .with_write_key(bucket, object),
+                    );
+                }
+                Err(error) => {
+                    failures.insert((*object).to_owned(), error);
+                }
+            }
         }
         opts.no_lock = true;
         for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
@@ -3490,7 +3527,7 @@ impl ECStore {
         }
         opts.ensure_namespace_lock_fence();
 
-        Ok(guards)
+        Ok((guards, failures))
     }
 
     pub(super) async fn acquire_object_read_lock_if_needed(
@@ -6165,10 +6202,21 @@ impl ECStore {
         if current_bucket_incarnation_id.is_some() {
             pause_delete_after_object_lock_snapshot(bucket).await;
         }
-        let _object_lock_guards = Arc::new(match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
+        let (object_lock_guards, object_lock_failures) =
-            Ok(guards) => guards,
+            match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
+                Ok(result) => result,
                 Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
-        });
+            };
+        // Lock outcomes are per key: a locked-out key fails alone and the rest
+        // of the batch proceeds, matching the set layer's per-key lock errors.
+        if !object_lock_failures.is_empty() {
+            for (index, object) in objects.iter().enumerate() {
+                if let Some(error) = object_lock_failures.get(&object.object_name) {
+                    del_errs[index] = Some(error.clone());
+                }
+            }
+        }
+        let _object_lock_guards = Arc::new(object_lock_guards);
         opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), _object_lock_guards.clone())));
         #[cfg(any(test, feature = "test-util"))]
         if !_object_lock_guards.is_empty() {
@@ -6288,8 +6336,11 @@ impl ECStore {
         inject_conditional_delete_post_admission_mutations(self, bucket).await;
 
         if !has_conditions {
-            let indices = (0..objects.len()).collect::<Vec<_>>();
+            // Keys whose write lock failed are already reported in `del_errs`;
-            (del_objects, del_errs, accounting) = self
+            // the phase only runs for the keys this batch actually holds, and
+            // only its outcomes replace the outer per-key state.
+            let indices = (0..objects.len()).filter(|idx| del_errs[*idx].is_none()).collect::<Vec<_>>();
+            let (phase_deleted, mut phase_errors, mut phase_accounting) = self
                 .run_batch_delete_phase(
                     bucket,
                     &objects,
@@ -6301,6 +6352,11 @@ impl ECStore {
                     },
                 )
                 .await;
+            for idx in indices {
+                del_objects[idx] = phase_deleted[idx].clone();
+                del_errs[idx] = phase_errors[idx].take();
+                accounting[idx] = phase_accounting[idx].take();
+            }
         } else {
             let mut conditional_actions = HashMap::with_capacity(objects.len());
             let mut conditional_indices = Vec::with_capacity(objects.len());
@@ -12944,11 +13000,12 @@ mod tests {
         ];
         let mut opts = ObjectOptions::default();
 
-        let guards = store
+        let (guards, failures) = store
             .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
             .await
             .expect("delete object locks should be acquired");
 
+        assert!(failures.is_empty(), "uncontended keys must all lock: {failures:?}");
         assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
         assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
         assert!(
@@ -13580,3 +13637,129 @@ mod missing_key_delete_quorum_tests {
         .await;
     }
 }
+
+/// The bulk-delete lock rounds are batched and per key: one `lock_batch` wave
+/// per lock layer replaces one quorum round-trip per key, a locked-out key
+/// fails alone, and the rest of the batch proceeds (rustfs/backlog#3146).
+#[cfg(test)]
+mod batch_delete_lock_tests {
+    use super::*;
+    use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
+
+    async fn seeded_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("fixture bucket");
+        for key in ["a-object", "b-object"] {
+            let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
+            store
+                .put_object(
+                    bucket,
+                    key,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("seed object");
+        }
+        (dirs, store)
+    }
+
+    fn two_key_batch() -> Vec<ObjectToDelete> {
+        ["b-object", "a-object"]
+            .into_iter()
+            .map(|object_name| ObjectToDelete {
+                object_name: object_name.to_owned(),
+                ..Default::default()
+            })
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn uncontended_batch_deletes_every_key() {
+        let (_dirs, store) = seeded_store("lock-batch-success").await;
+
+        let (deleted, errors, _) = store
+            .delete_objects_with_tier_delete_journal_and_accounting(
+                "lock-batch-success",
+                two_key_batch(),
+                ObjectOptions::default(),
+            )
+            .await;
+        assert!(errors.iter().all(Option::is_none), "batch outcome: {errors:?}");
+        assert_eq!(deleted.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn contended_key_fails_alone_and_the_rest_of_the_batch_proceeds() {
+        let (_dirs, store) = seeded_store("lock-batch-partial").await;
+
+        // Hold one key's write lock: the batched round must fail that key
+        // alone (per-key outcome) while the other key still deletes.
+        let held = store
+            .acquire_object_write_lock("delete_objects", "lock-batch-partial", "a-object")
+            .await
+            .expect("hold contended key");
+        let (deleted, errors, _) = tokio::time::timeout(
+            Duration::from_secs(30),
+            store.delete_objects_with_tier_delete_journal_and_accounting(
+                "lock-batch-partial",
+                two_key_batch(),
+                ObjectOptions::default(),
+            ),
+        )
+        .await
+        .expect("per-key lock outcomes must not block the whole batch");
+
+        assert!(errors[1].is_some(), "contended a-object must fail alone: {errors:?}");
+        assert!(errors[0].is_none(), "b-object must proceed: {errors:?}");
+        assert!(deleted[1].object_name.is_empty(), "failed key must not report success");
+        assert_eq!(deleted[0].object_name, "b-object");
+        drop(held);
+    }
+
+    #[tokio::test]
+    async fn overlapping_batches_reach_per_key_order_without_deadlock() {
+        let (_dirs, store) = seeded_store("lock-batch-overlap").await;
+
+        let first = store.clone();
+        let second = store.clone();
+        let run = |store: Arc<ECStore>| {
+            tokio::spawn(async move {
+                store
+                    .delete_objects_with_tier_delete_journal_and_accounting(
+                        "lock-batch-overlap",
+                        two_key_batch(),
+                        ObjectOptions::default(),
+                    )
+                    .await
+            })
+        };
+        let first = run(first);
+        let second = run(second);
+
+        // Whatever mix of per-key lock outcomes the overlap produces, both
+        // batches must finish: the batched round is deadlock-free by
+        // construction (global key order at each lock server).
+        for batch in [first, second] {
+            let (_, errors, _) = tokio::time::timeout(Duration::from_secs(30), batch)
+                .await
+                .expect("overlapping batches must not deadlock")
+                .expect("delete task");
+            for error in errors.iter().flatten() {
+                let message = error.to_string();
+                assert!(
+                    message.contains("lock") || message.contains("Lock") || message.contains("namespace"),
+                    "overlap may only surface lock outcomes: {message}"
+                );
+            }
+        }
+    }
+}

crates/filemeta/src/metacache.rs

@@ -925,6 +925,13 @@ impl MetaCacheEntriesSorted {
 
 const METACACHE_STREAM_VERSION: u8 = 2;
 const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
+/// Upper bound on one incremental reservation while accumulating a single
+/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
+/// up to this bound so a legitimately large xl.meta does not pay one small
+/// read per chunk, while a corrupt length prefix still cannot force a
+/// reservation larger than the payload that has already arrived (see
+/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
+const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
 /// msgpack `false`: the entry-stream terminator, and the only byte a legacy
 /// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
 const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1157,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         // A whole xl.meta can legitimately exceed the limit on its individual
         // msgpack values. Grow only as payload arrives, so a corrupt length
         // prefix cannot cause a speculative allocation of the advertised size.
+        let mut chunk = METACACHE_READ_CHUNK_SIZE;
         while self.buf.len() < read_size {
             let start = self.buf.len();
             let extra = if lookahead {
                 let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
-                (read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).clamp(min_chunk, chunk)
             } else {
-                (read_size - start).min(METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).min(chunk)
             };
             if let Err(e) = self.buf.try_reserve(extra) {
                 let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1194,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
                     return Err(err);
                 }
             }
+            chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
         }
 
         self.pos = read_size;
@@ -2155,6 +2164,76 @@ mod tests {
         }
     }
 
+    /// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
+    /// decode completely even when the aggregate stream exceeds the per-value
+    /// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
+    /// element.
+    #[tokio::test]
+    async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
+        // ~2.9 KiB of metadata per entry matches the per-entry cost observed in
+        // the #9120 production report; 6,000 entries push the stream past 16 MiB.
+        const ENTRIES: usize = 6_000;
+        let metadata = vec![0xab; 2_880];
+        let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
+            .map(|i| MetaCacheEntry {
+                name: format!("bigdir/object-{i:05}"),
+                metadata: metadata.clone(),
+                ..Default::default()
+            })
+            .collect();
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("wide listing must encode");
+        writer.close().await.expect("stream must close");
+        let bytes = output.into_inner();
+        assert!(
+            bytes.len() > 16 << 20,
+            "fixture must exceed the per-value limit, got {} bytes",
+            bytes.len()
+        );
+        let mut reader = MetacacheReader::new(Cursor::new(bytes));
+        let decoded = reader.read_all().await.expect("wide listing must decode");
+        assert_eq!(decoded, entries);
+    }
+
+    /// The #9120 failure shape seen in production: one entry whose metadata is
+    /// an xl.meta that grew past 16 MiB with its version history, listed among
+    /// ordinary neighbors and skipped past without losing the framing.
+    #[tokio::test]
+    async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
+        let small = |i: u8| MetaCacheEntry {
+            name: format!("bigdir/object-{i}"),
+            metadata: vec![i; 64],
+            ..Default::default()
+        };
+        let oversized = MetaCacheEntry {
+            name: "bigdir/huge-versions".to_string(),
+            metadata: vec![0xcd; (16 << 20) + 12_345],
+            ..Default::default()
+        };
+        let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("mixed listing must encode");
+        writer.close().await.expect("stream must close");
+
+        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
+        assert_eq!(
+            reader.peek().await.expect("first entry").expect("present"),
+            entries[0],
+            "an ordinary entry before the oversized one must decode"
+        );
+        reader.skip(1).await.expect("skip the first entry");
+        assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
+        assert_eq!(
+            reader.peek().await.expect("oversized entry").expect("present"),
+            oversized,
+            "an entry past the per-value limit must round-trip whole"
+        );
+        reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
+        assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
+    }
+
     #[test]
     fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
         let version_id = Uuid::new_v4();

crates/heal/src/error.rs

@@ -128,6 +128,20 @@ impl Error {
         Error::TransientSkip { message: message.into() }
     }
 
+    /// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
+    /// Callers must cut their own concurrency instead of retrying at full speed.
+    pub(crate) fn is_slow_down(&self) -> bool {
+        match self {
+            Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
+            Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
+            Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
+                message.contains("Please reduce your request rate")
+            }
+            Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
+            _ => false,
+        }
+    }
+
     /// Whether a heal operation can be retried without changing its inputs.
     pub(crate) fn is_recoverable_heal(&self) -> bool {
         match classify_heal_error(self) {
@@ -449,6 +463,22 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
+    #[test]
+    fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
+        assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
+        assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
+        assert!(
+            Error::TaskExecutionFailed {
+                message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
+            }
+            .is_slow_down()
+        );
+        assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
+        assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
+    }
+
     #[test]
     fn pool_metadata_failures_keep_their_typed_recovery_classification() {
         use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};

crates/heal/src/heal/erasure_healer.rs

@@ -14,6 +14,7 @@
 
 use crate::error::{HealErrorClass, classify_heal_error};
 use crate::heal::{
+    pacing::HealPageRate,
     progress::{HealProgress, add_bytes, increment_counter},
     resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -128,12 +129,12 @@ pub struct ErasureSetHealer {
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
     replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
     work_parent: Option<super::manager::HealWork>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 async fn acquire_page_permit(
     semaphore: Arc<Semaphore>,
-    pacer: Option<&super::pacing::MainlinePacer>,
+    rate: &HealPageRate,
     cancel: &tokio_util::sync::CancellationToken,
 ) -> Result<tokio::sync::OwnedSemaphorePermit> {
     let acquire = || async {
@@ -146,19 +147,17 @@ async fn acquire_page_permit(
     let mut paid_pause = false;
     loop {
         let permit = acquire().await?;
-        if let Some(pacer) = pacer {
         // Keep the real permit on the low-pressure path. Every acquisition
         // gets a fresh decision, including a waiter that queued a second
         // time. One completed pause is a bounded minimum-progress grant.
-            match pacer.admission_decision() {
+        match rate.admission_decision() {
             super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
                 drop(permit);
-                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
+                paid_pause = rate.wait_after_admission(cancel, pressure).await?;
                 continue;
             }
             _ => {}
         }
-        }
         return Ok(permit);
     }
 }
@@ -166,7 +165,7 @@ async fn acquire_page_permit(
 #[cfg(test)]
 mod mainline_pacing_tests {
     use super::*;
-    use crate::heal::pacing::{MainlinePacer, TestPressure};
+    use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
     use tokio_util::sync::CancellationToken;
@@ -180,15 +179,15 @@ mod mainline_pacing_tests {
             .await
             .expect("existing object owns capacity");
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
-        let pacer = Arc::new(
+        let pacer =
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
-        );
+        let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
         let cancel = CancellationToken::new();
         let waiting = tokio::spawn({
             let semaphore = semaphore.clone();
-            let pacer = pacer.clone();
+            let rate = rate.clone();
             let cancel = cancel.clone();
-            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
+            async move { acquire_page_permit(semaphore, &rate, &cancel).await }
         });
         tokio::task::yield_now().await;
         provider.active.store(100, Ordering::SeqCst);
@@ -200,12 +199,13 @@ mod mainline_pacing_tests {
         assert_eq!(semaphore.available_permits(), 1);
         let deadline = tokio::time::timeout(
             Duration::from_millis(10),
-            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
+            acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
         )
         .await;
         assert!(deadline.is_err());
         assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
-        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
+        let unpaced = HealPageRate::new(1, None);
+        let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
             .await
             .expect("unpaced admission");
         assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,10 +221,11 @@ mod mainline_pacing_tests {
             let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
             let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let rate = HealPageRate::new(1, Some(pacer));
             let cancel = CancellationToken::new();
-            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
-            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
             assert!(futures::poll!(first.as_mut()).is_pending());
             assert!(futures::poll!(second.as_mut()).is_pending());
             drop(occupied);
@@ -329,6 +330,20 @@ impl ErasureSetHealer {
         }
     }
 
+    /// Page fan-out for one whole erasure-set pass (stable across buckets).
+    fn pass_page_concurrency_limit(&self) -> usize {
+        if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
+            // Versions of the same object must not race without its namespace lock.
+            1
+        } else {
+            Self::effective_heal_page_object_concurrency_for_source(
+                self.source,
+                self.heal_opts.scan_mode,
+                self.replacement_task_id.is_some(),
+            )
+        }
+    }
+
     /// Map the shared heal error class onto this healer's per-object actions.
     ///
     /// `classify_heal_error` is the only classifier; this adapter decides what
@@ -373,7 +388,7 @@ impl ErasureSetHealer {
         }
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }
@@ -464,12 +479,7 @@ impl ErasureSetHealer {
             None
         };
         let buckets = admin_buckets.as_deref().unwrap_or(buckets);
-        if checkpoint_manager
+        if checkpoint_manager.admin_completed().await {
-            .get_checkpoint()
-            .await
-            .admin
-            .is_some_and(|admin| admin.completed)
-        {
             self.initialize_progress(buckets, &resume_manager.get_state().await).await;
             return Ok(());
         }
@@ -490,14 +500,7 @@ impl ErasureSetHealer {
         }
 
         if self.admin_task.is_some() {
-            let mut admin = checkpoint_manager
+            checkpoint_manager.complete_admin().await?;
-                .get_checkpoint()
-                .await
-                .admin
-                .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
-            admin.completed = true;
-            admin.outcome.finish(None);
-            checkpoint_manager.set_admin(admin).await?;
         } else {
             checkpoint_manager.cleanup().await?;
             resume_manager.cleanup().await?;
@@ -718,7 +721,10 @@ impl ErasureSetHealer {
             }
             return Err(Error::DanglingDeleteDeferred { retry_not_before });
         }
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        // One rate controller for the whole pass: SlowDown cool-down and page
+        // fan-out persist across buckets instead of resetting at each one.
+        let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
+        let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
 
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -726,14 +732,17 @@ impl ErasureSetHealer {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
             set_disk_id,
-            current_bucket_index = checkpoint.current_bucket_index,
+            current_bucket_index,
-            current_object_index = checkpoint.current_object_index,
+            current_object_index,
             state = "resuming",
             "Erasure set resumed"
         );
 
         // 2. initialize progress
         self.initialize_progress(buckets, &state).await;
+        // Snapshot before any baseline write: set_progress_baseline overwrites
+        // the saved generation, which would hide a real generation mismatch.
+        let checkpoint = checkpoint_manager.progress_snapshot().await;
         let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
             let baseline = self.progress.read().await;
             (
@@ -760,17 +769,7 @@ impl ErasureSetHealer {
         let mut successful_objects = state.successful_objects;
         let mut failed_objects = state.failed_objects;
         let mut skipped_objects = state.skipped_objects;
-        let checkpoint_has_progress = checkpoint.baseline_known
+        let checkpoint_has_progress = checkpoint.has_progress();
-            || checkpoint.successful_objects > 0
-            || checkpoint.failed_object_count > 0
-            || checkpoint.skipped_object_count > 0
-            || checkpoint.skipped_new_versions > 0
-            || checkpoint.skipped_ilm_expired > 0
-            || checkpoint.processed_bytes > 0
-            || checkpoint.total_objects > 0
-            || checkpoint.total_bytes > 0
-            || checkpoint.baseline_generation.is_some()
-            || checkpoint.counter_unknown;
         let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
         let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
         if checkpoint_has_progress {
@@ -854,6 +853,7 @@ impl ErasureSetHealer {
                     resume_manager,
                     checkpoint_manager,
                     state.start_time,
+                    &page_rate,
                 )
                 .await;
 
@@ -1020,14 +1020,14 @@ impl ErasureSetHealer {
         if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
             if self.replacement_task_id.is_some() {
                 let state = resume_manager.get_state().await;
-                let checkpoint = checkpoint_manager.get_checkpoint().await;
+                let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
                 if state.can_retry()
                     && failed_objects == 0
                     && failed_buckets == 0
                     && skipped_objects > 0
-                    && skipped_objects == checkpoint.dangling_delete_grace_objects
+                    && skipped_objects == grace_objects
-                    && !checkpoint.counter_unknown
+                    && !counter_unknown
-                    && let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
+                    && let Some(retry_not_before) = retry_not_before
                 {
                     // Publish the generation's rewind and wait first. If the
                     // checkpoint reset is interrupted, initialization reapplies
@@ -1167,14 +1167,9 @@ impl ErasureSetHealer {
 
         let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
         let checkpoint_key = compose_key(&object_key, None);
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        if checkpoint_manager.is_object_settled(&checkpoint_key).await {
-        if checkpoint.processed_objects.contains(&checkpoint_key)
-            || checkpoint.failed_objects.contains(&checkpoint_key)
-            || checkpoint.skipped_objects.contains(&checkpoint_key)
-        {
             return Ok(());
         }
-        drop(checkpoint);
 
         self.verify_replacement_identity_fence("pool metadata").await?;
         resume_manager
@@ -1428,7 +1423,7 @@ impl ErasureSetHealer {
 
     /// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
     #[allow(clippy::too_many_arguments)]
-    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
+    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
     async fn heal_bucket_with_resume(
         &self,
         bucket: &str,
@@ -1442,6 +1437,7 @@ impl ErasureSetHealer {
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
         started_at_secs: u64,
+        page_rate: &Arc<HealPageRate>,
     ) -> Result<bool> {
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -1491,17 +1487,6 @@ impl ErasureSetHealer {
         // detects a non-advancing cursor even though next_marker embeds a
         // transient cache id that changes between identical pages.
         let mut previous_page_last: Option<(String, Option<String>)> = None;
-        let page_concurrency_limit =
-            if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
-                // Versions of the same object must not race without its namespace lock.
-                1
-            } else {
-                Self::effective_heal_page_object_concurrency_for_source(
-                    self.source,
-                    self.heal_opts.scan_mode,
-                    self.replacement_task_id.is_some(),
-                )
-            };
         let in_flight = Arc::new(AtomicUsize::new(0));
         // Per-bucket sample caps for per-object warn! lines: a flapping rebuild
         // disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1523,9 +1508,7 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
-            if let Some(pacer) = &self.mainline_pacer {
+            page_rate.wait_before_page(&self.cancel_token).await?;
-                pacer.wait(&self.cancel_token).await?;
-            }
             self.verify_replacement_identity_fence("page scan").await?;
             if let Some(expected) = incarnation {
                 self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1547,9 +1530,8 @@ impl ErasureSetHealer {
                     .await?
             };
             let page_is_empty = objects.is_empty();
-            let checkpoint = checkpoint_manager.get_checkpoint().await;
             let page_resume_index = *current_object_index;
-            let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
+            let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
             let mut page_tasks = FuturesUnordered::new();
             let mut completed_in_page = 0usize;
 
@@ -1563,10 +1545,7 @@ impl ErasureSetHealer {
 
                 // Per-version dedup identity — the single canonical key.
                 let key = compose_key(&item.name, item.version_id.as_deref());
-                if checkpoint.processed_objects.contains(&key)
+                if checkpoint_manager.is_object_settled(&key).await {
-                    || checkpoint.failed_objects.contains(&key)
-                    || checkpoint.skipped_objects.contains(&key)
-                {
                     continue;
                 }
 
@@ -1724,7 +1703,7 @@ impl ErasureSetHealer {
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
                 let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
-                let mainline_pacer = self.mainline_pacer.clone();
+                let page_rate = page_rate.clone();
 
                 let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
                 let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1736,14 +1715,15 @@ impl ErasureSetHealer {
                     .map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
                 let work = async move {
                     let mut retries = 0;
+                    let mut saw_slow_down = false;
                     let mut admin_identity = admin_identity;
                     let mut admin_task_id = admin_task_id;
                     loop {
-                        let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
+                        let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
 
                         let permit = match permit {
                             Ok(permit) => permit,
-                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
+                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
                         };
 
                         let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1751,7 +1731,8 @@ impl ErasureSetHealer {
                         if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
                             let (result, canonical) =
                                 admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
-                            return (dedup_key, object_name, version_id, result, canonical);
+                            let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
+                            return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
                         }
 
                         // Always go through heal_object. Genuine absence flows through
@@ -1853,18 +1834,23 @@ impl ErasureSetHealer {
 
                         drop(in_flight_guard);
                         drop(permit);
+                        if matches!(&result.1, Err(err) if err.is_slow_down()) {
+                            saw_slow_down = true;
+                        }
                         if retry_replacement
                             && retries < REPLACEMENT_OBJECT_MAX_RETRIES
                             && matches!(&result.1, Err(Error::TransientSkip { .. }))
                         {
                             retries += 1;
                             tokio::select! {
-                                _ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
+                                _ = cancel_token.cancelled() => {
+                                    return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
+                                }
                                 _ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
                             }
                             continue;
                         }
-                        return (dedup_key, object_name, version_id, result, None);
+                        return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
                     }
                 };
                 page_tasks.push(async move {
@@ -1875,7 +1861,7 @@ impl ErasureSetHealer {
                         };
                         match result {
                             Ok(result) => result,
-                            Err(error) => (key, object, version, (0, Err(error)), None),
+                            Err(error) => (key, object, version, (0, Err(error), false), None),
                         }
                     } else {
                         work.await
@@ -1884,7 +1870,16 @@ impl ErasureSetHealer {
             }
 
             while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
-                let (object_size, result) = result;
+                let (object_size, result, saw_slow_down) = result;
+                // Exactly one rate observation per finished object, including
+                // SlowDown that later succeeded on a replacement retry.
+                page_rate.observe_object_outcome(
+                    &match &result {
+                        Ok(_) => Ok(true),
+                        Err(err) => Err(err),
+                    },
+                    saw_slow_down,
+                );
                 let mut telemetry_unknown = false;
                 let checkpoint_outcome = match result {
                     Ok(true) => {
@@ -2482,6 +2477,7 @@ mod resume_loop_tests {
     use super::{
         ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
     };
+    use crate::heal::pacing::HealPageRate;
     use crate::heal::progress::HealProgress;
     use crate::heal::resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -3063,6 +3059,7 @@ mod resume_loop_tests {
         let mut successful = 0u64;
         let mut failed = 0u64;
         let mut skipped = 0u64;
+        let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
         let result = env
             .healer
             .heal_bucket_with_resume(
@@ -3077,6 +3074,7 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 state.start_time,
+                &page_rate,
             )
             .await;
         (processed, successful, failed, skipped, result.map(|_| ()))
@@ -3144,6 +3142,7 @@ mod resume_loop_tests {
         let mut skipped = 0;
         let started_at = env.resume.get_state().await.start_time;
 
+        let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
         let error = healer
             .heal_bucket_with_resume(
                 "b",
@@ -3157,6 +3156,7 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 started_at,
+                &page_rate,
             )
             .await
             .expect_err("a remounted target must not begin a new page scan");

crates/heal/src/heal/manager.rs

@@ -810,7 +810,14 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
 fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
     active_heals
         .iter()
-        .find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
+        .find(|(_, task)| {
+            PriorityHealQueue::make_dedup_key_for_owner(
+                &task.heal_type,
+                &task.options,
+                task.source,
+                task.expected_mrf_bucket_incarnation_id,
+            ) == key
+        })
         .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
 }
 

crates/heal/src/heal/manager/queue.rs

@@ -611,7 +611,104 @@ impl PriorityHealQueue {
 
     /// Create a deduplication key from a heal request
     pub(super) fn make_dedup_key(request: &HealRequest) -> String {
-        Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
+        Self::make_dedup_key_for_owner(
+            &request.heal_type,
+            &request.options,
+            request.source,
+            request.expected_mrf_bucket_incarnation_id,
+        )
+    }
+
+    /// Bound MRF work can only merge with work for the same bucket lifetime.
+    /// Length-prefix user-controlled components so a key cannot forge another
+    /// version, scope, or incarnation through delimiter characters.
+    pub(super) fn make_dedup_key_for_owner(
+        heal_type: &HealType,
+        options: &HealOptions,
+        source: HealRequestSource,
+        incarnation: Option<Uuid>,
+    ) -> String {
+        if source == HealRequestSource::Mrf
+            && let Some(incarnation) = incarnation
+        {
+            use std::fmt::Write as _;
+
+            let (pool, set) = heal_scope_indices(heal_type, options);
+            let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
+            let mut part = |value: &str| {
+                // Formatting into a String is infallible.
+                let _ = write!(key, "{}:{value}", value.len());
+            };
+            match heal_type {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("object");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::Metadata { bucket, object } => {
+                    part("metadata");
+                    part(bucket);
+                    part(object);
+                }
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("ecdecode");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::DeleteMarkerPurge {
+                    bucket,
+                    object,
+                    version_id,
+                    purge,
+                } => {
+                    part("delete-marker-purge");
+                    part(bucket);
+                    part(object);
+                    part(version_id);
+                    part(&purge.bucket_incarnation_id.to_string());
+                    part(&purge.marker_incarnation_id.to_string());
+                    part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
+                }
+                HealType::Bucket { bucket } => {
+                    part("bucket");
+                    part(bucket);
+                }
+                HealType::Prefix { bucket, prefix } => {
+                    part("prefix");
+                    part(bucket);
+                    part(prefix);
+                }
+                // Cluster and erasure-set work are not bucket-bound; preserve
+                // their existing keys, including contains_erasure_set lookups.
+                HealType::Cluster | HealType::ErasureSet { .. } => {
+                    return Self::make_dedup_key_for_scope(heal_type, options);
+                }
+            }
+            return key;
+        }
+        Self::make_dedup_key_for_scope(heal_type, options)
     }
 
     pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {

crates/heal/src/heal/manager/scheduler.rs

@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
     request: &HealRequest,
     config: &HealConfig,
     provider: &Option<WorkloadSnapshotProviderRef>,
-) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
+) -> Option<crate::heal::pacing::ForegroundPacing> {
     let pace_running = request.source == HealRequestSource::Admin
         || (request.source == HealRequestSource::AutoHeal
             && matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,14 +43,13 @@ pub(super) fn running_mainline_pacer(
         return None;
     }
     provider.as_ref().and_then(|provider| {
-        crate::heal::pacing::MainlinePacer::new(
+        crate::heal::pacing::ForegroundPacing::new(
             provider.clone(),
             config.mainline_read_utilization_high_percent,
             config.mainline_write_utilization_high_percent,
             config.mainline_max_sleep,
             request.source,
         )
-        .map(Arc::new)
     })
 }
 

crates/heal/src/heal/manager/tests.rs

@@ -117,6 +117,453 @@ async fn process_manager_queue_once(manager: &HealManager) {
 
 struct MockStorage;
 
+fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
+    let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
+    request.source = HealRequestSource::Mrf;
+    request.expected_mrf_bucket_incarnation_id = Some(incarnation);
+    request.options.pool_index = Some(0);
+    request.options.set_index = Some(0);
+    request
+}
+
+async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
+    let old_incarnation = Uuid::new_v4();
+    let new_incarnation = Uuid::new_v4();
+    let old_request = incarnation_bound_mrf_fallback(old_incarnation);
+    let old_id = old_request.id.clone();
+    match existing_state {
+        "queued" => {
+            let receipt = manager
+                .submit_heal_request_with_receipt(old_request)
+                .await
+                .expect("old fallback must be admitted");
+            assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+        }
+        "active" => {
+            manager
+                .active_heals
+                .lock()
+                .await
+                .insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
+        }
+        "retrying" => {
+            manager.retrying_heals.lock().await.insert(
+                old_id.clone(),
+                RetryingHeal {
+                    request: old_request,
+                    error: "retry pending".to_owned(),
+                    cancel_token: CancellationToken::new(),
+                },
+            );
+        }
+        other => panic!("unknown fixture state: {other}"),
+    }
+
+    let new_request = incarnation_bound_mrf_fallback(new_incarnation);
+    let new_id = new_request.id.clone();
+    assert_ne!(new_id, old_id, "the fallback must use a new request ID");
+    let receipt = manager
+        .submit_heal_request_with_receipt(new_request)
+        .await
+        .expect("new incarnation admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
+    );
+    assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
+    let queue = manager.heal_queue.lock().await;
+    assert!(
+        queue
+            .requests()
+            .any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
+    );
+    if existing_state == "queued" {
+        assert!(
+            queue
+                .requests()
+                .any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
+        );
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("active").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
+}
+
+fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
+    let mut request = incarnation_bound_mrf_fallback(incarnation);
+    if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
+        *id = version_id.map(str::to_owned);
+    } else {
+        unreachable!("fallback fixture must be an object request");
+    }
+    request
+}
+
+#[tokio::test]
+async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
+    for version in [Uuid::nil(), Uuid::new_v4()] {
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+        let incarnation = Uuid::new_v4();
+        let version_id = version.to_string();
+        let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
+        let original_id = original.id.clone();
+        let accepted = manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first queued fallback");
+        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
+        let merged = manager
+            .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
+            .await
+            .expect("duplicate queued fallback");
+        assert_eq!(merged.result, HealAdmissionResult::Merged);
+        assert_eq!(merged.task_id, original_id);
+        assert_eq!(manager.heal_queue.lock().await.len(), 1);
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let original = incarnation_bound_mrf_fallback(incarnation);
+    let original_id = original.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let same = manager
+        .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
+        .await
+        .expect("same-incarnation fallback");
+    assert_eq!(same.result, HealAdmissionResult::Merged);
+    assert_eq!(same.task_id, original_id);
+
+    let mut versioned = incarnation_bound_mrf_fallback(incarnation);
+    versioned.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "object".to_owned(),
+        version_id: Some(Uuid::new_v4().to_string()),
+    };
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(versioned)
+            .await
+            .expect("different version")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
+    other_scope.options.pool_index = Some(1);
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(other_scope)
+            .await
+            .expect("different pool")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    same_id_other_incarnation.id = original_id;
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(same_id_other_incarnation)
+            .await
+            .expect("same-ID conflict must be decided")
+            .result,
+        HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
+    );
+}
+
+#[test]
+fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
+    let incarnation = Uuid::new_v4();
+    let bound = incarnation_bound_mrf_fallback(incarnation);
+    let mut unbound = bound.clone();
+    unbound.expected_mrf_bucket_incarnation_id = None;
+    let mut non_mrf = bound.clone();
+    non_mrf.source = HealRequestSource::ReadRepair;
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key(&non_mrf),
+        PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
+        "an unrelated source retains its previous scope key even if the MRF-only field is set"
+    );
+    assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
+
+    let mut left = bound.clone();
+    left.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part:a".to_owned(),
+        version_id: Some("b".to_owned()),
+    };
+    let mut right = bound.clone();
+    right.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part".to_owned(),
+        version_id: Some("a:b".to_owned()),
+    };
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
+        PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
+        "the legacy delimiter-only key demonstrates the collision"
+    );
+    assert_ne!(
+        PriorityHealQueue::make_dedup_key(&left),
+        PriorityHealQueue::make_dedup_key(&right),
+        "length-prefixed object and version components must not alias"
+    );
+    let mut forged = unbound;
+    forged.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: PriorityHealQueue::make_dedup_key(&bound),
+        version_id: None,
+    };
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
+}
+
+#[test]
+fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
+    let incarnation = Uuid::new_v4();
+    for decode in [false, true] {
+        let make_request = |object: &str, version_id: Option<&str>| {
+            let mut request = incarnation_bound_mrf_fallback(incarnation);
+            let bucket = "mrf-recreated-bucket".to_owned();
+            let object = object.to_owned();
+            let version_id = version_id.map(str::to_owned);
+            request.heal_type = if decode {
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            } else {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            };
+            request
+        };
+        let absent = make_request("object", None);
+        let empty = make_request("object", Some(""));
+        let forged_object = make_request("object4:some0:", None);
+        let forged_version = make_request("object", Some("4:none"));
+        let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
+        for (index, key) in keys.iter().enumerate() {
+            assert!(keys.iter().skip(index + 1).all(|other| key != other));
+        }
+
+        let mut queue = PriorityHealQueue::new();
+        assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
+        assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
+        assert_eq!(queue.len(), 2);
+
+        for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
+            let mut legacy_absent = absent.clone();
+            let mut legacy_empty = empty.clone();
+            legacy_absent.source = source;
+            legacy_empty.source = source;
+            if source == HealRequestSource::Mrf {
+                legacy_absent.expected_mrf_bucket_incarnation_id = None;
+                legacy_empty.expected_mrf_bucket_incarnation_id = None;
+            }
+            assert_eq!(
+                PriorityHealQueue::make_dedup_key(&legacy_absent),
+                PriorityHealQueue::make_dedup_key(&legacy_empty),
+                "unbound and non-MRF work retain the legacy empty-version key"
+            );
+        }
+    }
+}
+
+#[test]
+fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
+    let mut queue = PriorityHealQueue::new();
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let old_id = old.id.clone();
+    let old_key = PriorityHealQueue::make_dedup_key(&old);
+    let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let new_id = new.id.clone();
+    let new_key = PriorityHealQueue::make_dedup_key(&new);
+    let mut forced_old = old.clone();
+    forced_old.id = Uuid::new_v4().to_string();
+    forced_old.force_start = true;
+    let forced_id = forced_old.id.clone();
+
+    assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    queue
+        .remove_request_id(&old_id)
+        .expect("remove the old incarnation representative");
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
+    assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
+    assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
+    assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
+}
+
+#[tokio::test]
+async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let incarnation = Uuid::new_v4();
+    let request = incarnation_bound_mrf_fallback(incarnation);
+    let request_id = request.id.clone();
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from("object"),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let receipt = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(request, target)
+        .await
+        .expect("durable admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a durable new incarnation needs its own owner"
+    );
+    assert_eq!(receipt.task_id, request_id);
+    assert!(
+        manager
+            .heal_queue
+            .lock()
+            .await
+            .requests()
+            .any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
+    );
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
+#[tokio::test]
+async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let object = format!("same-generation-{}", Uuid::new_v4());
+    let make_request = || {
+        let mut request = incarnation_bound_mrf_fallback(incarnation);
+        request.heal_type = HealType::Object {
+            bucket: "mrf-recreated-bucket".to_owned(),
+            object: object.clone(),
+            version_id: None,
+        };
+        request
+    };
+    let old = make_request();
+    let old_id = old.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from(object.as_str()),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let merged = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
+        .await
+        .expect("durable same-incarnation admission");
+    assert_eq!(merged.result, HealAdmissionResult::Merged);
+    assert_eq!(merged.task_id, old_id);
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
         task_id: "fixture-token".to_string(),

crates/heal/src/heal/pacing.rs

@@ -7,8 +7,14 @@ use rustfs_concurrency::{
     workload::{ForegroundPressure, foreground_pressure},
 };
 use rustfs_heal_contracts::heal_channel::HealRequestSource;
-use std::{sync::Arc, time::Duration};
+use std::{
-use tokio::{sync::Mutex, time::Instant};
+    sync::{
+        Arc,
+        atomic::{AtomicU32, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{sync::Mutex as AsyncMutex, time::Instant};
 use tokio_util::sync::CancellationToken;
 
 #[derive(Default)]
@@ -22,17 +28,19 @@ pub(crate) enum PacingDecision {
     Wait(Option<ForegroundPressure>),
 }
 
-/// Cooperative pacing for one heal execution, not a storage admission permit.
+/// Foreground watermarks for one heal execution. Constructed once per request;
-pub(crate) struct MainlinePacer {
+/// disabled when watermarks are zero or no workload provider is configured.
+#[derive(Clone)]
+pub(crate) struct ForegroundPacing {
     provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
     read_high: usize,
     write_high: usize,
     pause: Duration,
     source: HealRequestSource,
-    state: Mutex<PacingState>,
+    state: Arc<AsyncMutex<PacingState>>,
 }
 
-impl MainlinePacer {
+impl ForegroundPacing {
     pub(crate) fn new(
         provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
         read_high: usize,
@@ -49,7 +57,7 @@ impl MainlinePacer {
             write_high: write_high.min(100),
             pause: pause.min(Duration::from_secs(1)),
             source,
-            state: Mutex::new(PacingState::default()),
+            state: Arc::new(AsyncMutex::new(PacingState::default())),
         })
     }
 
@@ -125,6 +133,142 @@ impl MainlinePacer {
     }
 }
 
+/// Single owner of "how hard may this heal pass push storage?".
+///
+/// One instance spans the whole erasure-set scan (all buckets and pages):
+/// - foreground admission pressure (`ForegroundPacing`)
+/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
+///
+/// Call `observe_object_outcome` exactly once per finished object so a single
+/// SlowDown cannot double-reduce concurrency.
+pub(crate) struct HealPageRate {
+    base_concurrency: usize,
+    /// 0 = full base concurrency; each SlowDown increases the reduction.
+    slow_down_step: AtomicU32,
+    slow_down_until: std::sync::Mutex<Option<Instant>>,
+    foreground: Option<ForegroundPacing>,
+}
+
+impl HealPageRate {
+    /// Always-on controller. Foreground yield is optional; SlowDown control is not.
+    pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
+        Self {
+            base_concurrency: base_concurrency.max(1),
+            slow_down_step: AtomicU32::new(0),
+            slow_down_until: std::sync::Mutex::new(None),
+            foreground,
+        }
+    }
+
+    /// Concurrency after applying the current SlowDown reduction. Always ≥ 1
+    /// so maintenance can still make progress under sustained rate pressure.
+    pub(crate) fn effective_concurrency(&self) -> usize {
+        let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
+        (self.base_concurrency >> step).max(1)
+    }
+
+    /// Fold one finished object into the rate state. Call once per outcome.
+    /// `saw_slow_down` is true when any attempt of this object hit SlowDown,
+    /// including attempts that later succeeded on retry.
+    pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
+        if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
+            self.observe_slow_down();
+            return;
+        }
+        if result.is_ok() {
+            self.observe_success();
+        }
+    }
+
+    /// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
+    fn observe_slow_down(&self) {
+        // fetch_add returns the previous step; +1 so the first SlowDown uses
+        // exponent 1 (2s), not exponent 0 (1s).
+        let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
+        let delay = Self::backoff(step);
+        let ready = Instant::now() + delay;
+        if let Ok(mut until) = self.slow_down_until.lock()
+            && until.is_none_or(|current| current < ready)
+        {
+            *until = Some(ready);
+        }
+    }
+
+    /// Restore one concurrency step only after the SlowDown cool-down expires.
+    /// A same-page success must not undo the reduction while the pass is still
+    /// in the rate-limited window (the page semaphore is fixed until the next page).
+    fn observe_success(&self) {
+        let cooling = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .is_some_and(|ready| ready > Instant::now());
+        if cooling {
+            return;
+        }
+        self.slow_down_step
+            .try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
+            .ok();
+    }
+
+    /// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
+    pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
+        self.wait_slow_down(cancel).await?;
+        if let Some(foreground) = &self.foreground {
+            foreground.wait(cancel).await?;
+        }
+        Ok(())
+    }
+
+    /// Foreground admission decision while the caller owns page capacity.
+    pub(crate) fn admission_decision(&self) -> PacingDecision {
+        match &self.foreground {
+            Some(foreground) => foreground.admission_decision(),
+            None => PacingDecision::Ready,
+        }
+    }
+
+    /// Foreground wait after a rejected admission. Returns whether a pause was paid.
+    pub(crate) async fn wait_after_admission(
+        &self,
+        cancel: &CancellationToken,
+        observed: Option<ForegroundPressure>,
+    ) -> Result<bool> {
+        match &self.foreground {
+            Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
+            None => Ok(false),
+        }
+    }
+
+    async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
+        let delay = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .map(|ready| ready.saturating_duration_since(Instant::now()));
+        let Some(delay) = delay else {
+            return Ok(());
+        };
+        if delay.is_zero() {
+            return Ok(());
+        }
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            _ = tokio::time::sleep(delay) => Ok(()),
+        }
+    }
+
+    fn backoff(step: u32) -> Duration {
+        // step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
+        // rate with exponential backoff, and heal must do the same to itself.
+        let exponent = step.clamp(1, 5);
+        Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
+    }
+}
+
 #[cfg(test)]
 pub(crate) struct TestPressure {
     pub(crate) active: std::sync::atomic::AtomicUsize,
@@ -164,12 +308,87 @@ mod tests {
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
 
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_first_slow_down_waits_two_seconds() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("cool-down wait");
+        assert_eq!(Instant::now() - before, Duration::from_secs(2));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_reduces_concurrency_and_backs_off() {
+        let rate = HealPageRate::new(8, None);
+        assert_eq!(rate.effective_concurrency(), 8);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        assert_eq!(rate.effective_concurrency(), 2);
+        // Floor is one so maintenance never stops entirely.
+        for _ in 0..8 {
+            rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        }
+        assert_eq!(rate.effective_concurrency(), 1);
+
+        let cancel = CancellationToken::new();
+        assert!(rate.wait_before_page(&cancel).await.is_ok());
+
+        // After cool-down, successes restore concurrency one step at a time.
+        for _ in 0..12 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert!(rate.effective_concurrency() > 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        // Same-page successes while cool-down is armed must not restore fan-out.
+        for _ in 0..4 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
+        let rate = HealPageRate::new(8, None);
+        // Replacement retry: SlowDown on attempt 1, success on attempt 2.
+        rate.observe_object_outcome(&Ok(true), true);
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_wait_is_cancellable() {
+        let rate = HealPageRate::new(4, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        let cancel = CancellationToken::new();
+        cancel.cancel();
+        assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
+        let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
+            .expect("foreground pacer");
+        let rate = HealPageRate::new(4, Some(foreground));
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
+        assert_eq!(Instant::now() - before, Duration::from_millis(250));
+    }
+
     #[tokio::test(start_paused = true)]
     async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
         for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
             let provider = Arc::new(TestPressure::new(class, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(
+            let pacer = ForegroundPacing::new(
                 provider.clone(),
                 if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
                 if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
@@ -197,9 +416,9 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
-        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
+        assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
-        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
+        assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
-        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
+        let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
         let before = Instant::now();
         pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
         assert_eq!(Instant::now() - before, Duration::from_secs(1));
@@ -209,7 +428,7 @@ mod tests {
     async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
         let paced = Arc::new(
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
         );
         let cancel_first = CancellationToken::new();
         let first = tokio::spawn({
@@ -228,7 +447,7 @@ mod tests {
         cancel_second.cancel();
         assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
         provider.active.store(0, Ordering::SeqCst);
-        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
+        let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
             .expect("independent task");
         let before = Instant::now();
         other_task

crates/heal/src/heal/resume.rs

@@ -35,7 +35,9 @@ mod replacement;
 mod utils;
 
 pub(crate) use checkpoint::AdminErasureCheckpoint;
-pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
+pub use checkpoint::{
+    CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
+};
 pub(crate) use gc::ResumeGc;
 pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
 pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};

crates/heal/src/heal/resume/checkpoint.rs

@@ -152,6 +152,42 @@ impl AdminErasureCheckpoint {
     }
 }
 
+/// Lightweight progress view for resume decisions. Omits the per-page object
+/// ledgers so callers do not clone a million-object checkpoint just to read
+/// counters and positions.
+#[derive(Debug, Clone, Copy)]
+pub struct CheckpointProgressSnapshot {
+    pub current_bucket_index: usize,
+    pub current_object_index: usize,
+    pub successful_objects: u64,
+    pub failed_object_count: u64,
+    pub skipped_object_count: u64,
+    pub skipped_new_versions: u64,
+    pub skipped_ilm_expired: u64,
+    pub processed_bytes: u64,
+    pub total_objects: u64,
+    pub total_bytes: u64,
+    pub baseline_generation: Option<u64>,
+    pub baseline_known: bool,
+    pub counter_unknown: bool,
+}
+
+impl CheckpointProgressSnapshot {
+    pub fn has_progress(&self) -> bool {
+        self.baseline_known
+            || self.successful_objects > 0
+            || self.failed_object_count > 0
+            || self.skipped_object_count > 0
+            || self.skipped_new_versions > 0
+            || self.skipped_ilm_expired > 0
+            || self.processed_bytes > 0
+            || self.total_objects > 0
+            || self.total_bytes > 0
+            || self.baseline_generation.is_some()
+            || self.counter_unknown
+    }
+}
+
 /// resume checkpoint
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ResumeCheckpoint {
@@ -715,6 +751,75 @@ impl CheckpointManager {
         self.checkpoint.read().await.clone()
     }
 
+    /// True when a page object is already settled in the current ledger.
+    /// Membership is checked under the lock without cloning the checkpoint,
+    /// which used to copy every page's dedup sets on the small-object path.
+    pub async fn is_object_settled(&self, key: &str) -> bool {
+        let checkpoint = self.checkpoint.read().await;
+        checkpoint.processed_objects.contains(key)
+            || checkpoint.failed_objects.contains(key)
+            || checkpoint.skipped_objects.contains(key)
+    }
+
+    /// True when the administrator checkpoint already finished.
+    pub async fn admin_completed(&self) -> bool {
+        self.checkpoint
+            .read()
+            .await
+            .admin
+            .as_ref()
+            .is_some_and(|admin| admin.completed)
+    }
+
+    /// Current resume position without cloning the object ledgers.
+    pub async fn resume_position(&self) -> (usize, usize) {
+        let checkpoint = self.checkpoint.read().await;
+        (checkpoint.current_bucket_index, checkpoint.current_object_index)
+    }
+
+    /// Dangling-delete grace summary without cloning the object ledgers.
+    pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
+        let checkpoint = self.checkpoint.read().await;
+        (
+            checkpoint.dangling_delete_grace_objects,
+            checkpoint.dangling_delete_retry_not_before,
+            checkpoint.counter_unknown,
+        )
+    }
+
+    /// Aggregate progress counters without cloning the page dedup sets.
+    pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
+        let checkpoint = self.checkpoint.read().await;
+        CheckpointProgressSnapshot {
+            current_bucket_index: checkpoint.current_bucket_index,
+            current_object_index: checkpoint.current_object_index,
+            successful_objects: checkpoint.successful_objects,
+            failed_object_count: checkpoint.failed_object_count,
+            skipped_object_count: checkpoint.skipped_object_count,
+            skipped_new_versions: checkpoint.skipped_new_versions,
+            skipped_ilm_expired: checkpoint.skipped_ilm_expired,
+            processed_bytes: checkpoint.processed_bytes,
+            total_objects: checkpoint.total_objects,
+            total_bytes: checkpoint.total_bytes,
+            baseline_generation: checkpoint.baseline_generation,
+            baseline_known: checkpoint.baseline_known,
+            counter_unknown: checkpoint.counter_unknown,
+        }
+    }
+
+    /// Finish the administrator outcome in place and persist it.
+    pub async fn complete_admin(&self) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        let admin = checkpoint
+            .admin
+            .as_mut()
+            .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
+        admin.completed = true;
+        admin.outcome.finish(None);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
     /// update position
     pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;

crates/heal/src/heal/task.rs

@@ -485,7 +485,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 impl HealTask {
@@ -586,7 +586,7 @@ impl HealTask {
         task
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }

crates/heal/src/heal/task/heal_object.rs

@@ -480,8 +480,9 @@ impl HealTask {
                 message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
             })?;
         if current_bucket_incarnation_id != bucket_incarnation_id {
-            return Err(Error::TaskExecutionFailed {
+            return Err(Error::StaleBucketIncarnation {
-                message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
+                bucket: bucket.to_owned(),
+                expected: Some(bucket_incarnation_id),
             });
         }
         expected.bucket_incarnation_id = Some(bucket_incarnation_id);

crates/heal/src/heal/task/tests.rs

@@ -3375,10 +3375,67 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
 
     let result = task.heal_object("bucket", "object", None).await;
 
-    assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
+    assert!(matches!(
+        result,
+        Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
+    ));
     assert!(storage.heal_object_calls.lock().unwrap().is_empty());
 }
 
+#[tokio::test]
+async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
+    for object in ["timed out", "network error", "ordinary"] {
+        let original_incarnation = Uuid::new_v4();
+        let recreated_incarnation = Uuid::new_v4();
+        let storage = Arc::new(MockStorage {
+            bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
+            ..Default::default()
+        });
+        let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
+        request.source = HealRequestSource::Mrf;
+        request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
+        request.options.pool_index = Some(0);
+        request.options.set_index = Some(1);
+        let task = HealTask::from_request(request, storage.clone());
+
+        let error = task
+            .execute()
+            .await
+            .expect_err("a recreated bucket must reject the old MRF responsibility");
+        assert!(
+            !error.is_recoverable_heal(),
+            "user-controlled object names must not make a stale incarnation retryable: {object}"
+        );
+        assert!(
+            matches!(
+                &error,
+                Error::StaleBucketIncarnation { bucket, expected }
+                    if bucket == "bucket" && *expected == Some(original_incarnation)
+            ),
+            "the failure must retain the original bucket incarnation: {error:?}"
+        );
+        assert!(
+            matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
+            "stale responsibility must be classified independently of the object name"
+        );
+        assert!(
+            storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
+            "an old responsibility must not reach storage repair in the recreated bucket"
+        );
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
+        assert!(
+            outcome.objects.iter().all(|item| !matches!(
+                &item.disposition,
+                HealObjectDisposition::Repaired
+                    | HealObjectDisposition::VerifiedHealthy
+                    | HealObjectDisposition::AuthoritativelyAbsent
+            )),
+            "the stale task must not produce a positive repair proof"
+        );
+    }
+}
+
 #[tokio::test(start_paused = true)]
 async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
     let storage = Arc::new(MockStorage {

crates/iam/src/cache.rs

@@ -15,7 +15,6 @@
 use std::{
     collections::{HashMap, HashSet},
     ops::{Deref, DerefMut},
-    ptr,
     sync::{Arc, Mutex, Weak},
 };
 
@@ -64,21 +63,46 @@ impl Default for CacheState {
 pub struct Cache {
     state: ArcSwap<CacheState>,
     write_lock: Mutex<()>,
+    reload_lock: AsyncMutex<()>,
+    reload_changes: Mutex<Weak<Mutex<ReloadChanges>>>,
     service_account_mutation_lock: AsyncMutex<()>,
-    sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
+    sts_account_mutation_locks: Arc<MutationLockRegistry>,
+    parent_mutation_locks: Arc<MutationLockRegistry>,
 }
 
-struct StsMutationLockRegistry {
+#[derive(Default)]
-    locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
+struct MutationLockRegistry {
+    locks: Mutex<HashMap<String, Weak<AsyncMutex<MutationLockState>>>>,
 }
 
-pub(crate) struct StsMutationLockState {
+pub(crate) struct MutationLockState {
     access_key: String,
-    registry: Weak<StsMutationLockRegistry>,
+    registry: Weak<MutationLockRegistry>,
-    lock: Weak<AsyncMutex<StsMutationLockState>>,
+    lock: Weak<AsyncMutex<MutationLockState>>,
 }
 
-impl Drop for StsMutationLockState {
+impl MutationLockRegistry {
+    fn lock_for(self: &Arc<Self>, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
+        let mut locks = self.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
+            return lock;
+        }
+
+        let registry = Arc::downgrade(self);
+        let access_key_owned = access_key.to_string();
+        let lock = Arc::new_cyclic(|lock| {
+            AsyncMutex::new(MutationLockState {
+                access_key: access_key_owned,
+                registry,
+                lock: lock.clone(),
+            })
+        });
+        locks.insert(access_key.to_string(), Arc::downgrade(&lock));
+        lock
+    }
+}
+
+impl Drop for MutationLockState {
     fn drop(&mut self) {
         let Some(registry) = self.registry.upgrade() else {
             return;
@@ -95,10 +119,52 @@ impl Default for Cache {
         Self {
             state: ArcSwap::new(Arc::new(CacheState::default())),
             write_lock: Mutex::new(()),
+            reload_lock: AsyncMutex::new(()),
+            reload_changes: Mutex::new(Weak::new()),
             service_account_mutation_lock: AsyncMutex::new(()),
-            sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
+            sts_account_mutation_locks: Arc::new(MutationLockRegistry::default()),
-                locks: Mutex::new(HashMap::new()),
+            parent_mutation_locks: Arc::new(MutationLockRegistry::default()),
-            }),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Hash, PartialEq, Eq)]
+pub(crate) enum CachePartition {
+    PolicyDocs,
+    Users,
+    UserPolicies,
+    StsAccounts,
+    StsPolicies,
+    Groups,
+    UserGroupMemberships,
+    GroupPolicies,
+}
+
+type ReloadChanges = HashMap<CachePartition, HashSet<String>>;
+
+/// A reload journals only keys changed during its own lifetime, including
+/// deletions. Unrelated STS traffic cannot suppress a persisted revocation.
+pub(crate) struct CacheReload<'a> {
+    _guard: tokio::sync::MutexGuard<'a, ()>,
+    changes: Arc<Mutex<ReloadChanges>>,
+}
+
+impl CacheReload<'_> {
+    pub(crate) fn merge_partition<T: Clone>(
+        &self,
+        partition: CachePartition,
+        loaded: &mut CacheEntity<T>,
+        current: &CacheEntity<T>,
+    ) {
+        let changes = self.changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(keys) = changes.get(&partition) {
+            for key in keys {
+                if let Some(value) = current.get(key) {
+                    loaded.insert(key.clone(), value.clone());
+                } else {
+                    loaded.remove(key);
+                }
+            }
         }
     }
 }
@@ -106,31 +172,27 @@ impl Default for Cache {
 pub(crate) type CacheSnapshot = Guard<Arc<CacheState>>;
 
 impl Cache {
+    pub(crate) async fn begin_reload(&self) -> CacheReload<'_> {
+        // Registration and publication both take write_lock before the journal
+        // lock, so no mutation can fall between the reload and its journal.
+        let guard = self.reload_lock.lock().await;
+        let changes = Arc::new(Mutex::new(HashMap::new()));
+        let _write = self.write_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        *self.reload_changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Arc::downgrade(&changes);
+        CacheReload { _guard: guard, changes }
+    }
+
     pub(crate) fn service_account_mutation_lock(&self) -> &AsyncMutex<()> {
         &self.service_account_mutation_lock
     }
 
-    pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
+    pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
-        let mut locks = self
+        self.sts_account_mutation_locks.lock_for(access_key)
-            .sts_account_mutation_locks
-            .locks
-            .lock()
-            .unwrap_or_else(|poisoned| poisoned.into_inner());
-        if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
-            return lock;
     }
 
-        let registry = Arc::downgrade(&self.sts_account_mutation_locks);
+    /// Orders derived-credential publication with changes to its regular parent.
-        let access_key_owned = access_key.to_string();
+    pub(crate) fn parent_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
-        let lock = Arc::new_cyclic(|lock| {
+        self.parent_mutation_locks.lock_for(access_key)
-            AsyncMutex::new(StsMutationLockState {
-                access_key: access_key_owned,
-                registry,
-                lock: lock.clone(),
-            })
-        });
-        locks.insert(access_key.to_string(), Arc::downgrade(&lock));
-        lock
     }
 
     pub(crate) fn snapshot(&self) -> CacheSnapshot {
@@ -142,7 +204,13 @@ impl Cache {
         let current = self.state.load_full();
         let mut locked = LockedCache {
             state: CacheState::clone(&current),
+            #[cfg(test)]
             current_ptr: Arc::as_ptr(&current),
+            reload_changes: self
+                .reload_changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .upgrade(),
             dirty: false,
         };
         let ret = f(&mut locked);
@@ -215,7 +283,9 @@ impl Cache {
 
 pub(crate) struct LockedCache {
     state: CacheState,
+    #[cfg(test)]
     current_ptr: *const CacheState,
+    reload_changes: Option<Arc<Mutex<ReloadChanges>>>,
     dirty: bool,
 }
 
@@ -224,11 +294,19 @@ impl LockedCache {
         &self.state
     }
 
+    #[cfg(test)]
     pub(crate) fn matches_snapshot(&self, snapshot: &CacheSnapshot) -> bool {
-        ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
+        std::ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
     }
 
-    fn exec<T: Clone>(target: &mut Arc<CacheEntity<T>>, t: OffsetDateTime, mut op: impl FnMut(&mut CacheEntity<T>)) -> bool {
+    fn exec<T: Clone>(
+        target: &mut Arc<CacheEntity<T>>,
+        t: OffsetDateTime,
+        changes: Option<&Arc<Mutex<ReloadChanges>>>,
+        partition: CachePartition,
+        key: &str,
+        mut op: impl FnMut(&mut CacheEntity<T>),
+    ) -> bool {
         if target.load_time >= t {
             return false;
         }
@@ -236,6 +314,14 @@ impl LockedCache {
         let mut new = CacheEntity::clone(target);
         op(&mut new);
         *target = Arc::new(new);
+        if let Some(changes) = changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(partition)
+                .or_default()
+                .insert(key.to_string());
+        }
         true
     }
 
@@ -244,139 +330,315 @@ impl LockedCache {
     }
 
     pub(crate) fn replace_policy_docs(&mut self, value: CacheEntity<PolicyDoc>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::PolicyDocs)
+                .or_default()
+                .extend(self.state.policy_docs.keys().chain(value.keys()).cloned());
+        }
         self.state.policy_docs = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_users(&mut self, value: CacheEntity<UserIdentity>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::Users)
+                .or_default()
+                .extend(self.state.users.keys().chain(value.keys()).cloned());
+        }
         self.state.users = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_user_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::UserPolicies)
+                .or_default()
+                .extend(self.state.user_policies.keys().chain(value.keys()).cloned());
+        }
         self.state.user_policies = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_sts_accounts(&mut self, value: CacheEntity<UserIdentity>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::StsAccounts)
+                .or_default()
+                .extend(self.state.sts_accounts.keys().chain(value.keys()).cloned());
+        }
         self.state.sts_accounts = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_sts_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::StsPolicies)
+                .or_default()
+                .extend(self.state.sts_policies.keys().chain(value.keys()).cloned());
+        }
         self.state.sts_policies = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_groups(&mut self, value: CacheEntity<GroupInfo>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::Groups)
+                .or_default()
+                .extend(self.state.groups.keys().chain(value.keys()).cloned());
+        }
         self.state.groups = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_group_policies(&mut self, value: CacheEntity<MappedPolicy>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::GroupPolicies)
+                .or_default()
+                .extend(self.state.group_policies.keys().chain(value.keys()).cloned());
+        }
         self.state.group_policies = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn replace_user_group_memberships(&mut self, value: CacheEntity<HashSet<String>>) {
+        if let Some(changes) = &self.reload_changes {
+            changes
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner())
+                .entry(CachePartition::UserGroupMemberships)
+                .or_default()
+                .extend(self.state.user_group_memberships.keys().chain(value.keys()).cloned());
+        }
         self.state.user_group_memberships = Self::replaced(value);
         self.dirty = true;
     }
 
     pub(crate) fn add_or_update_policy_doc(&mut self, key: &str, value: &PolicyDoc, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.policy_docs,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::PolicyDocs,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_user(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.users, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.users,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::Users,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_user_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.user_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::UserPolicies,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_sts_account(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.sts_accounts,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::StsAccounts,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_sts_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.sts_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::StsPolicies,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_group(&mut self, key: &str, value: &GroupInfo, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.groups,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::Groups,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_user_group_membership(&mut self, key: &str, value: &HashSet<String>, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.user_group_memberships,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::UserGroupMemberships,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn add_or_update_group_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.group_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::GroupPolicies,
+            key,
+            |map| {
                 map.insert(key.to_string(), value.clone());
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_policy_doc(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.policy_docs,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::PolicyDocs,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_user(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.users, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.users,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::Users,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_user_policy(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.user_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::UserPolicies,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_user_group_membership(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.user_group_memberships,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::UserGroupMemberships,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_sts_account(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.sts_accounts,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::StsAccounts,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_sts_policy(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.sts_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::StsPolicies,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_group(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.groups,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::Groups,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn delete_group_policy(&mut self, key: &str, t: OffsetDateTime) {
-        self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
+        self.dirty |= Self::exec(
+            &mut self.state.group_policies,
+            t,
+            self.reload_changes.as_ref(),
+            CachePartition::GroupPolicies,
+            key,
+            |map| {
                 map.remove(key);
-        });
+            },
+        );
     }
 
     pub(crate) fn build_user_group_memberships(&mut self) {
@@ -494,6 +756,50 @@ mod tests {
     use crate::cache::Cache;
     use crate::store::MappedPolicy;
 
+    #[tokio::test]
+    async fn reload_journal_preserves_same_key_mutations_and_expires_on_cancellation() {
+        use super::{CacheEntity, CachePartition};
+        let cache = Cache::default();
+        let stale = UserIdentity {
+            version: 1,
+            ..Default::default()
+        };
+        let current = UserIdentity {
+            version: 2,
+            ..Default::default()
+        };
+        let reload = cache.begin_reload().await;
+        cache.add_or_update_user("updated", &current, OffsetDateTime::now_utc());
+        cache.add_or_update_user("removed", &current, OffsetDateTime::now_utc());
+        cache.delete_user("removed", OffsetDateTime::now_utc());
+        // Even an absent key's deletion is recorded: storage may have returned
+        // an entry created and deleted entirely during the reload's reads.
+        cache.delete_user("absent", OffsetDateTime::now_utc());
+        let mut loaded = CacheEntity::new(std::collections::HashMap::from([
+            ("updated".to_string(), stale.clone()),
+            ("removed".to_string(), stale.clone()),
+            ("absent".to_string(), stale.clone()),
+            ("disk-only".to_string(), stale),
+        ]));
+        cache.with_write_lock(|locked| {
+            reload.merge_partition(CachePartition::Users, &mut loaded, &locked.state().users);
+            locked.replace_users(loaded);
+        });
+        let snapshot = cache.snapshot();
+        assert_eq!(snapshot.users["updated"].version, 2);
+        assert!(!snapshot.users.contains_key("removed"));
+        assert!(!snapshot.users.contains_key("absent"));
+        assert!(snapshot.users.contains_key("disk-only"));
+        drop(reload);
+        assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
+        let cancelled = cache.begin_reload().await;
+        cache.delete_user("disk-only", OffsetDateTime::now_utc());
+        drop(cancelled);
+        assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
+        let next = cache.begin_reload().await;
+        assert!(next.changes.lock().expect("new journal").is_empty());
+    }
+
     #[test]
     fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
         let cache = Cache::default();

crates/iam/src/federation/mapper.rs

@@ -16,6 +16,7 @@ use super::{FederatedClaims, VerifiedFederatedIdentity};
 use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
 use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
 use rustfs_utils::HashAlgorithm;
+use serde_json::Value;
 use std::{collections::HashMap, sync::Arc};
 use tracing::debug;
 
@@ -66,6 +67,15 @@ impl FederatedAuthorization {
         &self.provider_id
     }
 
+    /// Issuer from the verified upstream identity.
+    pub fn issuer(&self) -> Option<&str> {
+        self.claims
+            .raw
+            .get("iss")
+            .and_then(Value::as_str)
+            .filter(|issuer| !issuer.is_empty())
+    }
+
     pub fn subject(&self) -> &str {
         &self.claims.sub
     }

crates/iam/src/manager.rs

@@ -415,6 +415,16 @@ where
 
         self.api.save_iam_config(IAMFormat::new_version_1(), path).await
     }
+    /// Current built-in IAM group membership, independent of credential snapshots.
+    pub fn user_group_memberships(&self, name: &str) -> Vec<String> {
+        self.cache
+            .snapshot()
+            .user_group_memberships
+            .get(name)
+            .map(|groups| groups.iter().cloned().collect())
+            .unwrap_or_default()
+    }
+
     pub async fn get_user(&self, access_key: &str) -> Option<UserIdentity> {
         let cache = self.cache.snapshot();
         cache
@@ -1005,6 +1015,17 @@ where
     }
 
     pub async fn sts_policy_db_get(&self, name: &str, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
+        self.sts_policy_db_get_with_identity_mapping(name, groups)
+            .await
+            .map(|(policies, _)| policies)
+    }
+
+    /// Resolve STS policies and whether that same lookup found a nonempty identity mapping.
+    pub async fn sts_policy_db_get_with_identity_mapping(
+        &self,
+        name: &str,
+        groups: &Option<Vec<String>>,
+    ) -> Result<(Vec<String>, bool)> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1031,7 +1052,10 @@ where
             }
         };
 
-        self.policy_db_get_with_groups(mapped_policy.to_slice(), groups).await
+        let direct_policies = mapped_policy.to_slice();
+        let has_identity_mapping = !direct_policies.is_empty();
+        let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
+        Ok((policies, has_identity_mapping))
     }
 
     async fn policy_db_get_with_groups(&self, mut policies: Vec<String>, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
@@ -1158,7 +1182,7 @@ where
         if let Some(groups) = u.credentials.groups.as_ref() {
             for group in groups.iter() {
                 if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
-                    return Ok((Vec::new(), OffsetDateTime::now_utc()));
+                    continue;
                 }
 
                 let mp = match group_policies.get(group) {
@@ -1189,7 +1213,7 @@ where
 
         for group in user_group_memberships.get(name).cloned().unwrap_or_default().iter() {
             if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
-                return Ok((Vec::new(), OffsetDateTime::now_utc()));
+                continue;
             }
 
             let mp = match group_policies.get(group) {
@@ -1486,7 +1510,27 @@ where
         args: &AddOrUpdateUserReq,
         updated_at: OffsetDateTime,
     ) -> Result<OffsetDateTime> {
+        self.add_user_at_with_secret_change(access_key, args, updated_at)
+            .await
+            .map(|(updated_at, _, _guard)| updated_at)
+    }
+
+    /// Return whether this write replaced a different regular-user secret.
+    pub(crate) async fn add_user_at_with_secret_change(
+        &self,
+        access_key: &str,
+        args: &AddOrUpdateUserReq,
+        updated_at: OffsetDateTime,
+    ) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
+        let parent_lock = self.cache.parent_mutation_lock(access_key);
+        let parent_guard = parent_lock.lock_owned().await;
+        let _service_guard = self.cache.service_account_mutation_lock().lock().await;
+        let sts_lock = self.cache.sts_account_mutation_lock(access_key);
+        let _sts_guard = sts_lock.lock().await;
         let cache = self.cache.snapshot();
+        if cache.sts_accounts.contains_key(access_key) {
+            return Err(Error::AccessKeyAlreadyExists);
+        }
         let users = Arc::clone(&cache.users);
         if let Some(x) = users.get(access_key) {
             debug!(
@@ -1496,10 +1540,13 @@ where
                 state = "existing",
                 "Updating existing IAM user"
             );
-            if x.credentials.is_temp() {
+            if x.credentials.is_temp() || x.credentials.is_service_account() {
                 return Err(Error::AccessKeyAlreadyExists);
             }
         }
+        let secret_changed = users
+            .get(access_key)
+            .is_some_and(|user| user.credentials.secret_key != args.secret_key);
         drop(cache);
         drop(users);
 
@@ -1523,7 +1570,7 @@ where
 
         self.update_user_with_claims(access_key, user_entry)?;
 
-        Ok(updated_at)
+        Ok((updated_at, secret_changed, parent_guard))
     }
 
     pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
@@ -1540,12 +1587,17 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let parent_lock = (utype == UserType::Reg).then(|| self.cache.parent_mutation_lock(access_key));
+        let _parent_guard = match &parent_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
         let _sts_mutation_guard = match &sts_mutation_lock {
             Some(lock) => Some(lock.lock().await),
             None => None,
         };
-        let _service_account_guard = if utype == UserType::Svc {
+        let _service_account_guard = if matches!(utype, UserType::Svc | UserType::Reg) {
             Some(self.cache.service_account_mutation_lock().lock().await)
         } else {
             None
@@ -1640,17 +1692,46 @@ where
     }
 
     pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
+        self.update_user_secret_key_with_guard(access_key, secret_key, None)
+            .await
+            .map(|(updated_at, status, _guard)| (updated_at, status))
+    }
+
+    pub(crate) async fn update_user_secret_key_with_guard(
+        &self,
+        access_key: &str,
+        secret_key: &str,
+        expected_secret: Option<&str>,
+    ) -> Result<(
+        OffsetDateTime,
+        AccountStatus,
+        tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>,
+    )> {
         if access_key.is_empty() || secret_key.is_empty() {
             return Err(Error::InvalidArgument);
         }
 
+        let parent_lock = self.cache.parent_mutation_lock(access_key);
+        let parent_guard = parent_lock.lock_owned().await;
         let cache = self.cache.snapshot();
         let users = Arc::clone(&cache.users);
         let u = match users.get(access_key) {
             Some(u) => u,
+            None if expected_secret.is_some() => return Err(Error::IAMActionNotAllowed),
             None => return Err(Error::NoSuchUser(access_key.to_string())),
         };
 
+        if let Some(expected) = expected_secret {
+            use subtle::ConstantTimeEq as _;
+            if !u.credentials.is_valid()
+                || u.credentials.is_temp()
+                || u.credentials.is_service_account()
+                || !bool::from(u.credentials.secret_key.as_bytes().ct_eq(expected.as_bytes()))
+            {
+                return Err(Error::IAMActionNotAllowed);
+            }
+        }
+
         let mut cred = u.credentials.clone();
         cred.secret_key = secret_key.to_string();
 
@@ -1672,7 +1753,7 @@ where
             .await?;
 
         self.update_user_with_claims(access_key, u)?;
-        Ok((updated_at, status))
+        Ok((updated_at, status, parent_guard))
     }
 
     /// Add SSH public key for a user (for SFTP authentication)
@@ -1681,6 +1762,8 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let parent_lock = self.cache.parent_mutation_lock(access_key);
+        let _parent_guard = parent_lock.lock().await;
         let cache = self.cache.snapshot();
         let users = Arc::clone(&cache.users);
         let u = match users.get(access_key) {
@@ -1720,6 +1803,8 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let parent_lock = self.cache.parent_mutation_lock(access_key);
+        let _parent_guard = parent_lock.lock().await;
         let cache = self.cache.snapshot();
         let users = Arc::clone(&cache.users);
         let u = match users.get(access_key) {
@@ -2238,6 +2323,11 @@ where
         Ok(())
     }
     pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
+        let parent_lock = (user_type == UserType::Reg).then(|| self.cache.parent_mutation_lock(name));
+        let _parent_guard = match &parent_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
         let _sts_mutation_guard = match &sts_mutation_lock {
             Some(lock) => Some(lock.lock().await),
@@ -2985,6 +3075,58 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn disabled_group_preserves_direct_and_other_group_policies() {
+        for credential_groups in [false, true] {
+            let iam = build_test_iam_cache(FailingInitialLoadStore);
+            let groups = vec!["disabled".to_string(), "enabled".to_string()];
+            let user = UserIdentity::new(Credentials {
+                access_key: "alice".to_string(),
+                secret_key: "alice-secret".to_string(),
+                status: STATUS_ENABLED.to_string(),
+                groups: credential_groups.then(|| groups.clone()),
+                ..Default::default()
+            });
+            let now = OffsetDateTime::now_utc();
+            iam.cache.add_or_update_user("alice", &user, now);
+            iam.cache
+                .add_or_update_user_policy("alice", &MappedPolicy::new("direct-deny"), now);
+            if !credential_groups {
+                iam.cache
+                    .add_or_update_user_group_membership("alice", &groups.iter().cloned().collect(), now);
+            }
+            for (group, status, policy) in [
+                ("disabled", STATUS_DISABLED, "disabled-grant"),
+                ("enabled", STATUS_ENABLED, "other-group-deny"),
+            ] {
+                iam.cache.add_or_update_group(
+                    group,
+                    &GroupInfo {
+                        status: status.to_string(),
+                        members: vec!["alice".to_string()],
+                        ..Default::default()
+                    },
+                    now,
+                );
+                iam.cache.add_or_update_group_policy(group, &MappedPolicy::new(policy), now);
+            }
+            let policies: HashSet<_> = iam
+                .policy_db_get("alice", &None)
+                .await
+                .expect("user policy lookup")
+                .into_iter()
+                .collect();
+            assert_eq!(policies, HashSet::from(["direct-deny".to_string(), "other-group-deny".to_string()]));
+            assert!(
+                iam.policy_db_get_internal("disabled", true, false)
+                    .await
+                    .expect("disabled group lookup")
+                    .0
+                    .is_empty()
+            );
+        }
+    }
+
     #[tokio::test]
     async fn sts_policy_lookup_ignores_colliding_regular_user_mapping() {
         let iam = build_test_iam_cache(FailingInitialLoadStore);

crates/iam/src/oidc/provider.rs

@@ -16,19 +16,21 @@ use super::{
     transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
 };
 use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
-use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
+use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
 use openidconnect::{
-    AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
+    AsyncHttpClient, ClaimsVerificationError, ClientId, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
+    ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
 };
 use serde::Deserialize;
 use std::collections::HashMap;
-use std::sync::RwLock;
+use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration as StdDuration, Instant};
 use tokio::time::sleep;
 use tracing::{debug, warn};
 use url::Url;
 
 pub(super) const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
+const OIDC_JWKS_REFRESH_COOLDOWN: StdDuration = StdDuration::from_secs(60);
 pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
 pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
 pub(super) const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy";
@@ -96,9 +98,14 @@ pub(super) struct ProviderState {
     pub(super) discovered_at: Instant,
 }
 
+type RefreshAttempt = Option<(Instant, Result<ProviderState, String>)>;
+type ProviderRefreshLock = Arc<tokio::sync::Mutex<RefreshAttempt>>;
+
+#[derive(Clone)]
 pub(super) struct ProviderRuntime {
-    states: RwLock<HashMap<String, ProviderState>>,
+    states: Arc<RwLock<HashMap<String, ProviderState>>>,
-    http_client: ReqwestHttpClient,
+    http_client: Arc<ReqwestHttpClient>,
+    refreshes: Arc<Mutex<HashMap<String, ProviderRefreshLock>>>,
 }
 
 // Workload issuers do not implement the browser authorization flow. Keep their
@@ -199,19 +206,57 @@ impl DiscoveredProviderMetadata {
 
     pub(super) fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> {
         let client_id = ClientId::new(config.client_id.clone());
-        let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone()));
         let (issuer, jwks, algorithms) = match self {
             Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()),
             Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported),
         };
-        let verifier = match secret {
+        // Caller-supplied tokens must prove possession of the issuer's signing
-            Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()),
+        // key. A client secret is shared with clients and cannot provide that proof.
-            None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()),
+        let algorithms = algorithms
-        };
+            .iter()
-        verifier.set_allowed_algs(algorithms.clone())
+            .filter(|algorithm| {
+                !matches!(
+                    algorithm,
+                    CoreJwsSigningAlgorithm::HmacSha256
+                        | CoreJwsSigningAlgorithm::HmacSha384
+                        | CoreJwsSigningAlgorithm::HmacSha512
+                )
+            })
+            .cloned()
+            .collect::<Vec<_>>();
+        CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()).set_allowed_algs(algorithms)
     }
 }
 
+pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
+    token: &'a CoreIdToken,
+    verifier: &CoreIdTokenVerifier<'_>,
+    nonce_verifier: N,
+) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
+    let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
+        CoreJwsSigningAlgorithm::HmacSha256 => 32,
+        CoreJwsSigningAlgorithm::HmacSha384 => 48,
+        CoreJwsSigningAlgorithm::HmacSha512 => 64,
+        _ => return token.claims(verifier, nonce_verifier),
+    };
+    // RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
+    // openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
+    let invalid_length = || {
+        ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
+            "invalid HMAC signature length".to_owned(),
+        ))
+    };
+    let compact = token.to_string();
+    let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
+    let signature = base64_simd::URL_SAFE_NO_PAD
+        .decode_to_vec(signature)
+        .map_err(|_| invalid_length())?;
+    if signature.len() != expected_length {
+        return Err(invalid_length());
+    }
+    token.claims(verifier, nonce_verifier)
+}
+
 // This adapter is used only for discovery/JWKS fetches, never token exchange.
 struct JwksAcceptClient<'a> {
     inner: &'a ReqwestHttpClient,
@@ -243,8 +288,9 @@ impl ProviderState {
 impl ProviderRuntime {
     pub(super) fn new(http_client: ReqwestHttpClient, states: HashMap<String, ProviderState>) -> Self {
         Self {
-            states: RwLock::new(states),
+            states: Arc::new(RwLock::new(states)),
-            http_client,
+            http_client: Arc::new(http_client),
+            refreshes: Arc::new(Mutex::new(HashMap::new())),
         }
     }
 
@@ -296,9 +342,38 @@ impl ProviderRuntime {
         provider_id: &str,
         config: &OidcProviderConfig,
     ) -> Result<ProviderState, String> {
-        self.recover_provider_state(provider_id, config)
+        let refresh = self
+            .refreshes
+            .lock()
+            .map_err(|_| "provider refresh registry is unavailable".to_string())?
+            .entry(provider_id.to_string())
+            .or_default()
+            .clone();
+        let mut attempt = refresh.lock_owned().await;
+        if let Some((completed, result)) = attempt.as_ref()
+            && completed.elapsed() < OIDC_JWKS_REFRESH_COOLDOWN
+        {
+            return result.clone();
+        }
+
+        // The admitted fetch owns the per-provider lock. A disconnected request
+        // cannot abandon discovery while consuming the provider's cooldown.
+        // Waiting callers share its completed result, including genuine failures.
+        let runtime = tokio::runtime::Handle::try_current().map_err(|err| err.to_string())?;
+        let provider_runtime = self.clone();
+        let provider_id = provider_id.to_string();
+        let config = config.clone();
+        runtime
+            .spawn(async move {
+                let result = provider_runtime
+                    .recover_provider_state(&provider_id, &config)
                     .await
-            .map_err(|err| err.to_string())
+                    .map_err(|err| err.to_string());
+                *attempt = Some((Instant::now(), result.clone()));
+                result
+            })
+            .await
+            .map_err(|err| err.to_string())?
     }
 
     pub(super) async fn recover_provider_state(

crates/iam/src/oidc/provider_tests.rs

@@ -180,7 +180,7 @@ fn workload_metadata_rejects_duplicate_fields() {
     }
 }
 #[test]
-fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
+fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
     let secret = Nonce::new_random().secret().to_string();
     let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com");
     config.client_secret = Some(secret.clone());
@@ -195,7 +195,8 @@ fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
     let token: CoreIdToken = signed.parse().unwrap();
     for workload in [false, true] {
         for (algorithms, accepted) in [
-            (serde_json::json!(["HS256", "unsupported-future-algorithm"]), true),
+            (serde_json::json!(["RS256", "HS256"]), false),
+            (serde_json::json!(["HS256", "unsupported-future-algorithm"]), false),
             (serde_json::json!(["ES256"]), false),
             (serde_json::json!(["unsupported-future-algorithm"]), false),
         ] {
@@ -216,21 +217,6 @@ fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
                 accepted,
                 "workload={workload}, algorithms={algorithms}"
             );
-            if accepted {
-                assert!(
-                    token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(),
-                    "additional audiences require explicit trust"
-                );
-                let mut wrong_secret = config.clone();
-                wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string());
-                let verifier = metadata
-                    .verifier(&wrong_secret)
-                    .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-                assert!(
-                    token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
-                    "incorrect client secret must fail"
-                );
-            }
         }
     }
 }
@@ -242,7 +228,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
     let now = time::OffsetDateTime::now_utc().unix_timestamp();
     let payload = serde_json::json!({
         "iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
-        "iat": now, "exp": now + 300,
+        "iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
     });
 
     for (algorithm, name, signature_length) in [
@@ -275,22 +261,41 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
                 .expect("parse configured provider metadata");
             let verifier = metadata.verifier(&config);
             assert!(
-                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
+                "caller-supplied HMAC tokens must be rejected"
+            );
+            // The authorization-code flow obtains tokens directly from the
+            // provider and still supports confidential-client verification.
+            let verifier = openidconnect::core::CoreIdTokenVerifier::new_confidential_client(
+                openidconnect::ClientId::new(config.client_id.clone()),
+                openidconnect::ClientSecret::new(secret.to_owned()),
+                metadata.issuer().clone(),
+                openidconnect::core::CoreJsonWebKeySet::new(vec![]),
+            )
+            .set_allowed_algs(vec![serde_json::from_value(serde_json::json!(name)).expect("parse signing algorithm")]);
+            assert!(
+                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "valid {name} token must pass for workload={workload}"
             );
+            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
+            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
 
             for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
                 let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
                 let malformed: CoreIdToken = format!("{signing_input}.{signature}")
                     .parse()
-                    .expect("invalid signature length must still reach cryptographic verification");
+                    .expect("invalid signature length must remain parseable before verification");
                 assert!(
-                    malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
+                    verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
                     "{name} signature length {length} must return an error for workload={workload}"
                 );
+                assert!(
+                    verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
+                    "{name} signature length {length} must return an error with a nonce for workload={workload}"
+                );
             }
             assert!(
-                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "invalid inputs must not prevent later valid {name} verification"
             );
         }
@@ -774,3 +779,149 @@ async fn test_validate_oidc_provider_config_returns_detailed_errors() {
     assert!(err.contains(&format!("issuer '{base}/application/o/rustfs/'")));
     assert!(handle.join().is_ok());
 }
+
+#[tokio::test]
+async fn jwks_refresh_is_single_flight_and_survives_request_cancellation() {
+    let (initial_key, initial_jwk) = oidc_es256_key_and_jwk("initial");
+    let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated");
+    let (wrong_key, _) = oidc_es256_key_and_jwk("wrong");
+    let requests = Arc::new(AtomicUsize::new(0));
+    let started = Arc::new(tokio::sync::Notify::new());
+    let (release_tx, release_rx) = std::sync::mpsc::channel();
+    let server_requests = Arc::clone(&requests);
+    let server_started = Arc::clone(&started);
+    let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
+        |base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
+        6,
+        "ES256",
+        false,
+        move |fetch| {
+            let request = server_requests.fetch_add(1, Ordering::SeqCst);
+            if request == 2 {
+                server_started.notify_one();
+                release_rx
+                    .recv_timeout(StdDuration::from_secs(5))
+                    .expect("release refresh response");
+            }
+            serde_json::json!({"keys": [if fetch == 0 { &initial_jwk } else { &rotated_jwk }]}).to_string()
+        },
+    ) else {
+        return;
+    };
+    let mut config = build_mocked_oidc_provider_config("bounded-refresh", &base);
+    config.issuer = Some(base.clone());
+    let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
+    let state = discover_provider(&config, &http).await.expect("initial discovery");
+    let sys = Arc::new(OidcSys {
+        configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]),
+        provider_runtime: ProviderRuntime::new(http, HashMap::from([(config.id.clone(), state)])),
+        state_store: OidcStateStore::new(),
+        pending_discovery: Default::default(),
+    });
+    let now = time::OffsetDateTime::now_utc().unix_timestamp();
+    let payload = serde_json::json!({"iss": base, "sub": "user", "aud": config.client_id, "iat": now, "exp": now + 300});
+    let sign = |kid: &str, claims: &serde_json::Value, key: &EncodingKey| {
+        let mut header = Header::new(Algorithm::ES256);
+        header.kid = Some(kid.to_string());
+        jsonwebtoken::encode(&header, claims, key).expect("sign test token")
+    };
+    for (claim, value) in [
+        ("aud", serde_json::json!("wrong-audience")),
+        ("exp", serde_json::json!(now - 300)),
+    ] {
+        let mut invalid = payload.clone();
+        invalid[claim] = value;
+        assert!(
+            sys.verify_web_identity_token(&sign("initial", &invalid, &initial_key))
+                .await
+                .is_err()
+        );
+    }
+    assert_eq!(requests.load(Ordering::SeqCst), 2, "claim failures cannot refresh signing keys");
+    let invalid = sign("rotated", &payload, &wrong_key);
+    let attack = {
+        let sys = Arc::clone(&sys);
+        let invalid = invalid.clone();
+        tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await })
+    };
+    tokio::time::timeout(StdDuration::from_secs(5), started.notified())
+        .await
+        .expect("refresh started");
+    attack.abort();
+    assert!(attack.await.expect_err("request canceled").is_cancelled());
+    let mut retries = Vec::new();
+    for _ in 0..16 {
+        let sys = Arc::clone(&sys);
+        let invalid = invalid.clone();
+        retries.push(tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await }));
+    }
+    release_tx.send(()).expect("release provider");
+    for retry in retries {
+        assert!(
+            tokio::time::timeout(StdDuration::from_secs(5), retry)
+                .await
+                .expect("retry timeout")
+                .expect("retry join")
+                .is_err()
+        );
+    }
+    let rotated = sign("rotated", &payload, &rotated_key);
+    sys.verify_web_identity_token(&rotated)
+        .await
+        .expect("completed refresh must accept rotated keys after the original caller left");
+    assert_eq!(requests.load(Ordering::SeqCst), 4, "one discovery/JWKS pair for all rejected tokens");
+    sys.provider_runtime
+        .refresh_provider_state("independent-provider", &config)
+        .await
+        .expect("provider cooldowns must be independent");
+    assert_eq!(requests.load(Ordering::SeqCst), 6);
+    server.join().expect("mock server");
+}
+
+#[tokio::test]
+async fn jwks_refresh_cools_down_failures_and_recovers_after_the_interval() {
+    let (_, initial_jwk) = oidc_es256_key_and_jwk("initial");
+    let (_, recovered_jwk) = oidc_es256_key_and_jwk("recovered");
+    let requests = Arc::new(AtomicUsize::new(0));
+    let server_requests = Arc::clone(&requests);
+    let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
+        |base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
+        6,
+        "ES256",
+        false,
+        move |fetch| {
+            server_requests.fetch_add(1, Ordering::SeqCst);
+            match fetch {
+                0 => serde_json::json!({"keys": [initial_jwk]}).to_string(),
+                1 => "invalid JWKS".to_string(),
+                _ => serde_json::json!({"keys": [recovered_jwk]}).to_string(),
+            }
+        },
+    ) else {
+        return;
+    };
+    let mut config = build_mocked_oidc_provider_config("recovering-refresh", &base);
+    config.issuer = Some(base.clone());
+    let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
+    let initial = discover_provider(&config, &http).await.expect("initial discovery");
+    let runtime = ProviderRuntime::new(http, HashMap::from([(config.id.clone(), initial.clone())]));
+    assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
+    assert_eq!(requests.load(Ordering::SeqCst), 4);
+    for _ in 0..8 {
+        assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
+    }
+    assert_eq!(requests.load(Ordering::SeqCst), 4, "completed failures also consume a bounded cooldown");
+    assert_eq!(
+        runtime.get_provider_state(&config.id).expect("cached keys").discovered_at,
+        initial.discovered_at
+    );
+    let refresh = runtime.refreshes.lock().expect("registry")[&config.id].clone();
+    refresh.lock().await.as_mut().expect("completed attempt").0 = Instant::now() - OIDC_JWKS_REFRESH_COOLDOWN;
+    runtime
+        .refresh_provider_state(&config.id, &config)
+        .await
+        .expect("retry after cooldown");
+    assert_eq!(requests.load(Ordering::SeqCst), 6);
+    assert!(runtime.get_provider_state(&config.id).expect("recovered keys").discovered_at > initial.discovered_at);
+    server.join().expect("mock server");
+}

crates/iam/src/oidc/runtime.rs

@@ -16,7 +16,7 @@ use super::{
         OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
         SourcedOidcProviderConfig, load_oidc_config_snapshot,
     },
-    provider::{ProviderRuntime, discover_provider_for_recovery},
+    provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
     state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
     transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
 };
@@ -33,8 +33,8 @@ use crate::{
 };
 use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
 use openidconnect::{
-    Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce, PkceCodeChallenge,
+    Audience, AuthType, AuthorizationCode, ClaimsVerificationError, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce,
-    PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
+    PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
 };
 use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
 use rustfs_utils::logging::LogThrottle;
@@ -535,8 +535,11 @@ impl OidcSys {
         let verifier = client
             .id_token_verifier()
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
+        let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
         if let Err(e) = verified {
+            if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
+                return Err(format!("ID token verification failed: {e}"));
+            }
             let verification_error = e.to_string();
             warn!(
                 event = EVENT_OIDC_DIAGNOSTICS,
@@ -583,8 +586,7 @@ impl OidcSys {
             let verifier = client
                 .id_token_verifier()
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            id_token
+            verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
-                .claims(&verifier, &Nonce::new(session.nonce.clone()))
                 .map_err(|retry_err| {
                     error!(
                         event = EVENT_OIDC_DIAGNOSTICS,
@@ -767,7 +769,10 @@ impl OidcSys {
             .metadata
             .verifier(&config)
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
+        if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
+            if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
+                return Err(format!("ID token verification failed: {e}"));
+            }
             state = self
                 .provider_runtime
                 .refresh_provider_state(&provider_id, &config)
@@ -780,8 +785,7 @@ impl OidcSys {
                 .metadata
                 .verifier(&config)
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            id_token
+            verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
-                .claims(&verifier, |_: Option<&Nonce>| Ok(()))
                 .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
         }
 

crates/iam/src/storage_api.rs

@@ -14,6 +14,20 @@
 
 use std::sync::Arc;
 
+pub(crate) fn is_unreadable_bucket_tags(error: &crate::error::Error) -> bool {
+    let crate::error::Error::Io(error) = error else {
+        return false;
+    };
+    let Some(storage_error) = error.get_ref().and_then(|inner| inner.downcast_ref::<IamStorageError>()) else {
+        return false;
+    };
+    rustfs_ecstore::api::bucket::metadata::unreadable_config_refusal(storage_error)
+        .is_some_and(|failure| failure.config_file == rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG)
+}
+
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::api::bucket::metadata::UnreadableBucketConfig;
+
 pub(crate) use rustfs_ecstore::api::config::RUSTFS_CONFIG_PREFIX as IAM_CONFIG_ROOT_PREFIX;
 use rustfs_ecstore::api::config::com::{
     delete_config as ecstore_delete_config, read_config_no_lock as ecstore_read_config_no_lock,

crates/iam/src/store/object.rs

@@ -1154,7 +1154,7 @@ impl Store for ObjectStore {
         // node-counted lock quorum (rustfs#4304; startup contract rustfs#4056).
         const LOAD_ALL_MODE: LoadMode = LoadMode::BootstrapNoLock;
 
-        let cache_snapshot = cache.snapshot();
+        let reload = cache.begin_reload().await;
         let listed_config_items = self.list_all_iamconfig_items().await?;
 
         let mut policy_docs_cache = CacheEntity::new(get_default_policies());
@@ -1201,26 +1201,20 @@ impl Store for ObjectStore {
         }
 
         // groups
-        let mut groups_cache = None;
+        let mut groups_cache = CacheEntity::default();
         if let Some(item_name_list) = listed_config_items.get(GROUPS_LIST_KEY) {
-            let mut items_cache = CacheEntity::default();
-
             for item in item_name_list.iter() {
                 let name = rustfs_utils::path::dir(item);
                 debug!(group = %name, "IAM group loaded");
-                if let Err(err) = self.load_group_with(&name, &mut items_cache, LOAD_ALL_MODE).await {
+                if let Err(err) = self.load_group_with(&name, &mut groups_cache, LOAD_ALL_MODE).await {
                     return Err(Error::other(format!("load group failed: {err}")));
                 };
             }
-
-            groups_cache = Some(items_cache);
         }
 
         // user policies
-        let mut user_policies_cache = None;
+        let mut user_policies_cache = CacheEntity::default();
         if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) {
-            let mut items_cache = CacheEntity::default();
-
             // Load in fixed-size chunks so each mapping is fetched exactly once
             // (backlog#806 — the split_off loop was O(n^2)).
             for chunk in item_name_list.chunks(32) {
@@ -1235,32 +1229,26 @@ impl Store for ObjectStore {
 
                     let name = chunk[idx].trim_end_matches(".json").to_owned();
                     debug!(user = %name, "IAM user policy loaded");
-                    items_cache.insert(name, p);
+                    user_policies_cache.insert(name, p);
                 }
             }
-
-            user_policies_cache = Some(items_cache);
         }
 
         // group policy
-        let mut group_policies_cache = None;
+        let mut group_policies_cache = CacheEntity::default();
         if let Some(item_name_list) = listed_config_items.get(POLICY_DB_GROUPS_LIST_KEY) {
-            let mut items_cache = CacheEntity::default();
-
             for item in item_name_list.iter() {
                 let name = item.trim_end_matches(".json");
 
                 debug!(group = %name, "IAM group policy loaded");
                 if let Err(err) = self
-                    .load_mapped_policy_with(name, UserType::Reg, true, &mut items_cache, LOAD_ALL_MODE)
+                    .load_mapped_policy_with(name, UserType::Reg, true, &mut group_policies_cache, LOAD_ALL_MODE)
                     .await
                     && !is_err_no_such_policy(&err)
                 {
                     return Err(Error::other(format!("load group policy failed: {err}")));
                 };
             }
-
-            group_policies_cache = Some(items_cache);
         }
 
         let mut sts_policies_cache = CacheEntity::default();
@@ -1330,25 +1318,48 @@ impl Store for ObjectStore {
             }
         }
 
+        #[cfg(test)]
+        if let Ok(barrier) = LOAD_ALL_COMMIT_BARRIER.try_with(std::sync::Arc::clone) {
+            barrier.0.notify_one();
+            barrier.1.notified().await;
+        }
+
         cache.with_write_lock(|cache| {
-            if cache.matches_snapshot(&cache_snapshot) {
+            reload.merge_partition(
+                crate::cache::CachePartition::PolicyDocs,
+                &mut policy_docs_cache,
+                &cache.state().policy_docs,
+            );
+            reload.merge_partition(crate::cache::CachePartition::Groups, &mut groups_cache, &cache.state().groups);
+            reload.merge_partition(
+                crate::cache::CachePartition::UserPolicies,
+                &mut user_policies_cache,
+                &cache.state().user_policies,
+            );
+            reload.merge_partition(
+                crate::cache::CachePartition::GroupPolicies,
+                &mut group_policies_cache,
+                &cache.state().group_policies,
+            );
+            reload.merge_partition(crate::cache::CachePartition::Users, &mut user_items_cache, &cache.state().users);
+            reload.merge_partition(
+                crate::cache::CachePartition::StsAccounts,
+                &mut sts_items_cache,
+                &cache.state().sts_accounts,
+            );
+            reload.merge_partition(
+                crate::cache::CachePartition::StsPolicies,
+                &mut sts_policies_cache,
+                &cache.state().sts_policies,
+            );
             cache.replace_policy_docs(policy_docs_cache);
-                if let Some(groups_cache) = groups_cache {
             cache.replace_groups(groups_cache);
-                }
-                if let Some(user_policies_cache) = user_policies_cache {
             cache.replace_user_policies(user_policies_cache);
-                }
-                if let Some(group_policies_cache) = group_policies_cache {
             cache.replace_group_policies(group_policies_cache);
-                }
             cache.replace_users(user_items_cache);
             cache.replace_sts_accounts(sts_items_cache);
             cache.replace_sts_policies(sts_policies_cache);
             cache.build_user_group_memberships();
-            } else {
-                warn!("IAM full reload cache commit skipped due to concurrent cache changes");
-            }
         });
 
         Ok(())
@@ -1363,6 +1374,11 @@ fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
     }
 }
 
+#[cfg(test)]
+tokio::task_local! {
+    static LOAD_ALL_COMMIT_BARRIER: std::sync::Arc<(tokio::sync::Notify, tokio::sync::Notify)>;
+}
+
 #[cfg(test)]
 mod tests {
     use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
@@ -1372,6 +1388,99 @@ mod tests {
     use serial_test::serial;
     use temp_env::with_vars;
 
+    #[test]
+    #[serial]
+    fn test_load_all_clears_partitions_with_empty_prefix() {
+        use crate::cache::Cache;
+        use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
+
+        std::thread::Builder::new()
+            .stack_size(8 * 1024 * 1024)
+            .spawn(|| {
+                let runtime = tokio::runtime::Builder::new_multi_thread()
+                    .worker_threads(2)
+                    .enable_all()
+                    .build()
+                    .expect("build IAM reload test runtime");
+                runtime.block_on(async {
+                    let _credentials = test_cred();
+                    let temp = tempfile::tempdir().expect("create IAM reload store directory");
+                    let env = Box::pin(
+                        rustfs_test_utils::TestECStoreEnv::builder()
+                            .base_dir(temp.path())
+                            .init_bucket_metadata(false)
+                            .build(),
+                    )
+                    .await;
+                    let store = ObjectStore::new(env.ecstore);
+                    store
+                        .save_group_info("reload-group", GroupInfo::new(vec!["reload-user".to_string()]))
+                        .await
+                        .expect("seed group membership");
+                    store
+                        .save_mapped_policy("reload-user", UserType::Reg, false, MappedPolicy::new("readwrite"), None)
+                        .await
+                        .expect("seed user policy");
+                    store
+                        .save_mapped_policy("reload-group", UserType::Reg, true, MappedPolicy::new("readonly"), None)
+                        .await
+                        .expect("seed group policy");
+
+                    let cache = Cache::default();
+                    store.load_all(&cache).await.expect("load nonempty IAM partitions");
+                    {
+                        let snapshot = cache.snapshot();
+                        assert!(snapshot.groups.contains_key("reload-group"));
+                        assert_eq!(snapshot.user_policies["reload-user"].policies, "readwrite");
+                        assert_eq!(snapshot.group_policies["reload-group"].policies, "readonly");
+                        assert!(snapshot.user_group_memberships["reload-user"].contains("reload-group"));
+                    }
+
+                    // Mutate durable state without notifying this cache, as on
+                    // a peer that missed the last mapping/group deletion.
+                    store
+                        .delete_mapped_policy("reload-user", UserType::Reg, false)
+                        .await
+                        .expect("remove last user mapping");
+                    store
+                        .delete_mapped_policy("reload-group", UserType::Reg, true)
+                        .await
+                        .expect("remove last group mapping");
+                    store.delete_group_info("reload-group").await.expect("remove last group");
+                    assert!(cache.snapshot().groups.contains_key("reload-group"));
+                    // Simulate a missed peer revocation while STS publication
+                    // and a separate user update overlap this full disk reload.
+                    let stale = rustfs_policy::auth::UserIdentity {
+                        version: 1,
+                        ..Default::default()
+                    };
+                    cache.add_or_update_user("revoked-user", &stale, time::OffsetDateTime::now_utc());
+                    let barrier = std::sync::Arc::new((tokio::sync::Notify::new(), tokio::sync::Notify::new()));
+                    let loading = super::LOAD_ALL_COMMIT_BARRIER.scope(std::sync::Arc::clone(&barrier), store.load_all(&cache));
+                    let concurrent = async {
+                        barrier.0.notified().await;
+                        cache.add_or_update_sts_account("new-session", &stale, time::OffsetDateTime::now_utc());
+                        cache.add_or_update_user("new-service", &stale, time::OffsetDateTime::now_utc());
+                        barrier.1.notify_one();
+                    };
+                    let (loaded, ()) = tokio::join!(loading, concurrent);
+                    loaded.expect("reconcile revoked identities despite STS traffic");
+                    let snapshot = cache.snapshot();
+                    assert!(!snapshot.users.contains_key("revoked-user"));
+                    assert!(snapshot.users.contains_key("new-service"));
+                    assert!(snapshot.sts_accounts.contains_key("new-session"));
+                    assert!(snapshot.groups.is_empty());
+                    assert!(snapshot.user_policies.is_empty());
+                    assert!(snapshot.group_policies.is_empty());
+                    assert!(snapshot.user_group_memberships.is_empty());
+                    assert!(!snapshot.policy_docs.is_empty(), "built-in policies must remain available");
+                });
+            })
+            .expect("spawn IAM reload test thread")
+            .join()
+            .expect("IAM reload test should complete");
+    }
+
     /// rustfs#4304 / startup contract rustfs#4056: the bulk snapshot load
     /// (`load_all`) must never depend on the node-counted namespace lock
     /// quorum, so its reads have to carry `no_lock = true` down to the

crates/io-core/Cargo.toml

@@ -42,6 +42,7 @@ rustfs-io-metrics = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
+metrics = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
 
 [lib]

crates/io-core/src/pool.rs

@@ -83,7 +83,10 @@ struct PoolTier {
     /// Queue of available buffers for reuse
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
-    metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
+    metrics: Arc<BytesPoolMetrics>,
+    /// Serialize acquisitions before taking buffers so a slow recorder does not
+    /// accumulate allocated buffers behind hit-rate publication.
+    acquire_metrics_lock: Mutex<()>,
     /// Total acquisitions for this tier
     tier_total_acquires: AtomicU64,
     /// Total hits for this tier
@@ -185,16 +188,10 @@ impl BytesPool {
     /// ```
     pub fn with_config(config: BytesPoolConfig) -> Self {
         let metrics = Arc::new(BytesPoolMetrics::default());
-        let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small"));
+        let small_pool = Arc::new(PoolTier::new(config.small_size, config.small_max, "small", Arc::clone(&metrics)));
-        let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium"));
+        let medium_pool = Arc::new(PoolTier::new(config.medium_size, config.medium_max, "medium", Arc::clone(&metrics)));
-        let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large"));
+        let large_pool = Arc::new(PoolTier::new(config.large_size, config.large_max, "large", Arc::clone(&metrics)));
-        let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge"));
+        let xlarge_pool = Arc::new(PoolTier::new(config.xlarge_size, config.xlarge_max, "xlarge", Arc::clone(&metrics)));
-
-        // Set metrics reference in all tiers
-        small_pool.set_metrics(Arc::clone(&metrics));
-        medium_pool.set_metrics(Arc::clone(&metrics));
-        large_pool.set_metrics(Arc::clone(&metrics));
-        xlarge_pool.set_metrics(Arc::clone(&metrics));
 
         Self {
             small_pool,
@@ -225,7 +222,7 @@ impl BytesPool {
     /// ```
     pub async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
         let tier = self.select_tier(size);
-        let mut buffer = tier.acquire_buffer(size, &self.metrics).await;
+        let mut buffer = tier.acquire_buffer(size).await;
         if buffer._permit.is_some() {
             buffer.tier = Some(Arc::clone(tier));
         }
@@ -252,7 +249,7 @@ impl BytesPool {
     /// ```
     pub fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
         let tier = self.select_tier(size);
-        let mut buffer = tier.try_acquire_buffer(size, &self.metrics)?;
+        let mut buffer = tier.try_acquire_buffer(size)?;
         // Set tier reference for return on drop
         buffer.tier = Some(Arc::clone(tier));
         Some(buffer)
@@ -290,43 +287,38 @@ impl BytesPool {
 }
 
 impl PoolTier {
-    fn new(buffer_size: usize, max_buffers: usize, name: &'static str) -> Self {
+    fn new(buffer_size: usize, max_buffers: usize, name: &'static str, metrics: Arc<BytesPoolMetrics>) -> Self {
         Self {
             buffer_size,
             max_buffers,
             semaphore: Arc::new(Semaphore::new(max_buffers)),
             name,
             available_buffers: Mutex::new(Vec::new()),
-            metrics: Mutex::new(None),
+            metrics,
+            acquire_metrics_lock: Mutex::new(()),
             tier_total_acquires: AtomicU64::new(0),
             tier_pool_hits: AtomicU64::new(0),
             tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
-    fn set_metrics(&self, metrics: Arc<BytesPoolMetrics>) {
+    fn take_or_allocate_buffer(&self, size: usize) -> (BytesMut, bool) {
-        *self.metrics.lock().unwrap_or_else(|e| e.into_inner()) = Some(metrics);
+        let pool_metrics = &self.metrics;
-    }
-
-    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
         let buffer_opt = {
             let mut available = self.available_buffers.lock().unwrap_or_else(|e| e.into_inner());
-            available.pop()
+            let buffer = available.pop();
-        };
+            if buffer.is_some() {
-        let was_reused = buffer_opt.is_some();
-        if was_reused {
-            // A reused buffer leaves the available pool. `return_buffer` does a
-            // matching `fetch_add(1)` on push, so without this the
-            // `available_buffers` gauge only ever grows and never reflects the real
-            // pool size (backlog#806).
                 pool_metrics.available_buffers.fetch_sub(1, Ordering::Relaxed);
             }
+            buffer
+        };
+        let was_reused = buffer_opt.is_some();
 
         let buffer = if let Some(mut buf) = buffer_opt {
             let previous_capacity = buf.capacity();
             buf.clear();
             if previous_capacity < size {
-                buf.reserve(size - previous_capacity);
+                buf.reserve(size);
             }
             let current_capacity = buf.capacity();
             if current_capacity > previous_capacity {
@@ -353,7 +345,10 @@ impl PoolTier {
         (buffer, was_reused)
     }
 
-    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
+    fn record_acquire_metrics(&self, buffer_capacity: usize, was_reused: bool) {
+        let pool_metrics = &self.metrics;
+        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
         rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
 
         if was_reused {
@@ -374,7 +369,7 @@ impl PoolTier {
         rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 
-    async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
+    async fn acquire_buffer(&self, size: usize) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = match Arc::clone(&self.semaphore).acquire_owned().await {
             Ok(p) => p,
@@ -388,17 +383,10 @@ impl PoolTier {
             }
         };
 
-        // Use the pool's shared metrics for recording
+        let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
-        let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size);
-        let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
-
-        // Record acquisition
-        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
-        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
-
-        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
+        self.record_acquire_metrics(buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -407,21 +395,14 @@ impl PoolTier {
         }
     }
 
-    fn try_acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> Option<PooledBuffer> {
+    fn try_acquire_buffer(&self, size: usize) -> Option<PooledBuffer> {
         // Try to acquire permit without blocking
         let permit = Arc::clone(&self.semaphore).try_acquire_owned().ok()?;
 
-        // Use the pool's shared metrics for recording
+        let _guard = self.acquire_metrics_lock.lock().unwrap_or_else(|e| e.into_inner());
-        let _metrics_lock = self.metrics.lock().unwrap_or_else(|e| e.into_inner());
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size);
-        let _metrics = _metrics_lock.as_ref().expect("operation should succeed");
-
-        // Record acquisition
-        pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
-        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
-
-        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
+        self.record_acquire_metrics(buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -430,7 +411,7 @@ impl PoolTier {
         })
     }
 
-    /// Return a buffer to the pool for reuse without ever blocking the caller.
+    /// Return a buffer to the pool for reuse without waiting for the queue lock.
     fn return_buffer(&self, buffer: BytesMut) {
         let mut buffer = Some(buffer);
 
@@ -438,11 +419,7 @@ impl PoolTier {
             && available.len() < self.max_buffers
         {
             available.push(buffer.take().expect("buffer should be present until returned"));
-            if let Ok(metrics) = self.metrics.try_lock()
+            self.metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
-                && let Some(metrics) = metrics.as_ref()
-            {
-                metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
-            }
         }
 
         if let Some(buffer) = buffer {
@@ -452,17 +429,13 @@ impl PoolTier {
                     Some(current.saturating_sub(released_bytes))
                 })
                 .ok();
-            if let Ok(metrics) = self.metrics.try_lock()
+            self.metrics
-                && let Some(metrics) = metrics.as_ref()
-            {
-                metrics
                 .current_allocated_bytes
                 .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
                     Some(current.saturating_sub(released_bytes))
                 })
                 .ok();
         }
-        }
         rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
@@ -551,6 +524,44 @@ mod tests {
         assert!(buffer.capacity() >= 2048);
     }
 
+    #[tokio::test]
+    async fn reused_buffer_satisfies_larger_async_request() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_max: 1,
+            ..Default::default()
+        });
+        let mut first = pool.acquire_buffer(32 * 1024).await;
+        first.extend_from_slice(b"previous contents");
+        assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
+        drop(first);
+
+        let reused = pool.acquire_buffer(48 * 1024).await;
+        assert!(reused.is_empty());
+        assert!(reused.capacity() >= 48 * 1024);
+        assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
+        assert_eq!(
+            pool.metrics().current_allocated_bytes.load(Ordering::Relaxed),
+            u64::try_from(reused.capacity()).expect("buffer capacity fits in metrics")
+        );
+    }
+
+    #[test]
+    fn reused_buffer_satisfies_larger_try_request() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_max: 1,
+            ..Default::default()
+        });
+        let mut first = pool.try_acquire_buffer(32 * 1024).expect("first buffer is available");
+        first.extend_from_slice(b"previous contents");
+        assert!(first.capacity() < 48 * 1024, "fixture must require buffer growth");
+        drop(first);
+
+        let reused = pool.try_acquire_buffer(48 * 1024).expect("returned buffer is available");
+        assert!(reused.is_empty());
+        assert!(reused.capacity() >= 48 * 1024);
+        assert_eq!(pool.metrics().pool_hits.load(Ordering::Relaxed), 1);
+    }
+
     #[tokio::test]
     async fn available_buffers_gauge_decrements_on_reuse() {
         // Regression (backlog#806): acquiring a pooled (reused) buffer must

crates/io-core/tests/bytes_pool_metrics_concurrency.rs

@@ -0,0 +1,121 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Mutex, mpsc};
+use std::thread;
+use std::time::Duration;
+
+use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
+use rustfs_io_core::{BytesPool, BytesPoolConfig};
+
+const WAIT: Duration = Duration::from_secs(5);
+const GATE_WAIT: Duration = Duration::from_secs(20);
+
+struct AcquisitionGate {
+    entered: mpsc::Sender<()>,
+    release: Mutex<mpsc::Receiver<()>>,
+}
+
+impl Recorder for AcquisitionGate {
+    fn describe_counter(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
+
+    fn describe_gauge(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
+
+    fn describe_histogram(&self, _key: KeyName, _unit: Option<Unit>, _description: SharedString) {}
+
+    fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
+        if key.name() == "rustfs_bytes_pool_acquisitions_total" {
+            self.entered
+                .send(())
+                .expect("notify test that acquisition reached the recorder");
+            self.release
+                .lock()
+                .expect("lock acquisition gate")
+                .recv_timeout(GATE_WAIT)
+                .expect("release the acquisition recorder gate");
+        }
+        Counter::noop()
+    }
+
+    fn register_gauge(&self, _key: &Key, _metadata: &Metadata<'_>) -> Gauge {
+        Gauge::noop()
+    }
+
+    fn register_histogram(&self, _key: &Key, _metadata: &Metadata<'_>) -> Histogram {
+        Histogram::noop()
+    }
+}
+
+#[test]
+fn returned_buffer_is_counted_while_an_acquisition_recorder_is_busy() {
+    let pool = BytesPool::with_config(BytesPoolConfig {
+        small_max: 2,
+        ..Default::default()
+    });
+    let held = pool.try_acquire_buffer(8 * 1024).expect("acquire the buffer to return");
+    assert_eq!(pool.available_buffers(), 0);
+
+    // This integration test is a dedicated test process: the recorder is local
+    // to the worker, while the metrics enablement flag is process-wide.
+    rustfs_io_metrics::set_metrics_enabled(true);
+    let (entered_tx, entered_rx) = mpsc::channel();
+    let (release_tx, release_rx) = mpsc::channel();
+    let worker_pool = pool.clone();
+    let acquirer = thread::spawn(move || {
+        let gate = AcquisitionGate {
+            entered: entered_tx,
+            release: Mutex::new(release_rx),
+        };
+        metrics::with_local_recorder(&gate, || {
+            worker_pool
+                .try_acquire_buffer(8 * 1024)
+                .expect("acquire a second same-tier buffer")
+        })
+    });
+
+    let entered = entered_rx.recv_timeout(WAIT);
+    if entered.is_err() {
+        let _ = release_tx.send(());
+        let _ = acquirer.join();
+        panic!("acquisition did not reach the recorder gate: {entered:?}");
+    }
+
+    let (dropped_tx, dropped_rx) = mpsc::channel();
+    let returner = thread::spawn(move || {
+        drop(held);
+        let _ = dropped_tx.send(());
+    });
+    let returned_while_blocked = dropped_rx.recv_timeout(WAIT);
+
+    // Always release before joining so a blocking Drop cannot deadlock the test.
+    let _ = release_tx.send(());
+    let returned = returner.join();
+    let acquired = acquirer.join();
+    rustfs_io_metrics::set_metrics_enabled(false);
+
+    assert!(returned_while_blocked.is_ok(), "returning a buffer must not wait on the recorder");
+    returned.expect("return worker should finish");
+    let acquired = acquired.expect("acquisition worker should finish");
+    let after_return = pool.available_buffers();
+    let reused = pool.try_acquire_buffer(8 * 1024).expect("reuse the returned buffer");
+    let after_reuse = pool.available_buffers();
+    drop(reused);
+    drop(acquired);
+    let after_both_return = pool.available_buffers();
+    assert_eq!(
+        (after_return, after_reuse, after_both_return),
+        (1, 0, 2),
+        "returned buffers must be counted, and reuse must not underflow the available gauge"
+    );
+}

crates/io-metrics/src/internode_metrics.rs

@@ -1411,6 +1411,28 @@ pub fn record_peer_reachable(addr: &str) {
     publish_offline_gauge(&peers);
 }
 
+/// Record a successful interaction with a cluster peer, but only when the peer is currently
+/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
+/// and lock-path successes that do not go through a fresh dial.
+///
+/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
+/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
+/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
+pub fn record_peer_reachable_if_offline(addr: &str) {
+    let key = normalize_peer_key(addr);
+    // Fast path: read lock to check whether the peer is currently offline.
+    {
+        let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
+        match peers.get(key) {
+            None => return,
+            Some(entry) if entry.online => return,
+            Some(_) => {} // offline: fall through to the write path
+        }
+    }
+    // Slow path: the peer is offline, so flip it back online and republish the gauge.
+    record_peer_reachable(addr);
+}
+
 /// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
 /// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
 pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
@@ -2188,6 +2210,34 @@ mod tests {
         assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
     }
 
+    #[test]
+    fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
+        let addr = "http://cluster-peer-recover-if-offline-test:9000";
+
+        // Mark the peer offline.
+        record_peer_unreachable(addr, 1);
+        assert!(cluster_peer_is_offline(addr));
+
+        // A successful cached-channel RPC (no fresh dial) clears the offline state.
+        record_peer_reachable_if_offline(addr);
+        assert!(!cluster_peer_is_offline(addr));
+        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
+    }
+
+    #[test]
+    fn record_peer_reachable_if_offline_no_op_for_online_peer() {
+        let addr = "http://cluster-peer-recover-noop-test:9000";
+
+        record_peer_reachable(addr);
+        // Should not panic or change state for an already-online peer.
+        record_peer_reachable_if_offline(addr);
+        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
+
+        // Should also be a no-op for unknown peers.
+        record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
+        assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
+    }
+
     #[test]
     fn cluster_peer_should_bypass_is_self_healing() {
         let addr = "http://cluster-peer-bypass-selfheal-test:9000";

crates/keystone/src/auth.rs

@@ -85,6 +85,17 @@ impl KeystoneAuthProvider {
         Ok(self.keystone_token_to_credentials(&keystone_token))
     }
 
+    /// Revalidate a long-lived subscription directly with Keystone, bypassing
+    /// the request cache so token revocation terminates an existing stream.
+    /// Returns the same authentication and expiry errors as a new token login.
+    pub async fn revalidate_subscription_token(&self, token: &str) -> Result<Credentials> {
+        let validated = self.client.validate_token(token).await?;
+        if validated.is_expired() {
+            return Err(KeystoneError::TokenExpired);
+        }
+        Ok(self.keystone_token_to_credentials(&validated))
+    }
+
     /// Authenticate using EC2 credentials (S3 API with AWS SigV4)
     pub async fn authenticate_with_ec2(&self, access_key: &str, signature: &str, string_to_sign: &str) -> Result<Credentials> {
         // Check cache

crates/keystone/src/client.rs

@@ -121,7 +121,7 @@ impl KeystoneClient {
             .await
             .map_err(|e| {
                 error!("Failed to send token validation request: {}", e);
-                KeystoneError::HttpError(e.to_string())
+                KeystoneError::HttpError(e.without_url().to_string())
             })?;
 
         let status = response.status();
@@ -138,7 +138,10 @@ impl KeystoneClient {
             )));
         }
 
-        let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
+        let body: serde_json::Value = response
+            .json()
+            .await
+            .map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
 
         self.parse_token_v3(&body)
     }
@@ -260,13 +263,16 @@ impl KeystoneClient {
             .header("X-Auth-Token", admin_token)
             .send()
             .await
-            .map_err(|e| KeystoneError::HttpError(e.to_string()))?;
+            .map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
 
         if !response.status().is_success() {
             return Ok(vec![]);
         }
 
-        let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
+        let body: serde_json::Value = response
+            .json()
+            .await
+            .map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
 
         let credentials = body
             .get("credentials")
@@ -365,7 +371,7 @@ impl KeystoneClient {
             .json(&auth_payload)
             .send()
             .await
-            .map_err(|e| KeystoneError::HttpError(e.to_string()))?;
+            .map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
 
         if !response.status().is_success() {
             return Err(KeystoneError::AuthenticationFailed("Admin authentication failed".to_string()));
@@ -379,7 +385,10 @@ impl KeystoneClient {
             .to_string();
 
         // Parse expiration from response body
-        let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
+        let body: serde_json::Value = response
+            .json()
+            .await
+            .map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
 
         let expires_at = body
             .get("token")
@@ -410,6 +419,133 @@ impl KeystoneClient {
 mod tests {
     use super::*;
 
+    #[tokio::test]
+    async fn subscription_token_refresh_bypasses_a_still_valid_login_cache() {
+        use std::io::{Read, Write};
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("fake Keystone");
+        let endpoint = format!("http://{}", listener.local_addr().unwrap());
+        let expires = (OffsetDateTime::now_utc() + time::Duration::hours(1))
+            .format(&time::format_description::well_known::Rfc3339)
+            .unwrap();
+        let body = json!({"token":{"user":{"id":"listener","name":"listener"},
+            "roles":[{"name":"Member"}],"expires_at":expires}})
+        .to_string();
+        let server = std::thread::spawn(move || {
+            for response in [
+                format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                ),
+                "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
+            ] {
+                let (mut socket, _) = listener.accept().expect("token validation request");
+                socket.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
+                let mut request = [0; 4096];
+                let length = socket.read(&mut request).expect("request headers");
+                assert!(String::from_utf8_lossy(&request[..length]).contains("subscription-token"));
+                socket.write_all(response.as_bytes()).expect("validation response");
+            }
+        });
+        let mut client = KeystoneClient::new(
+            endpoint,
+            KeystoneVersion::V3,
+            None,
+            None,
+            None,
+            "Default".into(),
+            true,
+            std::time::Duration::from_secs(5),
+        );
+        client.client = reqwest::Client::builder()
+            .no_proxy()
+            .timeout(std::time::Duration::from_secs(5))
+            .build()
+            .unwrap();
+        let provider = crate::KeystoneAuthProvider::new(client, 10, std::time::Duration::from_secs(60), true);
+        assert!(provider.authenticate_with_token("subscription-token").await.is_ok());
+        assert!(
+            provider.authenticate_with_token("subscription-token").await.is_ok(),
+            "ordinary login still uses the valid cached token"
+        );
+        assert!(matches!(
+            provider.revalidate_subscription_token("subscription-token").await,
+            Err(KeystoneError::InvalidToken)
+        ));
+        server.join().expect("fake Keystone finished");
+    }
+
+    #[tokio::test]
+    async fn middleware_keystone_failures_do_not_disclose_upstream_locations() {
+        use http_body_util::{BodyExt, Full};
+        use std::io::{Read, Write};
+        use tower::{Layer, ServiceExt};
+
+        for (response, details) in [
+            (None, "Keystone token validation failed"),
+            (
+                Some("HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nnot-json"),
+                "Keystone token validation failed",
+            ),
+            (
+                Some("HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"),
+                "Invalid token",
+            ),
+        ] {
+            for header in ["X-Auth-Token", "X-Storage-Token"] {
+                let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind fake Keystone");
+                let endpoint = format!("http://{}/private-deployment", listener.local_addr().unwrap());
+                let server = response.map(|response| {
+                    std::thread::spawn(move || {
+                        let (mut stream, _) = listener.accept().expect("Keystone request");
+                        stream.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
+                        let mut request = [0; 4096];
+                        assert!(stream.read(&mut request).expect("read token validation request") > 0);
+                        stream.write_all(response.as_bytes()).expect("send fake response");
+                    })
+                });
+                let mut client = KeystoneClient::new(
+                    endpoint.clone(),
+                    KeystoneVersion::V3,
+                    None,
+                    None,
+                    None,
+                    "Default".to_owned(),
+                    true,
+                    std::time::Duration::from_secs(5),
+                );
+                client.client = reqwest::Client::builder()
+                    .no_proxy()
+                    .timeout(std::time::Duration::from_secs(5))
+                    .build()
+                    .unwrap();
+                let provider = crate::KeystoneAuthProvider::new(client, 1, std::time::Duration::from_secs(1), false);
+                let inner = tower::service_fn(|_| async {
+                    Ok::<_, std::convert::Infallible>(http::Response::new(Full::new(bytes::Bytes::new())))
+                });
+                let response = crate::middleware::KeystoneAuthLayer::new(Some(Arc::new(provider)))
+                    .layer(inner)
+                    .oneshot(
+                        http::Request::builder()
+                            .header(header, "private-token")
+                            .body(Full::new(bytes::Bytes::new()))
+                            .unwrap(),
+                    )
+                    .await
+                    .unwrap();
+                assert_eq!(response.status(), http::StatusCode::UNAUTHORIZED);
+                let body = response.into_body().collect().await.unwrap().to_bytes();
+                let body = std::str::from_utf8(&body).unwrap();
+                assert!(body.contains(&format!("<Details>{details}</Details>")));
+                for secret in [&endpoint, "private-deployment", "private-token", "/v3/auth/tokens"] {
+                    assert!(!body.contains(secret), "upstream diagnostics must not appear in the response");
+                }
+                if let Some(server) = server {
+                    server.join().unwrap();
+                }
+            }
+        }
+    }
+
     #[test]
     fn test_client_creation() {
         let client = KeystoneClient::new(

crates/keystone/src/middleware.rs

@@ -173,7 +173,11 @@ where
     <Message>Invalid Keystone token</Message>
     <Details>{}</Details>
 </Error>"#,
-                            xml_escape(&e.to_string())
+                            xml_escape(match e {
+                                crate::KeystoneError::InvalidToken => "Invalid token",
+                                crate::KeystoneError::TokenExpired => "Token expired",
+                                _ => "Keystone token validation failed",
+                            })
                         );
 
                         let body: BoxBody = Full::new(Bytes::from(error_xml))

crates/kms/src/manager.rs

@@ -214,6 +214,9 @@ impl KmsManager {
     }
 
     async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
+        if request.key_name.as_deref() == Some(crate::probe::PROBE_KEY_ID) {
+            return Err(KmsError::invalid_operation("the KMS probe key is reserved for internal health checks"));
+        }
         // A blank name is neither "generate one" (that is `None`) nor a usable
         // id: the Local backend would write a key file with an empty stem and
         // the Vault backends would address their mount root, each failing with
@@ -438,6 +441,9 @@ impl KmsManager {
     /// any backend sees the request. The backends repeat the window bound as a
     /// defensive assertion for callers that hold a backend handle directly.
     async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
+        if request.key_id == crate::probe::PROBE_KEY_ID {
+            return Err(KmsError::invalid_operation("the KMS probe key cannot be deleted through key management"));
+        }
         self.check_deletion_request(&request)?;
         if request.force_immediate.unwrap_or(false) {
             self.refuse_referenced_immediate_deletion(&request.key_id).await?;
@@ -591,7 +597,11 @@ impl KmsManager {
     /// Disable a key on behalf of `context`'s principal
     pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
         let started = Instant::now();
-        let result = self.backend.disable_key(key_id).await;
+        let result = if key_id == crate::probe::PROBE_KEY_ID {
+            Err(KmsError::invalid_operation("the KMS probe key cannot be disabled through key management"))
+        } else {
+            self.backend.disable_key(key_id).await
+        };
         if result.is_ok() {
             self.invalidate_cached_metadata(key_id).await;
         }

crates/kms/src/probe.rs

@@ -700,6 +700,57 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn key_management_cannot_preempt_or_disable_the_readiness_probe() {
+        let temp_dir = tempfile::tempdir().expect("key directory");
+        let backend = local_backend(&temp_dir).await;
+        let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        config.allow_immediate_deletion = true;
+        let manager = crate::manager::KmsManager::new(backend.clone(), config);
+        let request = CreateKeyRequest {
+            key_name: Some(PROBE_KEY_ID.to_owned()),
+            ..Default::default()
+        };
+        assert!(matches!(manager.create_key(request).await, Err(KmsError::InvalidOperation { .. })));
+        let worker = worker(backend.clone());
+        assert_eq!(worker.round().await, ProbeResult::Success, "the probe still provisions its reserved key");
+        assert!(matches!(manager.disable_key(PROBE_KEY_ID).await, Err(KmsError::InvalidOperation { .. })));
+        for immediate in [false, true] {
+            assert!(matches!(
+                manager
+                    .delete_key(DeleteKeyRequest {
+                        key_id: PROBE_KEY_ID.to_owned(),
+                        pending_window_in_days: Some(7),
+                        force_immediate: Some(immediate),
+                        confirm_key_id: Some(PROBE_KEY_ID.to_owned())
+                    })
+                    .await,
+                Err(KmsError::InvalidOperation { .. })
+            ));
+            assert_eq!(worker.round().await, ProbeResult::Success);
+        }
+        let customer = manager
+            .create_key(CreateKeyRequest {
+                key_name: Some("customer-probe-control".to_owned()),
+                ..Default::default()
+            })
+            .await
+            .expect("customer key");
+        manager
+            .disable_key(&customer.key_id)
+            .await
+            .expect("customer key can be disabled");
+        manager
+            .delete_key(DeleteKeyRequest {
+                key_id: customer.key_id,
+                pending_window_in_days: Some(7),
+                ..Default::default()
+            })
+            .await
+            .expect("customer deletion remains available");
+        assert_eq!(worker.round().await, ProbeResult::Success);
+    }
+
     #[test]
     fn read_only_backend_is_reported_unsupported_without_failures() {
         let (snapshot, result) = record_metrics(|| {

crates/lock/src/distributed_lock.rs

@@ -694,6 +694,132 @@ impl DistributedLock {
         self.acquire_guard(&req).await
     }
 
+    /// Acquire exclusive locks for many independent resources in one batched
+    /// round: one [`LockClient::acquire_locks_batch`] RPC per client instead of
+    /// one quorum round-trip per key. Returns one guard or error per resource.
+    ///
+    /// Bulk operations (S3 `DeleteObjects`) are the only multi-key write-lock
+    /// holders, and their keys are independent. Per-key outcomes are
+    /// deadlock-free without any serial fallback: the wire call is a single
+    /// round per client and every lock server orders concurrent batches by the
+    /// same global (shard, key) order, so no acquirer can hold one key while
+    /// waiting for another. A key that fails to reach quorum releases its
+    /// partial grants (awaited) before its error is returned.
+    ///
+    /// Guards keep the same TTL heartbeat and lock-lost semantics as
+    /// [`Self::acquire_guard`]. Contention is waited out server-side up to
+    /// `timeout` (each request carries it), matching the single-key path.
+    pub async fn acquire_guard_batch(
+        &self,
+        resources: &[ObjectKey],
+        owner: &str,
+        timeout: Duration,
+        ttl: Duration,
+    ) -> Vec<Result<DistributedLockGuard>> {
+        if resources.is_empty() {
+            return Vec::new();
+        }
+        if self.clients.is_empty() {
+            return resources
+                .iter()
+                .map(|_| Err(LockError::internal("No lock clients available")))
+                .collect();
+        }
+
+        let required_quorum = self.required_quorum(LockType::Exclusive);
+        let attempt_started = Instant::now();
+        let requests: Vec<LockRequest> = resources
+            .iter()
+            .map(|resource| {
+                LockRequest::new(resource.clone(), LockType::Exclusive, owner)
+                    .with_acquire_timeout(timeout)
+                    .with_ttl(ttl)
+            })
+            .collect();
+
+        // One batched round per client; a client-level failure is a per-key
+        // failure for every key that still needed that client's vote.
+        let rounds = join_all(self.clients.iter().map(|client| {
+            let requests = requests.clone();
+            async move { client.acquire_locks_batch(&requests).await }
+        }))
+        .await;
+
+        let mut results: Vec<Result<DistributedLockGuard>> = Vec::with_capacity(resources.len());
+        let mut orphaned_entries: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
+        for (key_idx, resource) in resources.iter().enumerate() {
+            let mut entries: Vec<HeldLockEntry> = Vec::new();
+            let mut failure: Option<String> = None;
+            for (client_idx, client) in self.clients.iter().enumerate() {
+                match &rounds[client_idx] {
+                    Ok(responses) => match responses.get(key_idx) {
+                        Some(response) if response.success => {
+                            let lock_id = response
+                                .lock_info
+                                .as_ref()
+                                .map(|info| info.id.clone())
+                                .unwrap_or_else(|| LockId::new_unique(resource));
+                            entries.push(HeldLockEntry {
+                                lock_id,
+                                client: client.clone(),
+                                valid_until: attempt_started.checked_add(ttl).unwrap_or(attempt_started),
+                            });
+                        }
+                        Some(response) => {
+                            failure.get_or_insert_with(|| {
+                                response
+                                    .error
+                                    .clone()
+                                    .unwrap_or_else(|| "distributed lock acquisition failed".to_string())
+                            });
+                        }
+                        None => {
+                            failure.get_or_insert_with(|| "client returned incomplete batch lock response".to_string());
+                        }
+                    },
+                    Err(err) => {
+                        failure.get_or_insert_with(|| err.to_string());
+                    }
+                }
+            }
+
+            if entries.len() >= required_quorum {
+                let heartbeat = LockHeartbeatConfig {
+                    ttl,
+                    interval: derive_refresh_interval(entries.len(), ttl, None),
+                    refresh_quorum: required_quorum,
+                    owner: owner.to_string(),
+                    resource: resource.clone(),
+                };
+                results.push(Ok(DistributedLockGuard::new(
+                    LockId::new_unique(resource),
+                    entries,
+                    LockType::Exclusive,
+                    heartbeat,
+                )));
+            } else {
+                let achieved = entries.len();
+                orphaned_entries.extend(entries.iter().map(HeldLockEntry::release_entry));
+                if let Some(error) = failure {
+                    tracing::debug!(resource = %resource, owner, required_quorum, achieved, "batch lock acquire failed: {error}");
+                }
+                results.push(Err(LockError::QuorumNotReached {
+                    required: required_quorum,
+                    achieved,
+                }));
+            }
+        }
+
+        // One bounded synchronous pass so the caller cannot race these
+        // unlocks on an immediate retry; anything a slow peer still holds
+        // keeps retrying in the background instead of parking the request
+        // on the deferred unlock schedule.
+        if !orphaned_entries.is_empty() {
+            Self::release_entries_bounded(orphaned_entries, "distributed_lock_batch_quorum_rollback").await;
+        }
+        results
+    }
+
     fn spawn_lock_requests_until(
         &self,
         pending: &mut JoinSet<LockAcquireTaskResult>,
@@ -755,6 +881,19 @@ impl DistributedLock {
         Self::release_entries_deferred(pending, context).await;
     }
 
+    /// Release entries with one synchronous pass, then continue the retry
+    /// schedule in the background.
+    ///
+    /// The batch-acquire rollback path runs on the request: it must not park
+    /// the caller on the deferred unlock schedule when a peer is unreachable,
+    /// while still confirming the healthy quorum is free before returning.
+    async fn release_entries_bounded(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
+        let pending = Self::release_pending_once(entries, 1, context).await;
+        if !pending.is_empty() {
+            Self::spawn_release_cleanup(pending, context);
+        }
+    }
+
     async fn release_pending_once(
         pending: Vec<(LockId, Arc<dyn LockClient>)>,
         attempt: usize,
@@ -2856,4 +2995,120 @@ mod tests {
 
         drop(guard);
     }
+
+    /// Per-object scripted client: grants only the named objects and records
+    /// every released lock's resource.
+    #[derive(Debug)]
+    struct ScriptedBatchClient {
+        grants: Vec<String>,
+        releases: Arc<Mutex<Vec<String>>>,
+    }
+
+    impl ScriptedBatchClient {
+        fn build(grants: &[&str]) -> (Arc<dyn LockClient>, Arc<Mutex<Vec<String>>>) {
+            let releases = Arc::new(Mutex::new(Vec::new()));
+            let client = Arc::new(Self {
+                grants: grants.iter().map(|object| (*object).to_string()).collect(),
+                releases: Arc::clone(&releases),
+            }) as Arc<dyn LockClient>;
+            (client, releases)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl LockClient for ScriptedBatchClient {
+        async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+            if self.grants.iter().any(|object| object == request.resource.object.as_ref()) {
+                let info = LockInfo {
+                    id: request.lock_id.clone(),
+                    resource: request.resource.clone(),
+                    lock_type: request.lock_type,
+                    status: crate::types::LockStatus::Acquired,
+                    owner: request.owner.clone(),
+                    acquired_at: std::time::SystemTime::now(),
+                    expires_at: std::time::SystemTime::now() + request.ttl,
+                    last_refreshed: std::time::SystemTime::now(),
+                    metadata: request.metadata.clone(),
+                    priority: request.priority,
+                    wait_start_time: None,
+                };
+                Ok(LockResponse::success(info, Duration::ZERO))
+            } else {
+                Ok(LockResponse::failure("denied by test script", Duration::ZERO))
+            }
+        }
+
+        async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+            self.releases
+                .lock()
+                .expect("release log")
+                .push(lock_id.resource.object.to_string());
+            Ok(true)
+        }
+
+        async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
+            Ok(true)
+        }
+
+        async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+            self.release(lock_id).await
+        }
+
+        async fn check_status(&self, _lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+            Ok(None)
+        }
+
+        async fn get_stats(&self) -> crate::Result<LockStats> {
+            Ok(LockStats::default())
+        }
+
+        async fn close(&self) -> crate::Result<()> {
+            Ok(())
+        }
+
+        async fn is_online(&self) -> bool {
+            true
+        }
+
+        async fn is_local(&self) -> bool {
+            false
+        }
+    }
+
+    /// A batch key granted by fewer than quorum clients fails alone, and its
+    /// partial grants are released synchronously before the call returns
+    /// (PR9155 review coverage for the distributed rollback path).
+    #[tokio::test]
+    async fn acquire_guard_batch_releases_partial_grants_below_quorum() {
+        let (c0, c0_releases) = ScriptedBatchClient::build(&["kept", "lost"]);
+        let (c1, _c1_releases) = ScriptedBatchClient::build(&["kept"]);
+        let (c2, _c2_releases) = ScriptedBatchClient::build(&[]);
+        let lock = DistributedLock::new("batch-test".to_string(), vec![c0, c1, c2], 2);
+
+        let resources = vec![ObjectKey::new("bucket", "kept"), ObjectKey::new("bucket", "lost")];
+        let results = lock
+            .acquire_guard_batch(&resources, "owner", Duration::from_millis(200), Duration::from_secs(30))
+            .await;
+
+        assert!(results[0].is_ok(), "kept reached quorum: {:?}", results[0].as_ref().err());
+        let error = results[1].as_ref().expect_err("lost must fail below quorum");
+        assert!(
+            matches!(
+                error,
+                LockError::QuorumNotReached {
+                    required: 2,
+                    achieved: 1
+                }
+            ),
+            "lost must report its partial vote: {error:?}"
+        );
+        // Exactly the orphaned grant is released before the call returns,
+        // while "kept" stays held by its live guard.
+        assert_eq!(
+            c0_releases.lock().expect("release log").clone(),
+            vec!["lost".to_string()],
+            "only the below-quorum key's partial grant is released synchronously"
+        );
+        drop(results);
+    }
 }

crates/lock/src/fast_lock/shard.rs

@@ -153,7 +153,15 @@ impl LockShard {
         {
             return false;
         }
-        self.try_fast_path(request).is_some()
+        // Count the acquisition like `acquire_lock` does: batch callers use
+        // this path for every uncontended key, and the metric promises the
+        // total of successful acquisitions.
+        if self.try_fast_path(request).is_some() {
+            self.metrics.record_fast_path_success();
+            true
+        } else {
+            false
+        }
     }
 
     /// Try fast path lock acquisition (lock-free when possible)

crates/lock/src/local_lock.rs

@@ -15,7 +15,7 @@
 use crate::{
     GlobalLockManager, ObjectKey,
     error::Result,
-    fast_lock::{FastLockGuard, LockManager, LockMode, ObjectLockRequest},
+    fast_lock::{FastLockGuard, LockManager, LockMode, LockResult, ObjectLockRequest},
     types::{LockPriority, LockRequest, LockType},
 };
 use std::sync::Arc;
@@ -112,4 +112,61 @@ impl LocalLock {
             .with_ttl(ttl);
         self.acquire_guard(&req).await
     }
+
+    /// Acquire exclusive locks for many independent resources in one batched
+    /// pass; see [`crate::distributed_lock::DistributedLock::acquire_guard_batch`].
+    /// The manager's batch acquire sorts every request into one global
+    /// (shard, key) order, so concurrent batches cannot deadlock. Returns one
+    /// guard or error per resource, in `resources` order.
+    pub async fn acquire_guard_batch(
+        &self,
+        resources: &[ObjectKey],
+        owner: &str,
+        timeout: Duration,
+        ttl: Duration,
+    ) -> Vec<Result<FastLockGuard>> {
+        if resources.is_empty() {
+            return Vec::new();
+        }
+        let batch = crate::fast_lock::BatchLockRequest {
+            requests: resources
+                .iter()
+                .map(|resource| {
+                    ObjectLockRequest::new_write(resource.clone(), owner)
+                        .with_acquire_timeout(timeout)
+                        .with_lock_timeout(ttl)
+                })
+                .collect(),
+            owner: owner.into(),
+            all_or_nothing: false,
+        };
+        let outcome = self.manager.acquire_locks_batch(batch).await;
+        let mut guards = outcome
+            .successful_locks
+            .into_iter()
+            .zip(outcome.guards)
+            .collect::<std::collections::HashMap<ObjectKey, FastLockGuard>>();
+        let mut failures = outcome
+            .failed_locks
+            .into_iter()
+            .collect::<std::collections::HashMap<ObjectKey, LockResult>>();
+        resources
+            .iter()
+            .map(|resource| {
+                if let Some(guard) = guards.remove(resource) {
+                    return Ok(guard);
+                }
+                match failures.remove(resource) {
+                    Some(LockResult::Timeout) => Err(crate::error::LockError::timeout(resource.to_string(), timeout)),
+                    Some(LockResult::Conflict { current_owner, .. }) => Err(crate::error::LockError::AlreadyLocked {
+                        resource: resource.to_string(),
+                        owner: current_owner.to_string(),
+                    }),
+                    Some(LockResult::Acquired) | None => {
+                        Err(crate::error::LockError::internal("batch lock acquire returned no outcome for key"))
+                    }
+                }
+            })
+            .collect()
+    }
 }

crates/lock/src/namespace/mod.rs

@@ -88,6 +88,17 @@ impl NamespaceLockWrapper {
     pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
         self.lock.get_read_lock(self.resource.clone(), &self.owner, timeout).await
     }
+
+    /// Acquire write locks for `resources` in one batched round using this
+    /// wrapper's owner; see [`NamespaceLock::acquire_write_locks_batch`].
+    /// One guard or error per resource, in `resources` order.
+    pub async fn get_write_locks_batch(
+        &self,
+        resources: &[ObjectKey],
+        timeout: Duration,
+    ) -> Vec<std::result::Result<NamespaceLockGuard, crate::error::LockError>> {
+        self.lock.acquire_write_locks_batch(resources, &self.owner, timeout).await
+    }
 }
 
 impl NamespaceLockGuard {
@@ -349,6 +360,38 @@ impl NamespaceLock {
         }
     }
 
+    /// Acquire exclusive locks for many independent resources in one batched
+    /// round; see [`crate::distributed_lock::DistributedLock::acquire_guard_batch`].
+    /// Returns one guard or error per resource, in `resources` order.
+    ///
+    /// This is the lock primitive bulk mutations (S3 `DeleteObjects`) should
+    /// use: per-key quorum round-trips serialize a 1000-key batch, while the
+    /// batched form is one wire round per lock client and stays deadlock-free
+    /// without any canonical serial fallback (every lock server orders
+    /// concurrent batches by the same global key order).
+    pub async fn acquire_write_locks_batch(
+        &self,
+        resources: &[ObjectKey],
+        owner: &str,
+        timeout: Duration,
+    ) -> Vec<std::result::Result<NamespaceLockGuard, crate::error::LockError>> {
+        let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
+        match self {
+            Self::Distributed(lock) => lock
+                .acquire_guard_batch(resources, owner, timeout, ttl)
+                .await
+                .into_iter()
+                .map(|result| result.map(NamespaceLockGuard::Standard))
+                .collect(),
+            Self::Local(lock) => lock
+                .acquire_guard_batch(resources, owner, timeout, ttl)
+                .await
+                .into_iter()
+                .map(|result| result.map(NamespaceLockGuard::Fast))
+                .collect(),
+        }
+    }
+
     /// Get health information
     pub async fn get_health(&self) -> crate::types::HealthInfo {
         let lock_stats = self.get_stats().await;