fix(replication): replay the MRF journal periodically (#9139)

* fix(replication): replay the MRF journal periodically

The persisted MRF journal was replayed only at startup. Delete-marker
purges whose watch window drained with targets unreachable, and
replications that overflowed the worker queues, reach the journal while
the server runs, so they waited for a restart to converge.

Move the startup pass into replay_mrf_journal and rerun it every five
minutes from a background loop. The existing recovery leader lock keeps
one pass at a time across the cluster, including the startup pass.

* fix(replication): space periodic MRF replay across nodes

Every node runs the MRF replay loop and the recovery leader lock only
serializes passes, so an N-node cluster replayed the same retained
backlog N times per interval while a target stayed down.

Record the start time of a pass that finds entries in a shared replay
stamp and skip a periodic pass when any node started one within the
interval. The startup pass still always runs.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6

.config/nextest.toml

@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -769,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -849,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -497,6 +497,63 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     # Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1397,6 +1454,7 @@ jobs:
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1424,6 +1482,7 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/nightly-gnu.yml

@@ -346,12 +346,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +360,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/rustfs-validation-sweep.yml

@@ -92,6 +92,17 @@ jobs:
           CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
         run: |
           set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
           python3 scripts/resolve_functional_candidate.py
           python3 - <<'PY'
           import json
@@ -99,15 +110,15 @@ jobs:
           from pathlib import Path
           root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
           root.mkdir(parents=True, exist_ok=True)
-          candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
-          manifest = candidate["manifest"]
+          manifest = chain["candidate"]["manifest"]
           (root / "candidate.json").write_text(json.dumps({
               "package_url": manifest["package_url"],
               "package_sha256": manifest["package_sha256"],
               "source_sha": manifest["source_sha"],
           }, sort_keys=True) + "\n")
           with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("testing_sha=" + candidate["testing_sha"] + "\n")
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
               output.write("package_url=" + manifest["package_url"] + "\n")
               output.write("source_sha=" + manifest["source_sha"] + "\n")
           PY
@@ -121,6 +132,26 @@ jobs:
           path: auto-testing
           persist-credentials: false
 
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
       - name: Select sweep cases
         id: select
         env:
@@ -224,7 +255,8 @@ jobs:
             # topology, this purge guarantees the next case starts clean even
             # when the previous one died mid-run.
             for node in ${RUSTFS_NODES}; do
-              ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
                 SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
                 ${SUDO} systemctl stop rustfs 2>/dev/null || true
                 if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
@@ -243,6 +275,7 @@ jobs:
           SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
           PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
           SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
         run: |
           set -euo pipefail
           python3 - <<'PY'
@@ -351,7 +384,10 @@ jobs:
             ${{ runner.temp }}/validation-sweep/sweep-cases.txt
             ${{ runner.temp }}/validation-sweep/candidate.json
             ${{ runner.temp }}/validation-sweep/results/
-          if-no-files-found: error
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
           retention-days: 30
 
       - name: Cleanup environment (after)

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -212,12 +212,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.18.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +235,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +254,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -337,7 +338,7 @@ strum = { version = "0.28.0" }
 sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +357,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -384,9 +385,9 @@ russh-sftp = "3.0.1"
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.2", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +428,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
+                after_lock_contention(&format!("{context} COPY retained"), || {
-                    .copy_object()
+                    copier
-                    .bucket(&bucket)
+                        .copy_object()
-                    .key(&copy_key)
+                        .bucket(&bucket)
-                    .copy_source(format!("{bucket}/{key}"))
+                        .key(&copy_key)
-                    .send()
+                        .copy_source(format!("{bucket}/{key}"))
-                    .await?;
+                        .send()
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                after_lock_contention(&format!("{context} DELETE source"), || {
-                let missing = reader
+                    writer.delete_object().bucket(&bucket).key(&key).send()
-                    .head_object()
+                })
-                    .bucket(&bucket)
+                .await??;
-                    .key(&key)
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
-                    .send()
+                    reader.head_object().bucket(&bucket).key(&key).send()
-                    .await
+                })
-                    .expect_err("deleted source key must not remain visible");
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
-    VersioningConfiguration,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
+    let wait_for_source_writeback = || async {
-        .delete_object()
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
+        .delete(
-        .version_id(&purged_version_id)
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
+            replay_mrf_journal(&storage, None).await;
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -3421,6 +3511,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3580,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3661,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3792,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -4171,6 +4296,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4348,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5528,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6494,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
+            // A batch purge or another worker may have removed this version
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+            // already. Remote failures still control the final outcome below.
-                component = LOG_COMPONENT_ECSTORE,
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            if !source_already_purged {
-                bucket = %bucket,
+                error!(
-                arn = %dobj.target_arn,
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                object = %dobj.delete_object.object_name,
+                    component = LOG_COMPONENT_ECSTORE,
-                operation = "apply_replication_delete_state",
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                error = %e,
+                    bucket = %bucket,
-                "Replication target operation failed"
+                    arn = %dobj.target_arn,
-            );
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1637,6 +1637,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1709,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1780,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2012,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2027,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,6 +445,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -672,6 +676,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -6839,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7234,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7288,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7301,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7314,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -14546,6 +14606,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14556,10 +14633,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -29482,6 +29559,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();

crates/ecstore/src/disk/disk_store.rs

@@ -853,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +882,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1577,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1932,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2638,6 +2673,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -6044,6 +6086,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);

crates/ecstore/src/disk/local.rs

@@ -3012,7 +3012,7 @@ fn should_fail_local_inline_rollback_hardlink(dst_path: &Path) -> bool {
         .remove(dst_path)
 }
 
-#[cfg(test)]
+#[cfg(all(test, target_os = "linux"))]
 pub(super) fn remove_dst_base_before_commit_std(
     dst_path: &str,
     guard: os::RenameCommitGuard,
@@ -3290,7 +3290,7 @@ fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
     false
 }
 
-#[cfg(not(test))]
+#[cfg(all(not(test), target_os = "linux"))]
 pub(super) fn remove_dst_base_before_commit_std(
     _dst_path: &str,
     guard: os::RenameCommitGuard,
@@ -5167,6 +5167,7 @@ impl LocalIoBackend for StdBackend {
                     Arc::new(FdCacheEntry {
                         file: Arc::new(file),
                         len: metadata_len,
+                        invalidated: AtomicBool::new(false),
                     })
                 });
                 #[cfg(not(target_os = "linux"))]
@@ -5607,6 +5608,9 @@ const FD_CACHE_CAPACITY: u64 = 512;
 #[cfg(any(target_os = "linux", all(unix, test)))]
 const FD_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(5);
 
+#[cfg(any(target_os = "linux", all(unix, test)))]
+const FD_CACHE_MAINTENANCE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1);
+
 #[cfg(target_os = "linux")]
 fn is_io_uring_fd_cache_enabled() -> bool {
     rustfs_utils::get_env_bool(ENV_RUSTFS_IO_URING_FD_CACHE, DEFAULT_RUSTFS_IO_URING_FD_CACHE)
@@ -5702,6 +5706,8 @@ struct FdCacheEntry {
     file: Arc<std::fs::File>,
     /// File length captured together with the descriptor.
     len: u64,
+    #[cfg(any(target_os = "linux", all(unix, test)))]
+    invalidated: AtomicBool,
 }
 
 /// Per-disk cache of open descriptors for io_uring reads (backlog#1145).
@@ -5735,6 +5741,13 @@ struct FdCacheEntry {
 #[cfg(any(target_os = "linux", all(unix, test)))]
 struct FdCache {
     cache: moka::future::Cache<FdKey, Arc<FdCacheEntry>>,
+    // Moka's entry count can lag inserts. Only a cache that has never accepted
+    // an insert can safely skip the foreground invalidation scan.
+    has_inserted_entries: AtomicBool,
+    // The only sender belongs to this cache. Dropping it lets the worker finish
+    // maintenance and release its cache clone without retaining the backend.
+    maintenance_tx: tokio::sync::watch::Sender<()>,
+    maintenance_task: std::sync::OnceLock<tokio::task::JoinHandle<()>>,
     /// Bumped by every invalidation. A miss-path open snapshots this before it
     /// opens and refuses to insert if it moved, so an fd opened before a
     /// heal/delete commit can never be resurrected into the cache after the
@@ -5752,20 +5765,78 @@ impl FdCache {
     /// `new` (`FD_CACHE_TTL`); tests inject a short TTL to exercise the backstop
     /// eviction (rustfs/backlog#1180) without a multi-second wait.
     fn with_ttl(ttl: std::time::Duration) -> Self {
+        let (maintenance_tx, _) = tokio::sync::watch::channel(());
         Self {
             cache: moka::future::Cache::builder()
                 .max_capacity(FD_CACHE_CAPACITY)
                 .time_to_live(ttl)
-                // Required for `invalidate_entries_if`; without it that call
-                // fails at runtime instead of dropping stale descriptors.
-                .support_invalidation_closures()
                 .build(),
+            has_inserted_entries: AtomicBool::new(false),
+            maintenance_tx,
+            maintenance_task: std::sync::OnceLock::new(),
             generation: std::sync::atomic::AtomicU64::new(0),
         }
     }
 
+    fn schedule_maintenance(&self) {
+        self.maintenance_task.get_or_init(|| {
+            let cache = self.cache.clone();
+            let mut notifications = self.maintenance_tx.subscribe();
+            // Invalidation runs on the disk API's Tokio runtime. Starting lazily
+            // keeps construction runtime-independent and idle caches task-free.
+            tokio::spawn(async move {
+                let mut interval = tokio::time::interval_at(
+                    tokio::time::Instant::now() + FD_CACHE_MAINTENANCE_INTERVAL,
+                    FD_CACHE_MAINTENANCE_INTERVAL,
+                );
+                interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+                loop {
+                    let stopping = tokio::select! {
+                        changed = notifications.changed() => changed.is_err(),
+                        _ = interval.tick() => false,
+                    };
+                    if stopping {
+                        cache.invalidate_all();
+                    }
+                    cache.run_pending_tasks().await;
+                    if stopping {
+                        break;
+                    }
+                    if cache.entry_count() != 0 {
+                        for (key, entry) in cache.iter() {
+                            if entry.invalidated.load(Ordering::Acquire) {
+                                // Concurrent replacement may evict a fresh fd;
+                                // a later read safely reopens it.
+                                cache.invalidate(key.as_ref()).await;
+                            }
+                        }
+                        cache.run_pending_tasks().await;
+                    }
+                }
+            })
+        });
+        self.maintenance_tx.send_replace(());
+    }
+
+    fn invalidate_matching(&self, matches: impl Fn(&FdKey) -> bool) {
+        if !self.has_inserted_entries.load(Ordering::Acquire) {
+            return;
+        }
+        // Entries already present are visited unless concurrently removed. The
+        // generation fence rejects an older open inserted after this scan.
+        for (key, entry) in self.cache.iter() {
+            if matches(&key) {
+                entry.invalidated.store(true, Ordering::Release);
+            }
+        }
+        self.schedule_maintenance();
+    }
+
     async fn get(&self, key: &FdKey) -> Option<Arc<FdCacheEntry>> {
-        self.cache.get(key).await
+        self.cache
+            .get(key)
+            .await
+            .filter(|entry| !entry.invalidated.load(Ordering::Acquire))
     }
 
     /// Current invalidation generation, snapshotted by the miss path before it
@@ -5783,6 +5854,7 @@ impl FdCache {
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             return;
         }
+        self.has_inserted_entries.store(true, Ordering::Release);
         self.cache.insert(key.clone(), entry).await;
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             self.cache.invalidate(&key).await;
@@ -5809,38 +5881,24 @@ impl FdCache {
     /// Drop every descriptor for `volume` whose path is `prefix` or lies under it.
     /// A component-boundary check keeps `a/b` from invalidating `a/bc`.
     ///
-    /// moka applies the predicate to entries inserted at or before this call and
+    /// Mark matching entries immediately so reads reject them while maintenance
-    /// guarantees a later `get` never returns one of them, so the invalidation is
+    /// removes them. The scan borrows its paths and retains no invalidation
-    /// effective immediately even though the removal itself is deferred. Reserved
+    /// closures when the cache is empty or idle.
-    /// for paths whose exact keys are unknown (`delete`); every predicate is
-    /// re-evaluated by subsequent `get`s, so registering one per write would tax
-    /// the read path.
     fn invalidate_under(&self, volume: &str, prefix: &str) {
-        // Bump before registering the predicate so a concurrent miss-path insert
+        // Bump before scanning the cache so a concurrent miss-path insert
         // that snapshotted the old generation is refused (rustfs/backlog#1176).
         self.generation.fetch_add(1, Ordering::AcqRel);
-        let volume = volume.to_owned();
+        let prefix = prefix.trim_end_matches('/');
-        let prefix = prefix.trim_end_matches('/').to_owned();
+        self.invalidate_matching(|k| {
-        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| {
+            k.volume == volume && (k.path == prefix || k.path.strip_prefix(prefix).is_some_and(|r| r.starts_with('/')))
-            k.volume == volume && (k.path == prefix || k.path.strip_prefix(&prefix).is_some_and(|r| r.starts_with('/')))
+        });
-        };
-        if self.cache.invalidate_entries_if(matches).is_err() {
-            // Closure support is enabled at construction, so this is unreachable.
-            // If it ever changes, over-invalidate rather than serve a stale
-            // descriptor: correctness first, the cache refills on the next read.
-            self.cache.invalidate_all();
-        }
     }
 
     /// Drop every descriptor for `volume` — the whole bucket is gone
     /// (rustfs/backlog#1177).
     fn invalidate_volume(&self, volume: &str) {
         self.generation.fetch_add(1, Ordering::AcqRel);
-        let volume = volume.to_owned();
+        self.invalidate_matching(|k| k.volume == volume);
-        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| k.volume == volume;
-        if self.cache.invalidate_entries_if(matches).is_err() {
-            self.cache.invalidate_all();
-        }
     }
 
     /// Drop ALL descriptors — this disk instance is being retired
@@ -5848,6 +5906,7 @@ impl FdCache {
     fn clear(&self) {
         self.generation.fetch_add(1, Ordering::AcqRel);
         self.cache.invalidate_all();
+        self.schedule_maintenance();
     }
 
     /// Unconditional insert. Production reads go through `insert_if_fresh` (the
@@ -5856,7 +5915,17 @@ impl FdCache {
     #[cfg(test)]
     async fn insert(&self, key: FdKey, file: Arc<std::fs::File>) {
         let len = file.metadata().map(|metadata| metadata.len()).unwrap_or_default();
-        self.cache.insert(key, Arc::new(FdCacheEntry { file, len })).await;
+        self.has_inserted_entries.store(true, Ordering::Release);
+        self.cache
+            .insert(
+                key,
+                Arc::new(FdCacheEntry {
+                    file,
+                    len,
+                    invalidated: AtomicBool::new(false),
+                }),
+            )
+            .await;
     }
 
     #[cfg(test)]
@@ -6455,6 +6524,7 @@ impl UringBackend {
                 let file = Arc::new(FdCacheEntry {
                     file: Arc::new(file),
                     len,
+                    invalidated: AtomicBool::new(false),
                 });
                 if let (Some((cache, key)), Some(gen_at_open)) = (cache_entry, gen_at_open) {
                     cache.insert_if_fresh(key, Arc::clone(&file), gen_at_open).await;
@@ -6627,6 +6697,7 @@ impl UringBackend {
                 let entry = Arc::new(FdCacheEntry {
                     file: Arc::new(file),
                     len,
+                    invalidated: AtomicBool::new(false),
                 });
                 if let (Some(cache), Some(gen_at_open)) = (cache, gen_at_open) {
                     cache.insert_if_fresh(key.clone(), Arc::clone(&entry), gen_at_open).await;
@@ -8337,7 +8408,7 @@ impl LocalDisk {
                 },
             )
         {
-            multipart_source_state::put_active_multipart_source_head(admission.source().to_path_buf(), fm_bytes, cached_fi);
+            multipart_source_state::put_active_multipart_source_head(admission.source(), fm_bytes, cached_fi);
         }
         drop(namespace_owner);
 
@@ -34128,11 +34199,217 @@ mod test {
         });
     }
 
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_invalidation_releases_descriptors_without_further_operations() {
+        let cache = FdCache::new();
+        for round in 0..4 {
+            let file = Arc::new(tempfile::tempfile().expect("create a descriptor to invalidate"));
+            let released = Arc::downgrade(&file);
+            let parts = if round == 1 || round == 2 { FD_CACHE_CAPACITY } else { 1 };
+            for part in 0..parts {
+                cache
+                    .insert(
+                        FdKey {
+                            volume: "bucket".into(),
+                            path: format!("object-{round}/part.{part}"),
+                            direct: false,
+                        },
+                        file.clone(),
+                    )
+                    .await;
+            }
+            drop(file);
+            if round % 2 == 0 {
+                cache.invalidate_under("bucket", &format!("object-{round}"));
+            } else {
+                cache.invalidate_volume("bucket");
+            }
+            tokio::time::timeout(std::time::Duration::from_secs(10), async {
+                while released.upgrade().is_some() {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("invalidation must release descriptors without another cache operation");
+        }
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_invalidation_rejects_entries_before_maintenance() {
+        let cache = FdCache::new();
+        let key = FdKey {
+            volume: "bucket".into(),
+            path: "object/part.1".into(),
+            direct: false,
+        };
+        cache
+            .insert(key.clone(), Arc::new(tempfile::tempfile().expect("create a cached descriptor")))
+            .await;
+        assert!(cache.get(&key).await.is_some());
+        cache.invalidate_under("bucket", "object");
+
+        // No yield has allowed the maintenance task to remove the entry yet.
+        assert!(cache.cache.contains_key(&key));
+        assert!(cache.get(&key).await.is_none(), "a marked descriptor must never be served");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_idle_maintenance_releases_expired_descriptors() {
+        let cache = FdCache::with_ttl(std::time::Duration::from_millis(50));
+        let file = Arc::new(tempfile::tempfile().expect("create a descriptor with a short TTL"));
+        let released = Arc::downgrade(&file);
+        let key = FdKey {
+            volume: "retained".into(),
+            path: "part.1".into(),
+            direct: false,
+        };
+        cache.insert(key.clone(), file).await;
+        cache.invalidate_volume("other");
+        assert!(cache.get(&key).await.is_some(), "invalidation must preserve other volumes");
+
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while released.upgrade().is_some() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("periodic maintenance must release expired descriptors while idle");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_maintenance_exits_when_cache_is_dropped() {
+        let mut cache = FdCache::new();
+        let file = Arc::new(tempfile::tempfile().expect("create a descriptor to retire"));
+        let released = Arc::downgrade(&file);
+        cache
+            .insert(
+                FdKey {
+                    volume: "bucket".into(),
+                    path: "part.1".into(),
+                    direct: false,
+                },
+                file,
+            )
+            .await;
+        cache.invalidate_volume("bucket");
+        let maintenance = cache.maintenance_task.take().expect("invalidation starts maintenance");
+        drop(cache);
+        tokio::time::timeout(std::time::Duration::from_secs(10), maintenance)
+            .await
+            .expect("dropping the cache must stop its maintenance task")
+            .expect("maintenance must complete without a panic");
+        assert!(released.upgrade().is_none(), "shutdown must release cached descriptors");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_maintenance_survives_request_cancellation() {
+        let cache = FdCache::new();
+        let file = Arc::new(tempfile::tempfile().expect("create a descriptor to invalidate"));
+        let released = Arc::downgrade(&file);
+        cache
+            .insert(
+                FdKey {
+                    volume: "bucket".into(),
+                    path: "part.1".into(),
+                    direct: false,
+                },
+                file,
+            )
+            .await;
+        let mut request = Box::pin(async {
+            cache.invalidate_volume("bucket");
+            std::future::pending::<()>().await;
+        });
+        let mut context = std::task::Context::from_waker(std::task::Waker::noop());
+        assert!(std::future::Future::poll(request.as_mut(), &mut context).is_pending());
+        drop(request);
+
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while released.upgrade().is_some() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("maintenance must continue after the invalidating request is cancelled");
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn fd_cache_construction_does_not_require_a_runtime() {
+        let cache = FdCache::new();
+        assert!(cache.maintenance_task.get().is_none());
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_invalidation_fences_an_open_before_the_first_insert() {
+        for mode in 0..2 {
+            let cache = FdCache::new();
+            let key = FdKey {
+                volume: "bucket".into(),
+                path: "object/part.1".into(),
+                direct: false,
+            };
+            let opened = Arc::new(FdCacheEntry {
+                file: Arc::new(tempfile::tempfile().expect("open the descriptor before invalidation")),
+                len: 0,
+                invalidated: AtomicBool::new(false),
+            });
+            let generation = cache.generation();
+            match mode {
+                0 => cache.invalidate_under("bucket", "object"),
+                _ => cache.invalidate_volume("bucket"),
+            }
+            cache.insert_if_fresh(key.clone(), opened, generation).await;
+            assert!(cache.get(&key).await.is_none(), "mode {mode} must reject a pre-invalidation open");
+
+            let fresh = Arc::new(FdCacheEntry {
+                file: Arc::new(tempfile::tempfile().expect("open a fresh descriptor after invalidation")),
+                len: 0,
+                invalidated: AtomicBool::new(false),
+            });
+            cache.insert_if_fresh(key.clone(), fresh.clone(), cache.generation()).await;
+            let cached = cache.get(&key).await.expect("a post-invalidation open must remain cacheable");
+            assert!(Arc::ptr_eq(&cached, &fresh));
+        }
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn fd_cache_clear_releases_descriptors_without_further_reads() {
+        let cache = FdCache::new();
+        let file = Arc::new(tempfile::tempfile().expect("create a descriptor to retire"));
+        let released = Arc::downgrade(&file);
+        cache
+            .insert(
+                FdKey {
+                    volume: "bucket".into(),
+                    path: "part.1".into(),
+                    direct: false,
+                },
+                file,
+            )
+            .await;
+        cache.clear();
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while released.upgrade().is_some() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("clear must release retired descriptors without another cache operation");
+    }
+
     /// Prefix invalidation must respect component boundaries: dropping the
     /// descriptors for object `a/b` must not also drop those for `a/bc`.
-    /// moka applies the predicate lazily, so this also pins the guarantee we rely
+    /// A `get` after `invalidate_under` must reject matched entries while
-    /// on — a `get` after `invalidate_under` never returns a matched entry.
+    /// background maintenance removes them.
-    #[cfg(target_os = "linux")]
+    #[cfg(unix)]
     #[tokio::test(flavor = "multi_thread")]
     async fn fd_cache_prefix_invalidation_respects_component_boundary() {
         use std::fs::File;
@@ -34368,8 +34645,32 @@ mod test {
             }
         }
 
-        // Invalidating by the object prefix drops the cached descriptor.
+        let key = FdKey {
+            volume: volume.into(),
+            path: object.into(),
+            direct: false,
+        };
+        let released = {
+            let entry = cache
+                .get(&key)
+                .await
+                .expect("repeated reads must retain the cached descriptor");
+            Arc::downgrade(&entry.file)
+        };
+
+        // Reads reject invalidated entries immediately; maintenance releases the descriptor.
         backend.invalidate_cached_fds_under(volume, "obj/abc");
+        assert!(
+            cache.get(&key).await.is_none(),
+            "prefix invalidation must immediately reject the cached descriptor"
+        );
+        tokio::time::timeout(std::time::Duration::from_secs(10), async {
+            while released.upgrade().is_some() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("prefix invalidation must release the descriptor without further cache operations");
         assert_eq!(cache.entry_count().await, 0, "prefix invalidation must drop the cached descriptor");
     }
 

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -1002,21 +1002,17 @@ pub(super) fn inspection_with_markers(
 
 type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
 
-static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
+const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
-    std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
 
-fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
+static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
-    let mut map = PREPARED_DELETE_RECEIPTS.lock();
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
-    if map.len() >= 4096
+
-        && let Some(evict_key) = map.keys().next().cloned()
+fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
-    {
+    PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
-        map.remove(&evict_key);
-    }
-    map.insert(rollback, entry);
 }
 
 fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
-    PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
+    PREPARED_DELETE_RECEIPTS.remove(rollback)
 }
 
 impl LocalDisk {

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -270,21 +270,17 @@ struct ActiveMultipartSourceEntry {
     parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
 }
 
-static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
-    parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
-> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
 
-pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
     let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
         return;
     };
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
-    if map.len() >= 4096
-        && let Some(evict_key) = map.keys().next().cloned()
-    {
-        map.remove(&evict_key);
-    }
-    map.insert(
         source,
         ActiveMultipartSourceEntry {
             raw,
@@ -304,56 +300,50 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
 }
 
 pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
-    let (raw, fi, head_stamp) = {
+    let (raw, fi, head_stamp) =
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
-        let entry = map.get(source)?;
-        (entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
-    };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     Some((raw, fi))
 }
 
 pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
-    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     true
 }
 
 pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
-    let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
+    let Some((head_stamp, clean)) =
-        .lock()
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
-        .get(source)
-        .map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
     else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     clean
 }
 
 pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.unsettled_parts.insert(part_number);
         entry.parts.remove(&part_number);
-    }
+    });
 }
 
 pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
     let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .lock()
+        .with_get(source, |entry| entry.fi.data_dir)
-        .get(source)
+        .flatten()
-        .and_then(|entry| entry.fi.data_dir)
         .filter(|dir| !dir.is_nil())
     else {
         return;
@@ -362,9 +352,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
     let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
         return;
     };
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.parts.insert(part.number, (part_stamp, part));
-    }
+    });
 }
 
 pub(super) fn finish_active_multipart_part_transaction(
@@ -373,34 +363,29 @@ pub(super) fn finish_active_multipart_part_transaction(
     committed: bool,
     retired_existing_txn: bool,
 ) {
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
     let Some(part_number) = part_number.filter(|_| committed) else {
-        map.remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return;
     };
     if !retired_existing_txn {
         return;
     }
-    if let Some(entry) = map.get_mut(source)
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
-        && !entry.unsettled_parts.remove(&part_number)
-    {
-        map.remove(source);
-    }
 }
 
 pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
-    let (head_stamp, data_dir, expected_stamp, part) = {
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
+        .with_get(source, |entry| {
-        let entry = map.get(source)?;
+            if !entry.unsettled_parts.is_empty() {
-        if !entry.unsettled_parts.is_empty() {
+                return None;
-            return None;
+            }
-        }
+            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
-        let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
+            let (expected_stamp, part) = entry.parts.get(&part_number)?;
-        let (expected_stamp, part) = entry.parts.get(&part_number)?;
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
-        (entry.head_stamp, data_dir, *expected_stamp, part.clone())
+        })
-    };
+        .flatten()?;
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -411,11 +396,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
 }
 
 pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
 }
 
 pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
 }
 
 /// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -1936,6 +1921,7 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
     pub(super) fn prepare_and_install_head_on_thread(
         &self,
         new_metadata: &[u8],

crates/ecstore/src/disk/os.rs

@@ -1677,8 +1677,396 @@ const MAX_GLOBAL_FILE_SYNCS: usize = 512;
 #[cfg(test)]
 const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
+const PATH_LOCK_SHARDS: usize = 64;
+const PATH_LOCK_INITIAL_GC_THRESHOLD: usize = 64;
+
+#[cfg(unix)]
+#[derive(Clone, Copy, Default)]
+struct Xxh3SliceHasher(u64);
+
+#[cfg(unix)]
+impl std::hash::Hasher for Xxh3SliceHasher {
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) {
+        self.0 = xxhash_rust::xxh3::xxh3_64(bytes);
+    }
+
+    #[inline]
+    fn write_usize(&mut self, _i: usize) {}
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0
+    }
+}
+
+#[cfg(unix)]
+type Xxh3SliceBuildHasher = std::hash::BuildHasherDefault<Xxh3SliceHasher>;
+
+#[cfg(unix)]
+#[inline]
+fn is_canonical_unix_key_bytes(bytes: &[u8]) -> bool {
+    if bytes.starts_with(b"./") {
+        return !bytes[1..].windows(2).any(|w| w == b"//") && !bytes[1..].windows(3).any(|w| w == b"/./");
+    }
+    const SLASH_MASK: u64 = 0x2f2f_2f2f_2f2f_2f2f;
+    const LOW_BITS: u64 = 0x0101_0101_0101_0101;
+    const HIGH_BITS: u64 = 0x8080_8080_8080_8080;
+
+    let mut i = 0usize;
+    let len = bytes.len();
+    let mut prev_s = 0u64;
+    while i + 8 <= len {
+        let mut buf = [0u8; 8];
+        buf.copy_from_slice(&bytes[i..i + 8]);
+        let w = u64::from_le_bytes(buf);
+        let x = w ^ SLASH_MASK;
+        let s = (x.wrapping_sub(LOW_BITS)) & !x & HIGH_BITS;
+        let s_prev1 = (s << 8) | (prev_s >> 56);
+        let s_prev2 = (s << 16) | (prev_s >> 48);
+        if (s & (s_prev1 | s_prev2)) != 0 {
+            let start = i.saturating_sub(2);
+            let window = &bytes[start..i + 8];
+            if window.windows(2).any(|pair| pair == b"//") || window.windows(3).any(|triple| triple == b"/./") {
+                return false;
+            }
+        }
+        prev_s = s;
+        i += 8;
+    }
+    let tail_start = i.saturating_sub(2);
+    let tail = &bytes[tail_start..];
+    !tail.windows(2).any(|pair| pair == b"//") && !tail.windows(3).any(|triple| triple == b"/./")
+}
+
+#[cfg(unix)]
+#[cold]
+fn canonical_unix_path_key_slow(path: &Path) -> std::borrow::Cow<'static, [u8]> {
+    use std::os::unix::ffi::OsStrExt;
+    let mut out = Vec::with_capacity(path.as_os_str().len());
+    for comp in path.components() {
+        match comp {
+            Component::RootDir => {
+                out.clear();
+                out.push(b'/');
+            }
+            Component::CurDir => {
+                if out.is_empty() {
+                    out.push(b'.');
+                }
+            }
+            Component::ParentDir => {
+                if out.len() > 1 || (out.len() == 1 && out[0] != b'/') {
+                    out.push(b'/');
+                }
+                out.extend_from_slice(b"..");
+            }
+            Component::Normal(part) => {
+                if out.len() > 1 || (out.len() == 1 && out[0] != b'/') {
+                    out.push(b'/');
+                }
+                out.extend_from_slice(part.as_bytes());
+            }
+            Component::Prefix(_) => {}
+        }
+    }
+    std::borrow::Cow::Owned(out)
+}
+
+#[cfg(unix)]
+#[inline]
+fn canonical_unix_path_key(path: &Path) -> std::borrow::Cow<'_, [u8]> {
+    use std::os::unix::ffi::OsStrExt;
+    let mut bytes = path.as_os_str().as_bytes();
+    while bytes.len() > 1 {
+        if bytes[bytes.len() - 1] == b'/' {
+            bytes = &bytes[..bytes.len() - 1];
+        } else if bytes.len() >= 2 && bytes[bytes.len() - 2] == b'/' && bytes[bytes.len() - 1] == b'.' {
+            bytes = if bytes.len() == 2 { b"/" } else { &bytes[..bytes.len() - 2] };
+        } else {
+            break;
+        }
+    }
+    if is_canonical_unix_key_bytes(bytes) {
+        return std::borrow::Cow::Borrowed(bytes);
+    }
+    canonical_unix_path_key_slow(path)
+}
+
+#[cfg(unix)]
+#[inline]
+fn unix_canonical_key_starts_with(path_key: &[u8], root_key: &[u8]) -> bool {
+    if root_key.is_empty() {
+        return true;
+    }
+    if root_key == b"/" {
+        return path_key.starts_with(b"/");
+    }
+    path_key.starts_with(root_key) && (path_key.len() == root_key.len() || path_key.get(root_key.len()) == Some(&b'/'))
+}
+
+struct PathLockShard<L> {
+    #[cfg(unix)]
+    entries: HashMap<Box<[u8]>, Weak<L>, Xxh3SliceBuildHasher>,
+    #[cfg(not(unix))]
+    entries: HashMap<PathBuf, Weak<L>>,
+    next_gc_threshold: usize,
+}
+
+impl<L> Default for PathLockShard<L> {
+    fn default() -> Self {
+        Self {
+            entries: HashMap::default(),
+            next_gc_threshold: PATH_LOCK_INITIAL_GC_THRESHOLD,
+        }
+    }
+}
+
+struct ShardedPathLockRegistry<L> {
+    shards: [Mutex<PathLockShard<L>>; PATH_LOCK_SHARDS],
+}
+
+impl<L> ShardedPathLockRegistry<L> {
+    fn new() -> Self {
+        Self {
+            shards: std::array::from_fn(|_| Mutex::new(PathLockShard::default())),
+        }
+    }
+
+    #[cfg(unix)]
+    #[inline]
+    fn get_or_create_by_key(&self, key: &[u8], init: impl FnOnce() -> L) -> Arc<L> {
+        let hash = xxhash_rust::xxh3::xxh3_64(key);
+        let shard_idx = ((hash >> 32) as usize) & (PATH_LOCK_SHARDS - 1);
+        let mut shard = self.shards[shard_idx].lock();
+        if let Some(existing) = shard.entries.get(key).and_then(Weak::upgrade) {
+            return existing;
+        }
+        let created = Arc::new(init());
+        let weak = Arc::downgrade(&created);
+        if let Some(slot) = shard.entries.get_mut(key) {
+            *slot = weak;
+            return created;
+        }
+        if shard.entries.len() >= shard.next_gc_threshold {
+            shard.entries.retain(|_, entry| entry.strong_count() > 0);
+            shard.next_gc_threshold = shard
+                .entries
+                .len()
+                .saturating_add(1)
+                .saturating_mul(2)
+                .max(PATH_LOCK_INITIAL_GC_THRESHOLD);
+        }
+        shard.entries.insert(key.into(), weak);
+        created
+    }
+
+    #[inline]
+    fn get_or_create(&self, path: &Path, init: impl FnOnce() -> L) -> Arc<L> {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            self.get_or_create_by_key(key.as_ref(), init)
+        }
+        #[cfg(not(unix))]
+        {
+            use std::hash::{Hash, Hasher};
+            let mut hasher = std::collections::hash_map::DefaultHasher::new();
+            path.hash(&mut hasher);
+            let shard_idx = ((hasher.finish() >> 32) as usize) & (PATH_LOCK_SHARDS - 1);
+            let mut shard = self.shards[shard_idx].lock();
+            if let Some(existing) = shard.entries.get(path).and_then(Weak::upgrade) {
+                return existing;
+            }
+            let created = Arc::new(init());
+            let weak = Arc::downgrade(&created);
+            if let Some(slot) = shard.entries.get_mut(path) {
+                *slot = weak;
+                return created;
+            }
+            if shard.entries.len() >= shard.next_gc_threshold {
+                shard.entries.retain(|_, entry| entry.strong_count() > 0);
+                shard.next_gc_threshold = shard
+                    .entries
+                    .len()
+                    .saturating_add(1)
+                    .saturating_mul(2)
+                    .max(PATH_LOCK_INITIAL_GC_THRESHOLD);
+            }
+            shard.entries.insert(path.to_path_buf(), weak);
+            created
+        }
+    }
+}
+
+#[cfg(unix)]
+type PathCacheShardMap<V> = HashMap<Box<[u8]>, V, Xxh3SliceBuildHasher>;
+#[cfg(not(unix))]
+type PathCacheShardMap<V> = HashMap<PathBuf, V>;
+
+pub(crate) struct ShardedPathCache<V> {
+    shards: [Mutex<PathCacheShardMap<V>>; PATH_LOCK_SHARDS],
+    max_entries_per_shard: usize,
+}
+
+impl<V> ShardedPathCache<V> {
+    pub(crate) fn new(max_entries_per_shard: usize) -> Self {
+        Self {
+            shards: std::array::from_fn(|_| Mutex::new(HashMap::default())),
+            max_entries_per_shard: max_entries_per_shard.max(1),
+        }
+    }
+
+    #[cfg(unix)]
+    #[inline]
+    fn shard_index(key: &[u8]) -> usize {
+        let hash = xxhash_rust::xxh3::xxh3_64(key);
+        ((hash >> 32) as usize) & (PATH_LOCK_SHARDS - 1)
+    }
+
+    #[cfg(not(unix))]
+    #[inline]
+    fn shard_index(path: &Path) -> usize {
+        use std::hash::{Hash, Hasher};
+        let mut hasher = std::collections::hash_map::DefaultHasher::new();
+        path.hash(&mut hasher);
+        ((hasher.finish() >> 32) as usize) & (PATH_LOCK_SHARDS - 1)
+    }
+
+    #[inline]
+    pub(crate) fn insert(&self, path: &Path, value: V) {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            let shard_idx = Self::shard_index(key.as_ref());
+            let mut shard = self.shards[shard_idx].lock();
+            if let Some(slot) = shard.get_mut(key.as_ref()) {
+                *slot = value;
+                return;
+            }
+            if shard.len() >= self.max_entries_per_shard
+                && let Some(evict_key) = shard.keys().next().cloned()
+            {
+                shard.remove(&evict_key);
+            }
+            shard.insert(key.into_owned().into_boxed_slice(), value);
+        }
+        #[cfg(not(unix))]
+        {
+            let shard_idx = Self::shard_index(path);
+            let mut shard = self.shards[shard_idx].lock();
+            if let Some(slot) = shard.get_mut(path) {
+                *slot = value;
+                return;
+            }
+            if shard.len() >= self.max_entries_per_shard
+                && let Some(evict_key) = shard.keys().next().cloned()
+            {
+                shard.remove(&evict_key);
+            }
+            shard.insert(path.to_path_buf(), value);
+        }
+    }
+
+    #[inline]
+    pub(crate) fn remove(&self, path: &Path) -> Option<V> {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            let shard_idx = Self::shard_index(key.as_ref());
+            self.shards[shard_idx].lock().remove(key.as_ref())
+        }
+        #[cfg(not(unix))]
+        {
+            let shard_idx = Self::shard_index(path);
+            self.shards[shard_idx].lock().remove(path)
+        }
+    }
+
+    /// Runs `f` while holding the target shard mutex. `f` must not block, perform I/O, or re-enter `self`.
+    #[inline]
+    pub(crate) fn with_get<R>(&self, path: &Path, f: impl FnOnce(&V) -> R) -> Option<R> {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            let shard_idx = Self::shard_index(key.as_ref());
+            let shard = self.shards[shard_idx].lock();
+            shard.get(key.as_ref()).map(f)
+        }
+        #[cfg(not(unix))]
+        {
+            let shard_idx = Self::shard_index(path);
+            let shard = self.shards[shard_idx].lock();
+            shard.get(path).map(f)
+        }
+    }
+
+    /// Runs `f` while holding the target shard mutex. `f` must not block, perform I/O, or re-enter `self`.
+    #[inline]
+    pub(crate) fn with_get_mut<R>(&self, path: &Path, f: impl FnOnce(&mut V) -> R) -> Option<R> {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            let shard_idx = Self::shard_index(key.as_ref());
+            let mut shard = self.shards[shard_idx].lock();
+            shard.get_mut(key.as_ref()).map(f)
+        }
+        #[cfg(not(unix))]
+        {
+            let shard_idx = Self::shard_index(path);
+            let mut shard = self.shards[shard_idx].lock();
+            shard.get_mut(path).map(f)
+        }
+    }
+
+    /// Runs `should_remove` while holding the target shard mutex. `should_remove` must not block, perform I/O, or re-enter `self`.
+    #[inline]
+    pub(crate) fn remove_if_mut(&self, path: &Path, should_remove: impl FnOnce(&mut V) -> bool) {
+        #[cfg(unix)]
+        {
+            let key = canonical_unix_path_key(path);
+            let shard_idx = Self::shard_index(key.as_ref());
+            let mut shard = self.shards[shard_idx].lock();
+            if let Some(entry) = shard.get_mut(key.as_ref())
+                && should_remove(entry)
+            {
+                shard.remove(key.as_ref());
+            }
+        }
+        #[cfg(not(unix))]
+        {
+            let shard_idx = Self::shard_index(path);
+            let mut shard = self.shards[shard_idx].lock();
+            if let Some(entry) = shard.get_mut(path)
+                && should_remove(entry)
+            {
+                shard.remove(path);
+            }
+        }
+    }
+
+    pub(crate) fn retain_outside_prefix(&self, prefix: &Path) {
+        #[cfg(unix)]
+        {
+            let prefix_key = canonical_unix_path_key(prefix);
+            let prefix_bytes = prefix_key.as_ref();
+            for shard in &self.shards {
+                shard
+                    .lock()
+                    .retain(|key, _| !unix_canonical_key_starts_with(key, prefix_bytes));
+            }
+        }
+        #[cfg(not(unix))]
+        {
+            for shard in &self.shards {
+                shard.lock().retain(|key, _| !key.starts_with(prefix));
+            }
+        }
+    }
+}
+
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
-static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+static DISK_FILE_SYNC_LIMITERS: LazyLock<ShardedPathLockRegistry<Semaphore>> = LazyLock::new(ShardedPathLockRegistry::new);
 
 /// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
 /// configured with >1 threads, isolates device-bound fsync from the main
@@ -1742,12 +2130,10 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
         f()
     })
 }
-static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
+static DISK_VOLUME_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<RwLock<()>>> = LazyLock::new(ShardedPathLockRegistry::new);
-    LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
-type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
+static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<NamespaceMutationLock>> =
-static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
+    LazyLock::new(ShardedPathLockRegistry::new);
-    LazyLock::new(|| Mutex::new(HashMap::new()));
 
 fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
     let cpu_scaled = cpu_count
@@ -1771,15 +2157,7 @@ fn global_file_sync_limit() -> usize {
 
 /// Reuse a disk's limiter across reconnects while detached sync calls still hold it.
 pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
-    let mut limiters = DISK_FILE_SYNC_LIMITERS.lock();
+    DISK_FILE_SYNC_LIMITERS.get_or_create(root, || Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
-    limiters.retain(|_, limiter| limiter.strong_count() > 0);
-    if let Some(limiter) = limiters.get(root).and_then(Weak::upgrade) {
-        return limiter;
-    }
-
-    let limiter = Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS));
-    limiters.insert(root.to_path_buf(), Arc::downgrade(&limiter));
-    limiter
 }
 
 // Keep warm draining workers per disk/lane and spin briefly in user space
@@ -2147,45 +2525,131 @@ pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
     BlockingTestHookGuard { queue }
 }
 
-static DISK_BLOCKING_QUEUES: LazyLock<Mutex<HashMap<PathBuf, Weak<DiskBlockingQueue>>>> =
+struct RegisteredDiskBlockingQueue {
-    LazyLock::new(|| Mutex::new(HashMap::new()));
+    #[cfg(not(unix))]
+    root: PathBuf,
+    #[cfg(unix)]
+    canonical_root: Box<[u8]>,
+    queue: Weak<DiskBlockingQueue>,
+}
+
+static DISK_BLOCKING_QUEUES: LazyLock<parking_lot::RwLock<Vec<RegisteredDiskBlockingQueue>>> =
+    LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
 static FALLBACK_BLOCKING_QUEUES: LazyLock<[Arc<DiskBlockingQueue>; FALLBACK_NAMESPACE_QUEUE_SHARDS]> =
     LazyLock::new(|| std::array::from_fn(|_| DiskBlockingQueue::new()));
 
 pub(crate) fn register_disk_blocking_queue(root: &Path, io_root: &Path) -> Arc<DiskBlockingQueue> {
-    let mut queues = DISK_BLOCKING_QUEUES.lock();
+    #[cfg(unix)]
-    queues.retain(|_, q| q.strong_count() > 0);
+    let root_key = canonical_unix_path_key(root);
-    let queue = queues.get(root).and_then(Weak::upgrade).unwrap_or_else(|| {
+    #[cfg(unix)]
-        let queue = DiskBlockingQueue::new();
+    let io_root_key = canonical_unix_path_key(io_root);
-        queues.insert(root.to_path_buf(), Arc::downgrade(&queue));
+
-        queue
+    let mut queues = DISK_BLOCKING_QUEUES.write();
-    });
+    queues.retain(|entry| entry.queue.strong_count() > 0);
-    if io_root != root {
+    let queue = queues
-        queues.insert(io_root.to_path_buf(), Arc::downgrade(&queue));
+        .iter()
+        .find_map(|entry| {
+            #[cfg(unix)]
+            let matches = entry.canonical_root.as_ref() == root_key.as_ref();
+            #[cfg(not(unix))]
+            let matches = entry.root == root;
+            matches.then(|| entry.queue.upgrade()).flatten()
+        })
+        .unwrap_or_else(|| {
+            let queue = DiskBlockingQueue::new();
+            queues.push(RegisteredDiskBlockingQueue {
+                #[cfg(not(unix))]
+                root: root.to_path_buf(),
+                #[cfg(unix)]
+                canonical_root: root_key.as_ref().into(),
+                queue: Arc::downgrade(&queue),
+            });
+            queue
+        });
+    #[cfg(unix)]
+    let io_matches_root = io_root_key.as_ref() == root_key.as_ref();
+    #[cfg(not(unix))]
+    let io_matches_root = io_root == root;
+    if !io_matches_root {
+        let weak = Arc::downgrade(&queue);
+        if let Some(existing) = queues.iter_mut().find(|entry| {
+            #[cfg(unix)]
+            {
+                entry.canonical_root.as_ref() == io_root_key.as_ref()
+            }
+            #[cfg(not(unix))]
+            {
+                entry.root == io_root
+            }
+        }) {
+            existing.queue = weak;
+        } else {
+            queues.push(RegisteredDiskBlockingQueue {
+                #[cfg(not(unix))]
+                root: io_root.to_path_buf(),
+                #[cfg(unix)]
+                canonical_root: io_root_key.as_ref().into(),
+                queue: weak,
+            });
+        }
     }
     queue
 }
 
 fn disk_blocking_queue_for_root(root: &Path) -> Arc<DiskBlockingQueue> {
-    register_disk_blocking_queue(root, root)
+    #[cfg(unix)]
-}
+    let root_key = canonical_unix_path_key(root);
-
-fn disk_blocking_queue_for_path(path: &Path) -> Arc<DiskBlockingQueue> {
     {
-        let queues = DISK_BLOCKING_QUEUES.lock();
+        let queues = DISK_BLOCKING_QUEUES.read();
-        for (root, weak) in queues.iter() {
+        for entry in queues.iter() {
-            if path.starts_with(root)
+            #[cfg(unix)]
-                && let Some(queue) = weak.upgrade()
+            let matches = entry.canonical_root.as_ref() == root_key.as_ref();
-            {
+            #[cfg(not(unix))]
+            let matches = entry.root == root;
+            if matches && let Some(queue) = entry.queue.upgrade() {
                 return queue;
             }
         }
     }
-    use std::hash::{Hash, Hasher};
+    register_disk_blocking_queue(root, root)
-    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+}
-    path.hash(&mut hasher);
+
-    let idx = usize::try_from(hasher.finish() % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
+fn disk_blocking_queue_for_path(path: &Path) -> Arc<DiskBlockingQueue> {
-    Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
+    #[cfg(unix)]
+    {
+        let path_key = canonical_unix_path_key(path);
+        {
+            let queues = DISK_BLOCKING_QUEUES.read();
+            for entry in queues.iter() {
+                if unix_canonical_key_starts_with(path_key.as_ref(), entry.canonical_root.as_ref())
+                    && let Some(queue) = entry.queue.upgrade()
+                {
+                    return queue;
+                }
+            }
+        }
+        let hash = xxhash_rust::xxh3::xxh3_64(path_key.as_ref());
+        let idx = usize::try_from(hash % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
+        Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
+    }
+    #[cfg(not(unix))]
+    {
+        {
+            let queues = DISK_BLOCKING_QUEUES.read();
+            for entry in queues.iter() {
+                if path.starts_with(&entry.root)
+                    && let Some(queue) = entry.queue.upgrade()
+                {
+                    return queue;
+                }
+            }
+        }
+        use std::hash::{Hash, Hasher};
+        let mut hasher = std::collections::hash_map::DefaultHasher::new();
+        path.hash(&mut hasher);
+        let idx = usize::try_from(hasher.finish() % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
+        Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
+    }
 }
 
 /// Serialize a bucket's local metadata commits with physical bucket removal.
@@ -2194,32 +2658,46 @@ fn disk_blocking_queue_for_path(path: &Path) -> Arc<DiskBlockingQueue> {
 /// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
 /// disconnected disks do not keep the registry alive.
 pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
-    let key = root.join(volume);
+    #[cfg(unix)]
-    let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
+    {
-    locks.retain(|_, lock| lock.strong_count() > 0);
+        let vol_bytes = volume.as_bytes();
-    if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
+        if !vol_bytes.is_empty()
-        return lock;
+            && vol_bytes[0] != b'/'
+            && vol_bytes != b"."
+            && !vol_bytes.starts_with(b"./")
+            && !vol_bytes.ends_with(b"/")
+            && !vol_bytes.ends_with(b"/.")
+            && is_canonical_unix_key_bytes(vol_bytes)
+            && let std::borrow::Cow::Borrowed(root_bytes) = canonical_unix_path_key(root)
+            && !root_bytes.is_empty()
+            && root_bytes != b"/"
+        {
+            let total_len = root_bytes.len() + 1 + vol_bytes.len();
+            if total_len <= 512 {
+                let mut stack_key = [0u8; 512];
+                stack_key[..root_bytes.len()].copy_from_slice(root_bytes);
+                stack_key[root_bytes.len()] = b'/';
+                stack_key[root_bytes.len() + 1..total_len].copy_from_slice(vol_bytes);
+                return DISK_VOLUME_MUTATION_LOCKS.get_or_create_by_key(&stack_key[..total_len], || RwLock::new(()));
+            }
+        }
     }
-
+    let key = root.join(volume);
-    let lock = Arc::new(RwLock::new(()));
+    DISK_VOLUME_MUTATION_LOCKS.get_or_create(&key, || RwLock::new(()))
-    locks.insert(key, Arc::downgrade(&lock));
-    lock
 }
 
 fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
-    let mut locks = DISK_NAMESPACE_MUTATION_LOCKS.lock();
+    DISK_NAMESPACE_MUTATION_LOCKS.get_or_create(path, || AsyncMutex::new(()))
-    locks.retain(|_, lock| lock.strong_count() > 0);
-    if let Some(lock) = locks.get(path).and_then(Weak::upgrade) {
-        return lock;
-    }
-
-    let lock = Arc::new(AsyncMutex::new(()));
-    locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
-    lock
 }
 
-static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
+#[cfg(unix)]
-    LazyLock::new(|| Mutex::new(HashMap::new()));
+#[inline]
+fn disk_namespace_mutation_lock_by_key(key: &[u8]) -> Arc<NamespaceMutationLock> {
+    DISK_NAMESPACE_MUTATION_LOCKS.get_or_create_by_key(key, || AsyncMutex::new(()))
+}
+
+static DISK_METADATA_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<NamespaceMutationLock>> =
+    LazyLock::new(ShardedPathLockRegistry::new);
 
 /// Serializes the complete xl.meta read/modify/commit transaction. This domain
 /// precedes namespace/volume publication locks, whose narrower syscall leases
@@ -2233,17 +2711,7 @@ pub(crate) async fn acquire_metadata_mutation_lease(
     object: &Path,
     owner: Option<Arc<dyn Send + Sync>>,
 ) -> Arc<MetadataMutationLease> {
-    let lock = {
+    let lock = DISK_METADATA_MUTATION_LOCKS.get_or_create(object, || AsyncMutex::new(()));
-        let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
-        locks.retain(|_, lock| lock.strong_count() > 0);
-        if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
-            lock
-        } else {
-            let lock = Arc::new(AsyncMutex::new(()));
-            locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
-            lock
-        }
-    };
     Arc::new(MetadataMutationLease {
         _guard: lock.lock_owned().await,
         _owner: owner,
@@ -2306,17 +2774,45 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
 ) -> Arc<NamespaceMutationLease> {
     // Metadata ownership precedes namespace paths in lexical order, then volume
     // ownership. Deduplicate identical keys so TMP-to-TMP publication cannot self-lock.
-    let destination_lock = disk_namespace_mutation_lock(destination_object);
+    #[cfg(unix)]
-    let (namespace_guard, staging_guard) = match staging_prefix {
+    let (namespace_guard, staging_guard) = {
-        Some(source) if source < destination_object => {
+        let dst_key = canonical_unix_path_key(destination_object);
-            let staging = disk_namespace_mutation_lock(source).lock_owned().await;
+        let destination_lock = disk_namespace_mutation_lock_by_key(dst_key.as_ref());
-            (destination_lock.lock_owned().await, Some(staging))
+        match staging_prefix {
+            Some(source) => {
+                let src_key = canonical_unix_path_key(source);
+                match src_key.as_ref().cmp(dst_key.as_ref()) {
+                    std::cmp::Ordering::Less => {
+                        let staging = disk_namespace_mutation_lock_by_key(src_key.as_ref()).lock_owned().await;
+                        (destination_lock.lock_owned().await, Some(staging))
+                    }
+                    std::cmp::Ordering::Greater => {
+                        let destination = destination_lock.lock_owned().await;
+                        (
+                            destination,
+                            Some(disk_namespace_mutation_lock_by_key(src_key.as_ref()).lock_owned().await),
+                        )
+                    }
+                    std::cmp::Ordering::Equal => (destination_lock.lock_owned().await, None),
+                }
+            }
+            None => (destination_lock.lock_owned().await, None),
         }
-        Some(source) if source > destination_object => {
+    };
-            let destination = destination_lock.lock_owned().await;
+    #[cfg(not(unix))]
-            (destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
+    let (namespace_guard, staging_guard) = {
+        let destination_lock = disk_namespace_mutation_lock(destination_object);
+        match staging_prefix {
+            Some(source) if source < destination_object => {
+                let staging = disk_namespace_mutation_lock(source).lock_owned().await;
+                (destination_lock.lock_owned().await, Some(staging))
+            }
+            Some(source) if source > destination_object => {
+                let destination = destination_lock.lock_owned().await;
+                (destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
+            }
+            _ => (destination_lock.lock_owned().await, None),
         }
-        _ => (destination_lock.lock_owned().await, None),
     };
     let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
     let blocking_queue = disk_blocking_queue_for_root(root);
@@ -11992,6 +12488,174 @@ mod tests {
         );
         assert!(preserved_final.exists(), "final file must remain when intermediate removal fails");
     }
+
+    #[cfg(unix)]
+    #[test]
+    fn canonical_unix_path_key_preserves_std_path_equality_and_prefix_semantics() {
+        let samples = [
+            "",
+            "/",
+            "///",
+            "/.",
+            "/./.",
+            ".",
+            "./",
+            "./.",
+            "./a",
+            "././a/",
+            ".//a/.",
+            "a",
+            "a/",
+            "a/.",
+            "a/b/c",
+            "a/b/c/",
+            "a//b/./c/.",
+            "/tmp/d0/bucket/obj",
+            "/tmp/d0/bucket/obj/",
+            "/tmp/d0//bucket/./obj/.",
+            "/tmp/d0/.rustfs.sys/multipart/sha/upload",
+            "/proc/self/fd/12/.",
+            "/proc/self/fd/12",
+            "/proc/self/fd/12/./bucket/obj",
+            "/proc/self/fd/12/bucket/obj",
+            "/tmp/d1",
+            "/tmp/d10",
+            "a/../b",
+            "a/../b/",
+            "/utf8/\u{00af}/segment/",
+        ];
+        for s in samples {
+            let p = Path::new(s);
+            let fast = canonical_unix_path_key(p);
+            let slow = canonical_unix_path_key_slow(p);
+            assert_eq!(fast.as_ref(), slow.as_ref(), "fast and slow key mismatch for {s:?}");
+        }
+        for a in samples {
+            for b in samples {
+                let pa = Path::new(a);
+                let pb = Path::new(b);
+                let ka = canonical_unix_path_key(pa);
+                let kb = canonical_unix_path_key(pb);
+                assert_eq!(
+                    pa == pb,
+                    ka.as_ref() == kb.as_ref(),
+                    "Path::eq vs canonical key mismatch for {a:?} and {b:?}"
+                );
+                assert_eq!(
+                    pa.starts_with(pb),
+                    unix_canonical_key_starts_with(ka.as_ref(), kb.as_ref()),
+                    "Path::starts_with vs canonical key prefix mismatch for {a:?} and {b:?}"
+                );
+            }
+        }
+
+        let dir = tempdir().expect("tempdir");
+        let p_clean = dir.path().join("bucket/object");
+        let p_trailing = PathBuf::from(format!("{}/", p_clean.display()));
+        let p_redundant = PathBuf::from(format!("{}//bucket/./object/.", dir.path().display()));
+        let lock1 = disk_namespace_mutation_lock(&p_clean);
+        let lock2 = disk_namespace_mutation_lock(&p_trailing);
+        let lock3 = disk_namespace_mutation_lock(&p_redundant);
+        assert!(Arc::ptr_eq(&lock1, &lock2));
+        assert!(Arc::ptr_eq(&lock1, &lock3));
+
+        let vlock1 = disk_volume_mutation_lock(dir.path(), "bucket");
+        let vlock2 = disk_volume_mutation_lock(&PathBuf::from(format!("{}/.", dir.path().display())), "bucket/");
+        assert!(Arc::ptr_eq(&vlock1, &vlock2));
+
+        let io_root = dir.path().join("io_root/.");
+        let q_reg = register_disk_blocking_queue(dir.path(), &io_root);
+        let q_root = disk_blocking_queue_for_root(dir.path());
+        let q_path = disk_blocking_queue_for_path(&dir.path().join("io_root/bucket/key"));
+        assert!(Arc::ptr_eq(&q_reg, &q_root));
+        assert!(Arc::ptr_eq(&q_reg, &q_path));
+    }
+
+    #[test]
+    fn sharded_path_lock_registry_reuses_slots_and_bounds_dead_weak_entries() {
+        let registry: ShardedPathLockRegistry<AsyncMutex<()>> = ShardedPathLockRegistry::new();
+        let target = Path::new("/tmp/sharded-lock-test/persistent-key");
+        let held = registry.get_or_create(target, || AsyncMutex::new(()));
+        let held_again = registry.get_or_create(target, || AsyncMutex::new(()));
+        assert!(Arc::ptr_eq(&held, &held_again), "live lock must be shared");
+
+        // Re-acquiring a key after drop must reuse the existing slot in-place.
+        let transient = Path::new("/tmp/sharded-lock-test/reused-slot");
+        drop(registry.get_or_create(transient, || AsyncMutex::new(())));
+        let reused = registry.get_or_create(transient, || AsyncMutex::new(()));
+        drop(reused);
+
+        // Flood transient keys across shards and verify `held` is never evicted
+        // and total dead entries remain bounded by the amortized GC threshold.
+        for i in 0..(PATH_LOCK_SHARDS * PATH_LOCK_INITIAL_GC_THRESHOLD * 3) {
+            let p = PathBuf::from(format!("/tmp/sharded-lock-test/ephemeral-{i}"));
+            drop(registry.get_or_create(&p, || AsyncMutex::new(())));
+        }
+        let still_held = registry.get_or_create(target, || AsyncMutex::new(()));
+        assert!(Arc::ptr_eq(&held, &still_held), "GC must never evict a live lock");
+        for shard in &registry.shards {
+            let guard = shard.lock();
+            assert!(
+                guard.entries.len() <= guard.next_gc_threshold,
+                "shard entries ({}) exceeded GC threshold ({})",
+                guard.entries.len(),
+                guard.next_gc_threshold
+            );
+        }
+    }
+
+    #[test]
+    fn sharded_path_cache_normalizes_keys_reuses_slots_and_filters_prefixes() {
+        let cache: ShardedPathCache<usize> = ShardedPathCache::new(2);
+        let clean = Path::new("/tmp/sharded-cache-test/bucket/obj");
+        let non_canonical = Path::new("/tmp/sharded-cache-test//bucket/./obj/.");
+
+        cache.insert(clean, 10);
+        assert_eq!(cache.with_get(non_canonical, |v| *v), Some(10));
+
+        cache.with_get_mut(non_canonical, |v| *v += 5);
+        assert_eq!(cache.with_get(clean, |v| *v), Some(15));
+
+        cache.remove_if_mut(clean, |v| *v == 99);
+        assert_eq!(cache.with_get(clean, |v| *v), Some(15));
+        cache.remove_if_mut(non_canonical, |v| *v == 15);
+        assert_eq!(cache.with_get(clean, |v| *v), None);
+
+        cache.insert(Path::new("/tmp/sharded-cache-test/bucket/a"), 1);
+        cache.insert(Path::new("/tmp/sharded-cache-test/bucket/b"), 2);
+        cache.insert(Path::new("/tmp/sharded-cache-test/bucket-other/c"), 3);
+        cache.retain_outside_prefix(Path::new("/tmp/sharded-cache-test//bucket/."));
+        assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket/a"), |v| *v), None);
+        assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket/b"), |v| *v), None);
+        assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket-other/c"), |v| *v), Some(3));
+
+        for i in 0..512 {
+            cache.insert(&PathBuf::from(format!("/tmp/sharded-cache-test/cap/{i}")), i);
+        }
+        for shard in &cache.shards {
+            assert!(shard.lock().len() <= 2);
+        }
+
+        #[cfg(unix)]
+        {
+            let overwrite_cache: ShardedPathCache<usize> = ShardedPathCache::new(2);
+            let target_shard = ShardedPathCache::<usize>::shard_index(b"/tmp/sharded-cache-test/same-0");
+            let mut same_shard_paths = vec![PathBuf::from("/tmp/sharded-cache-test/same-0")];
+            let mut candidate = 1usize;
+            while same_shard_paths.len() < 2 {
+                let p = PathBuf::from(format!("/tmp/sharded-cache-test/same-{candidate}"));
+                if ShardedPathCache::<usize>::shard_index(canonical_unix_path_key(&p).as_ref()) == target_shard {
+                    same_shard_paths.push(p);
+                }
+                candidate += 1;
+            }
+            overwrite_cache.insert(&same_shard_paths[0], 100);
+            overwrite_cache.insert(&same_shard_paths[1], 200);
+            overwrite_cache.insert(&same_shard_paths[0], 101);
+            assert_eq!(overwrite_cache.with_get(&same_shard_paths[0], |v| *v), Some(101));
+            assert_eq!(overwrite_cache.with_get(&same_shard_paths[1], |v| *v), Some(200));
+        }
+    }
 }
 
 #[cfg(all(test, target_os = "linux"))]

crates/ecstore/src/services/rebalance/worker.rs

@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     }
 }
 
+fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
+    match rebalance_error_source(err) {
+        Error::Lock(err) => is_rebalance_transient_lock_error(err),
+        // No metadata was written when lock admission failed. Each attempt
+        // must acquire its quorum again before loading or saving this run.
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket,
+            object,
+            ..
+        } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
+        _ => false,
+    }
+}
+
 fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return true;
@@ -385,9 +400,7 @@ where
         match result {
             Ok(value) => return Ok(value),
             Err(err) => {
-                if attempt.saturating_add(1) >= max_attempts.max(1)
+                if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
-                    || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
-                {
                     return Err(err);
                 }
                 #[cfg(test)]
@@ -719,6 +732,82 @@ mod error_source_tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
+        let mut attempts = 0;
+        retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(if attempts == 1 {
+                resolve_rebalance_meta_save_result(
+                    Err(rebalance_meta_lock_error(
+                        rustfs_lock::LockError::QuorumNotReached {
+                            required: 1,
+                            achieved: 0,
+                        },
+                        "write",
+                    )),
+                    "stop_rebalance",
+                )
+            } else {
+                Ok(())
+            })
+        })
+        .await
+        .expect("a later quorum-protected admission must be allowed to succeed");
+        assert_eq!(attempts, 2);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
+        for max_attempts in [0, 1, 3] {
+            let mut attempts = 0;
+            let result = retry_rebalance_metadata_access(None, max_attempts, || {
+                attempts += 1;
+                std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )))
+            })
+            .await;
+            let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
+            assert_eq!(attempts, max_attempts.max(1));
+            assert!(matches!(
+                rebalance_error_source(&error),
+                Error::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    required: 1,
+                    achieved: 0,
+                    ..
+                }
+            ));
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
+        let mut attempts = 0;
+        let result = retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(Err::<(), _>(if attempts == 1 {
+                rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )
+            } else {
+                Error::OperationCanceled
+            }))
+        })
+        .await;
+        assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
+        assert!(matches!(result, Err(Error::OperationCanceled)));
+    }
+
     #[tokio::test]
     async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
         for err in [
@@ -731,6 +820,20 @@ mod error_source_tests {
                 required: 3,
                 achieved: 2,
             },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "data-bucket".to_string(),
+                object: REBAL_META_NAME.to_string(),
+                required: 1,
+                achieved: 0,
+            },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
+                object: "other-metadata.bin".to_string(),
+                required: 1,
+                achieved: 0,
+            },
             Error::other("stale rebalance run rejected: lock acquisition timed out"),
             Error::other("rebalance distributed run fence lost"),
             Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),

crates/ecstore/src/set_disk/mod.rs

@@ -4696,6 +4696,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
             .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
 }
 
+// Reusing a held writer additionally requires the same quorum, so repeated
+// clients must not give different weight to an otherwise identical domain.
+fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    !left.is_empty()
+        && left.len() == right.len()
+        && left.iter().all(|client| {
+            left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+                && right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+        })
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5645,9 +5656,9 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            (true, true) => {
+            // Distributed lock clients receive the raw ObjectKey; the pool
-                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
+            // namespace is not part of the key stored by their backends.
-            }
+            (true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
             _ => false,
         }
     }
@@ -9882,7 +9893,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn namespace_lock_domain_includes_pool_namespace() {
+    async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
 
         let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9921,11 +9932,25 @@ mod tests {
             "sets in the same pool share the object namespace lock domain"
         );
         assert!(
-            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
+            same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pool namespaces must not be deduplicated solely by identical clients"
+            "different pools using identical backends lock the same raw object key"
         );
     }
 
+    #[test]
+    fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
+        let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
+        assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
+        assert!(!same_distributed_lock_quorum(
+            &[a.clone(), a.clone(), b.clone()],
+            &[a.clone(), b.clone(), b]
+        ));
+        assert!(!same_distributed_lock_quorum(&[], &[]));
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -10060,7 +10060,7 @@ mod tests {
 
         // Real disk callbacks and health probes must share wall-clock deadlines.
         // Advancing Tokio time can expire unrelated healthy-drive probes.
-        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
             let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
             let bucket = "mpu-quorum-success-physical";
@@ -10087,6 +10087,8 @@ mod tests {
                 local.observe_multipart_cleanup_for_test(&upload_path)
             }).collect::<Vec<_>>();
             let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
+            // Fault only the held leg; healthy disks keep their normal I/O budgets.
+            let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
             let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
             let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
             let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10243,7 +10245,15 @@ mod tests {
                 let size = copy_info.get_actual_size().expect("actual source size");
                 copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
                 tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
-                    .await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
+                    .await.expect("cross-key Copy cannot wait on the old successful logical owner")
+                    .unwrap_or_else(|error| {
+                        let health = disks.iter().enumerate().map(|(slot, disk)| {
+                            let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
+                            let metrics = local.metrics_snapshot();
+                            (slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
+                        }).collect::<Vec<_>>();
+                        panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
+                    });
                 drop(copy_info);
                 let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
                 let mut actual = Vec::new();

crates/ecstore/src/store/heal.rs

@@ -3692,6 +3692,177 @@ mod tests {
         );
     }
 
+    // The RPC lock service uses the raw ObjectKey and one backend per node,
+    // even when the requesting sets carry different pool namespace strings.
+    async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
+        use rustfs_lock::client::{LockClient, local::LocalClient};
+
+        let clients = || {
+            (0..2)
+                .map(|_| {
+                    Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
+                })
+                .collect::<Vec<_>>()
+        };
+        let first = clients();
+        let second = if shared { first.clone() } else { clients() };
+        let mut pools = source.pools.clone();
+        for (pool, lockers) in pools.iter_mut().zip([first, second]) {
+            for set in &mut Arc::make_mut(pool).disk_set {
+                Arc::make_mut(set).lockers = lockers.clone();
+            }
+        }
+        let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
+        let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
+        Arc::new(ECStore {
+            id: Uuid::new_v4(),
+            disk_map: Default::default(),
+            pools,
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
+            pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
+            rebalance_meta: tokio::sync::RwLock::new(None),
+            decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(state),
+            ctx: Arc::clone(&source.ctx),
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
+        let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let first_lock = first
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("first pool lock");
+        let second_lock = second
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("second pool lock");
+        let guard = first_lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("first writer");
+        assert!(
+            second_lock
+                .get_write_lock(std::time::Duration::from_millis(150))
+                .await
+                .is_err(),
+            "different pool namespaces still conflict on the same backend and object key"
+        );
+        drop(guard);
+        second_lock
+            .get_write_lock(std::time::Duration::from_secs(3))
+            .await
+            .expect("second writer should acquire after the first releases");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    dry_run: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
+        .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let missing = remove_pool_meta_shard(&store, 1).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(10),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("cross-pool repair must not reenter the shared writer")
+        .expect("pool metadata repair should return a result");
+        assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+        assert!(
+            missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "the selected missing shard must be restored"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
+        let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let lock = target
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("target lock");
+        let guard = lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("hold target lock");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    POOL_META_NAME,
+                    "",
+                    &HealOpts {
+                        pool: Some(1),
+                        dry_run: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            barrier.wait_until_external_heal_target_lock_attempted(),
+        )
+        .await
+        .expect("inspection must attempt the independent target lock");
+        assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
+        drop(guard);
+        let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
+            .await
+            .expect("inspection should progress after target lock release")
+            .expect("heal task should not panic")
+            .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {

crates/filemeta/src/metacache.rs

@@ -925,6 +925,13 @@ impl MetaCacheEntriesSorted {
 
 const METACACHE_STREAM_VERSION: u8 = 2;
 const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
+/// Upper bound on one incremental reservation while accumulating a single
+/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
+/// up to this bound so a legitimately large xl.meta does not pay one small
+/// read per chunk, while a corrupt length prefix still cannot force a
+/// reservation larger than the payload that has already arrived (see
+/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
+const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
 /// msgpack `false`: the entry-stream terminator, and the only byte a legacy
 /// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
 const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1157,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         // A whole xl.meta can legitimately exceed the limit on its individual
         // msgpack values. Grow only as payload arrives, so a corrupt length
         // prefix cannot cause a speculative allocation of the advertised size.
+        let mut chunk = METACACHE_READ_CHUNK_SIZE;
         while self.buf.len() < read_size {
             let start = self.buf.len();
             let extra = if lookahead {
                 let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
-                (read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).clamp(min_chunk, chunk)
             } else {
-                (read_size - start).min(METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).min(chunk)
             };
             if let Err(e) = self.buf.try_reserve(extra) {
                 let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1194,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
                     return Err(err);
                 }
             }
+            chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
         }
 
         self.pos = read_size;
@@ -2155,6 +2164,76 @@ mod tests {
         }
     }
 
+    /// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
+    /// decode completely even when the aggregate stream exceeds the per-value
+    /// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
+    /// element.
+    #[tokio::test]
+    async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
+        // ~2.9 KiB of metadata per entry matches the per-entry cost observed in
+        // the #9120 production report; 6,000 entries push the stream past 16 MiB.
+        const ENTRIES: usize = 6_000;
+        let metadata = vec![0xab; 2_880];
+        let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
+            .map(|i| MetaCacheEntry {
+                name: format!("bigdir/object-{i:05}"),
+                metadata: metadata.clone(),
+                ..Default::default()
+            })
+            .collect();
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("wide listing must encode");
+        writer.close().await.expect("stream must close");
+        let bytes = output.into_inner();
+        assert!(
+            bytes.len() > 16 << 20,
+            "fixture must exceed the per-value limit, got {} bytes",
+            bytes.len()
+        );
+        let mut reader = MetacacheReader::new(Cursor::new(bytes));
+        let decoded = reader.read_all().await.expect("wide listing must decode");
+        assert_eq!(decoded, entries);
+    }
+
+    /// The #9120 failure shape seen in production: one entry whose metadata is
+    /// an xl.meta that grew past 16 MiB with its version history, listed among
+    /// ordinary neighbors and skipped past without losing the framing.
+    #[tokio::test]
+    async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
+        let small = |i: u8| MetaCacheEntry {
+            name: format!("bigdir/object-{i}"),
+            metadata: vec![i; 64],
+            ..Default::default()
+        };
+        let oversized = MetaCacheEntry {
+            name: "bigdir/huge-versions".to_string(),
+            metadata: vec![0xcd; (16 << 20) + 12_345],
+            ..Default::default()
+        };
+        let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("mixed listing must encode");
+        writer.close().await.expect("stream must close");
+
+        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
+        assert_eq!(
+            reader.peek().await.expect("first entry").expect("present"),
+            entries[0],
+            "an ordinary entry before the oversized one must decode"
+        );
+        reader.skip(1).await.expect("skip the first entry");
+        assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
+        assert_eq!(
+            reader.peek().await.expect("oversized entry").expect("present"),
+            oversized,
+            "an entry past the per-value limit must round-trip whole"
+        );
+        reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
+        assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
+    }
+
     #[test]
     fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
         let version_id = Uuid::new_v4();

crates/heal/src/error.rs

@@ -128,6 +128,20 @@ impl Error {
         Error::TransientSkip { message: message.into() }
     }
 
+    /// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
+    /// Callers must cut their own concurrency instead of retrying at full speed.
+    pub(crate) fn is_slow_down(&self) -> bool {
+        match self {
+            Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
+            Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
+            Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
+                message.contains("Please reduce your request rate")
+            }
+            Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
+            _ => false,
+        }
+    }
+
     /// Whether a heal operation can be retried without changing its inputs.
     pub(crate) fn is_recoverable_heal(&self) -> bool {
         match classify_heal_error(self) {
@@ -449,6 +463,22 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
+    #[test]
+    fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
+        assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
+        assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
+        assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
+        assert!(
+            Error::TaskExecutionFailed {
+                message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
+            }
+            .is_slow_down()
+        );
+        assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
+        assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
+    }
+
     #[test]
     fn pool_metadata_failures_keep_their_typed_recovery_classification() {
         use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};

crates/heal/src/heal/erasure_healer.rs

@@ -14,6 +14,7 @@
 
 use crate::error::{HealErrorClass, classify_heal_error};
 use crate::heal::{
+    pacing::HealPageRate,
     progress::{HealProgress, add_bytes, increment_counter},
     resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -128,12 +129,12 @@ pub struct ErasureSetHealer {
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
     replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
     work_parent: Option<super::manager::HealWork>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 async fn acquire_page_permit(
     semaphore: Arc<Semaphore>,
-    pacer: Option<&super::pacing::MainlinePacer>,
+    rate: &HealPageRate,
     cancel: &tokio_util::sync::CancellationToken,
 ) -> Result<tokio::sync::OwnedSemaphorePermit> {
     let acquire = || async {
@@ -146,18 +147,16 @@ async fn acquire_page_permit(
     let mut paid_pause = false;
     loop {
         let permit = acquire().await?;
-        if let Some(pacer) = pacer {
+        // Keep the real permit on the low-pressure path. Every acquisition
-            // Keep the real permit on the low-pressure path. Every acquisition
+        // gets a fresh decision, including a waiter that queued a second
-            // gets a fresh decision, including a waiter that queued a second
+        // time. One completed pause is a bounded minimum-progress grant.
-            // time. One completed pause is a bounded minimum-progress grant.
+        match rate.admission_decision() {
-            match pacer.admission_decision() {
+            super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
-                super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
+                drop(permit);
-                    drop(permit);
+                paid_pause = rate.wait_after_admission(cancel, pressure).await?;
-                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
+                continue;
-                    continue;
-                }
-                _ => {}
             }
+            _ => {}
         }
         return Ok(permit);
     }
@@ -166,7 +165,7 @@ async fn acquire_page_permit(
 #[cfg(test)]
 mod mainline_pacing_tests {
     use super::*;
-    use crate::heal::pacing::{MainlinePacer, TestPressure};
+    use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
     use tokio_util::sync::CancellationToken;
@@ -180,15 +179,15 @@ mod mainline_pacing_tests {
             .await
             .expect("existing object owns capacity");
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
-        let pacer = Arc::new(
+        let pacer =
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
-        );
+        let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
         let cancel = CancellationToken::new();
         let waiting = tokio::spawn({
             let semaphore = semaphore.clone();
-            let pacer = pacer.clone();
+            let rate = rate.clone();
             let cancel = cancel.clone();
-            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
+            async move { acquire_page_permit(semaphore, &rate, &cancel).await }
         });
         tokio::task::yield_now().await;
         provider.active.store(100, Ordering::SeqCst);
@@ -200,12 +199,13 @@ mod mainline_pacing_tests {
         assert_eq!(semaphore.available_permits(), 1);
         let deadline = tokio::time::timeout(
             Duration::from_millis(10),
-            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
+            acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
         )
         .await;
         assert!(deadline.is_err());
         assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
-        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
+        let unpaced = HealPageRate::new(1, None);
+        let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
             .await
             .expect("unpaced admission");
         assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,10 +221,11 @@ mod mainline_pacing_tests {
             let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
             let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
+            let rate = HealPageRate::new(1, Some(pacer));
             let cancel = CancellationToken::new();
-            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
-            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
             assert!(futures::poll!(first.as_mut()).is_pending());
             assert!(futures::poll!(second.as_mut()).is_pending());
             drop(occupied);
@@ -329,6 +330,20 @@ impl ErasureSetHealer {
         }
     }
 
+    /// Page fan-out for one whole erasure-set pass (stable across buckets).
+    fn pass_page_concurrency_limit(&self) -> usize {
+        if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
+            // Versions of the same object must not race without its namespace lock.
+            1
+        } else {
+            Self::effective_heal_page_object_concurrency_for_source(
+                self.source,
+                self.heal_opts.scan_mode,
+                self.replacement_task_id.is_some(),
+            )
+        }
+    }
+
     /// Map the shared heal error class onto this healer's per-object actions.
     ///
     /// `classify_heal_error` is the only classifier; this adapter decides what
@@ -373,7 +388,7 @@ impl ErasureSetHealer {
         }
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }
@@ -464,12 +479,7 @@ impl ErasureSetHealer {
             None
         };
         let buckets = admin_buckets.as_deref().unwrap_or(buckets);
-        if checkpoint_manager
+        if checkpoint_manager.admin_completed().await {
-            .get_checkpoint()
-            .await
-            .admin
-            .is_some_and(|admin| admin.completed)
-        {
             self.initialize_progress(buckets, &resume_manager.get_state().await).await;
             return Ok(());
         }
@@ -490,14 +500,7 @@ impl ErasureSetHealer {
         }
 
         if self.admin_task.is_some() {
-            let mut admin = checkpoint_manager
+            checkpoint_manager.complete_admin().await?;
-                .get_checkpoint()
-                .await
-                .admin
-                .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
-            admin.completed = true;
-            admin.outcome.finish(None);
-            checkpoint_manager.set_admin(admin).await?;
         } else {
             checkpoint_manager.cleanup().await?;
             resume_manager.cleanup().await?;
@@ -718,7 +721,10 @@ impl ErasureSetHealer {
             }
             return Err(Error::DanglingDeleteDeferred { retry_not_before });
         }
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        // One rate controller for the whole pass: SlowDown cool-down and page
+        // fan-out persist across buckets instead of resetting at each one.
+        let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
+        let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
 
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -726,14 +732,17 @@ impl ErasureSetHealer {
             component = LOG_COMPONENT_HEAL,
             subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
             set_disk_id,
-            current_bucket_index = checkpoint.current_bucket_index,
+            current_bucket_index,
-            current_object_index = checkpoint.current_object_index,
+            current_object_index,
             state = "resuming",
             "Erasure set resumed"
         );
 
         // 2. initialize progress
         self.initialize_progress(buckets, &state).await;
+        // Snapshot before any baseline write: set_progress_baseline overwrites
+        // the saved generation, which would hide a real generation mismatch.
+        let checkpoint = checkpoint_manager.progress_snapshot().await;
         let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
             let baseline = self.progress.read().await;
             (
@@ -760,17 +769,7 @@ impl ErasureSetHealer {
         let mut successful_objects = state.successful_objects;
         let mut failed_objects = state.failed_objects;
         let mut skipped_objects = state.skipped_objects;
-        let checkpoint_has_progress = checkpoint.baseline_known
+        let checkpoint_has_progress = checkpoint.has_progress();
-            || checkpoint.successful_objects > 0
-            || checkpoint.failed_object_count > 0
-            || checkpoint.skipped_object_count > 0
-            || checkpoint.skipped_new_versions > 0
-            || checkpoint.skipped_ilm_expired > 0
-            || checkpoint.processed_bytes > 0
-            || checkpoint.total_objects > 0
-            || checkpoint.total_bytes > 0
-            || checkpoint.baseline_generation.is_some()
-            || checkpoint.counter_unknown;
         let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
         let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
         if checkpoint_has_progress {
@@ -854,6 +853,7 @@ impl ErasureSetHealer {
                     resume_manager,
                     checkpoint_manager,
                     state.start_time,
+                    &page_rate,
                 )
                 .await;
 
@@ -1020,14 +1020,14 @@ impl ErasureSetHealer {
         if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
             if self.replacement_task_id.is_some() {
                 let state = resume_manager.get_state().await;
-                let checkpoint = checkpoint_manager.get_checkpoint().await;
+                let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
                 if state.can_retry()
                     && failed_objects == 0
                     && failed_buckets == 0
                     && skipped_objects > 0
-                    && skipped_objects == checkpoint.dangling_delete_grace_objects
+                    && skipped_objects == grace_objects
-                    && !checkpoint.counter_unknown
+                    && !counter_unknown
-                    && let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
+                    && let Some(retry_not_before) = retry_not_before
                 {
                     // Publish the generation's rewind and wait first. If the
                     // checkpoint reset is interrupted, initialization reapplies
@@ -1167,14 +1167,9 @@ impl ErasureSetHealer {
 
         let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
         let checkpoint_key = compose_key(&object_key, None);
-        let checkpoint = checkpoint_manager.get_checkpoint().await;
+        if checkpoint_manager.is_object_settled(&checkpoint_key).await {
-        if checkpoint.processed_objects.contains(&checkpoint_key)
-            || checkpoint.failed_objects.contains(&checkpoint_key)
-            || checkpoint.skipped_objects.contains(&checkpoint_key)
-        {
             return Ok(());
         }
-        drop(checkpoint);
 
         self.verify_replacement_identity_fence("pool metadata").await?;
         resume_manager
@@ -1428,7 +1423,7 @@ impl ErasureSetHealer {
 
     /// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
     #[allow(clippy::too_many_arguments)]
-    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
+    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
     async fn heal_bucket_with_resume(
         &self,
         bucket: &str,
@@ -1442,6 +1437,7 @@ impl ErasureSetHealer {
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
         started_at_secs: u64,
+        page_rate: &Arc<HealPageRate>,
     ) -> Result<bool> {
         debug!(
             target: "rustfs::heal::erasure_healer",
@@ -1491,17 +1487,6 @@ impl ErasureSetHealer {
         // detects a non-advancing cursor even though next_marker embeds a
         // transient cache id that changes between identical pages.
         let mut previous_page_last: Option<(String, Option<String>)> = None;
-        let page_concurrency_limit =
-            if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
-                // Versions of the same object must not race without its namespace lock.
-                1
-            } else {
-                Self::effective_heal_page_object_concurrency_for_source(
-                    self.source,
-                    self.heal_opts.scan_mode,
-                    self.replacement_task_id.is_some(),
-                )
-            };
         let in_flight = Arc::new(AtomicUsize::new(0));
         // Per-bucket sample caps for per-object warn! lines: a flapping rebuild
         // disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1523,9 +1508,7 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
-            if let Some(pacer) = &self.mainline_pacer {
+            page_rate.wait_before_page(&self.cancel_token).await?;
-                pacer.wait(&self.cancel_token).await?;
-            }
             self.verify_replacement_identity_fence("page scan").await?;
             if let Some(expected) = incarnation {
                 self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1547,9 +1530,8 @@ impl ErasureSetHealer {
                     .await?
             };
             let page_is_empty = objects.is_empty();
-            let checkpoint = checkpoint_manager.get_checkpoint().await;
             let page_resume_index = *current_object_index;
-            let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
+            let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
             let mut page_tasks = FuturesUnordered::new();
             let mut completed_in_page = 0usize;
 
@@ -1563,10 +1545,7 @@ impl ErasureSetHealer {
 
                 // Per-version dedup identity — the single canonical key.
                 let key = compose_key(&item.name, item.version_id.as_deref());
-                if checkpoint.processed_objects.contains(&key)
+                if checkpoint_manager.is_object_settled(&key).await {
-                    || checkpoint.failed_objects.contains(&key)
-                    || checkpoint.skipped_objects.contains(&key)
-                {
                     continue;
                 }
 
@@ -1724,7 +1703,7 @@ impl ErasureSetHealer {
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
                 let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
-                let mainline_pacer = self.mainline_pacer.clone();
+                let page_rate = page_rate.clone();
 
                 let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
                 let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1736,14 +1715,15 @@ impl ErasureSetHealer {
                     .map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
                 let work = async move {
                     let mut retries = 0;
+                    let mut saw_slow_down = false;
                     let mut admin_identity = admin_identity;
                     let mut admin_task_id = admin_task_id;
                     loop {
-                        let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
+                        let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
 
                         let permit = match permit {
                             Ok(permit) => permit,
-                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
+                            Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
                         };
 
                         let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1751,7 +1731,8 @@ impl ErasureSetHealer {
                         if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
                             let (result, canonical) =
                                 admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
-                            return (dedup_key, object_name, version_id, result, canonical);
+                            let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
+                            return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
                         }
 
                         // Always go through heal_object. Genuine absence flows through
@@ -1853,18 +1834,23 @@ impl ErasureSetHealer {
 
                         drop(in_flight_guard);
                         drop(permit);
+                        if matches!(&result.1, Err(err) if err.is_slow_down()) {
+                            saw_slow_down = true;
+                        }
                         if retry_replacement
                             && retries < REPLACEMENT_OBJECT_MAX_RETRIES
                             && matches!(&result.1, Err(Error::TransientSkip { .. }))
                         {
                             retries += 1;
                             tokio::select! {
-                                _ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
+                                _ = cancel_token.cancelled() => {
+                                    return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
+                                }
                                 _ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
                             }
                             continue;
                         }
-                        return (dedup_key, object_name, version_id, result, None);
+                        return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
                     }
                 };
                 page_tasks.push(async move {
@@ -1875,7 +1861,7 @@ impl ErasureSetHealer {
                         };
                         match result {
                             Ok(result) => result,
-                            Err(error) => (key, object, version, (0, Err(error)), None),
+                            Err(error) => (key, object, version, (0, Err(error), false), None),
                         }
                     } else {
                         work.await
@@ -1884,7 +1870,16 @@ impl ErasureSetHealer {
             }
 
             while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
-                let (object_size, result) = result;
+                let (object_size, result, saw_slow_down) = result;
+                // Exactly one rate observation per finished object, including
+                // SlowDown that later succeeded on a replacement retry.
+                page_rate.observe_object_outcome(
+                    &match &result {
+                        Ok(_) => Ok(true),
+                        Err(err) => Err(err),
+                    },
+                    saw_slow_down,
+                );
                 let mut telemetry_unknown = false;
                 let checkpoint_outcome = match result {
                     Ok(true) => {
@@ -2482,6 +2477,7 @@ mod resume_loop_tests {
     use super::{
         ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
     };
+    use crate::heal::pacing::HealPageRate;
     use crate::heal::progress::HealProgress;
     use crate::heal::resume::{
         CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -3063,6 +3059,7 @@ mod resume_loop_tests {
         let mut successful = 0u64;
         let mut failed = 0u64;
         let mut skipped = 0u64;
+        let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
         let result = env
             .healer
             .heal_bucket_with_resume(
@@ -3077,6 +3074,7 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 state.start_time,
+                &page_rate,
             )
             .await;
         (processed, successful, failed, skipped, result.map(|_| ()))
@@ -3144,6 +3142,7 @@ mod resume_loop_tests {
         let mut skipped = 0;
         let started_at = env.resume.get_state().await.start_time;
 
+        let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
         let error = healer
             .heal_bucket_with_resume(
                 "b",
@@ -3157,6 +3156,7 @@ mod resume_loop_tests {
                 &env.resume,
                 &env.checkpoint,
                 started_at,
+                &page_rate,
             )
             .await
             .expect_err("a remounted target must not begin a new page scan");

crates/heal/src/heal/manager.rs

@@ -810,7 +810,14 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
 fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
     active_heals
         .iter()
-        .find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
+        .find(|(_, task)| {
+            PriorityHealQueue::make_dedup_key_for_owner(
+                &task.heal_type,
+                &task.options,
+                task.source,
+                task.expected_mrf_bucket_incarnation_id,
+            ) == key
+        })
         .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
 }
 

crates/heal/src/heal/manager/queue.rs

@@ -611,7 +611,104 @@ impl PriorityHealQueue {
 
     /// Create a deduplication key from a heal request
     pub(super) fn make_dedup_key(request: &HealRequest) -> String {
-        Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
+        Self::make_dedup_key_for_owner(
+            &request.heal_type,
+            &request.options,
+            request.source,
+            request.expected_mrf_bucket_incarnation_id,
+        )
+    }
+
+    /// Bound MRF work can only merge with work for the same bucket lifetime.
+    /// Length-prefix user-controlled components so a key cannot forge another
+    /// version, scope, or incarnation through delimiter characters.
+    pub(super) fn make_dedup_key_for_owner(
+        heal_type: &HealType,
+        options: &HealOptions,
+        source: HealRequestSource,
+        incarnation: Option<Uuid>,
+    ) -> String {
+        if source == HealRequestSource::Mrf
+            && let Some(incarnation) = incarnation
+        {
+            use std::fmt::Write as _;
+
+            let (pool, set) = heal_scope_indices(heal_type, options);
+            let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
+            let mut part = |value: &str| {
+                // Formatting into a String is infallible.
+                let _ = write!(key, "{}:{value}", value.len());
+            };
+            match heal_type {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("object");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::Metadata { bucket, object } => {
+                    part("metadata");
+                    part(bucket);
+                    part(object);
+                }
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                } => {
+                    part("ecdecode");
+                    part(bucket);
+                    part(object);
+                    match version_id {
+                        None => part("none"),
+                        Some(version_id) => {
+                            part("some");
+                            part(version_id);
+                        }
+                    }
+                }
+                HealType::DeleteMarkerPurge {
+                    bucket,
+                    object,
+                    version_id,
+                    purge,
+                } => {
+                    part("delete-marker-purge");
+                    part(bucket);
+                    part(object);
+                    part(version_id);
+                    part(&purge.bucket_incarnation_id.to_string());
+                    part(&purge.marker_incarnation_id.to_string());
+                    part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
+                }
+                HealType::Bucket { bucket } => {
+                    part("bucket");
+                    part(bucket);
+                }
+                HealType::Prefix { bucket, prefix } => {
+                    part("prefix");
+                    part(bucket);
+                    part(prefix);
+                }
+                // Cluster and erasure-set work are not bucket-bound; preserve
+                // their existing keys, including contains_erasure_set lookups.
+                HealType::Cluster | HealType::ErasureSet { .. } => {
+                    return Self::make_dedup_key_for_scope(heal_type, options);
+                }
+            }
+            return key;
+        }
+        Self::make_dedup_key_for_scope(heal_type, options)
     }
 
     pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {

crates/heal/src/heal/manager/scheduler.rs

@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
     request: &HealRequest,
     config: &HealConfig,
     provider: &Option<WorkloadSnapshotProviderRef>,
-) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
+) -> Option<crate::heal::pacing::ForegroundPacing> {
     let pace_running = request.source == HealRequestSource::Admin
         || (request.source == HealRequestSource::AutoHeal
             && matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,14 +43,13 @@ pub(super) fn running_mainline_pacer(
         return None;
     }
     provider.as_ref().and_then(|provider| {
-        crate::heal::pacing::MainlinePacer::new(
+        crate::heal::pacing::ForegroundPacing::new(
             provider.clone(),
             config.mainline_read_utilization_high_percent,
             config.mainline_write_utilization_high_percent,
             config.mainline_max_sleep,
             request.source,
         )
-        .map(Arc::new)
     })
 }
 

crates/heal/src/heal/manager/tests.rs

@@ -117,6 +117,453 @@ async fn process_manager_queue_once(manager: &HealManager) {
 
 struct MockStorage;
 
+fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
+    let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
+    request.source = HealRequestSource::Mrf;
+    request.expected_mrf_bucket_incarnation_id = Some(incarnation);
+    request.options.pool_index = Some(0);
+    request.options.set_index = Some(0);
+    request
+}
+
+async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
+    let old_incarnation = Uuid::new_v4();
+    let new_incarnation = Uuid::new_v4();
+    let old_request = incarnation_bound_mrf_fallback(old_incarnation);
+    let old_id = old_request.id.clone();
+    match existing_state {
+        "queued" => {
+            let receipt = manager
+                .submit_heal_request_with_receipt(old_request)
+                .await
+                .expect("old fallback must be admitted");
+            assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+        }
+        "active" => {
+            manager
+                .active_heals
+                .lock()
+                .await
+                .insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
+        }
+        "retrying" => {
+            manager.retrying_heals.lock().await.insert(
+                old_id.clone(),
+                RetryingHeal {
+                    request: old_request,
+                    error: "retry pending".to_owned(),
+                    cancel_token: CancellationToken::new(),
+                },
+            );
+        }
+        other => panic!("unknown fixture state: {other}"),
+    }
+
+    let new_request = incarnation_bound_mrf_fallback(new_incarnation);
+    let new_id = new_request.id.clone();
+    assert_ne!(new_id, old_id, "the fallback must use a new request ID");
+    let receipt = manager
+        .submit_heal_request_with_receipt(new_request)
+        .await
+        .expect("new incarnation admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
+    );
+    assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
+    let queue = manager.heal_queue.lock().await;
+    assert!(
+        queue
+            .requests()
+            .any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
+    );
+    if existing_state == "queued" {
+        assert!(
+            queue
+                .requests()
+                .any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
+        );
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("active").await;
+}
+
+#[tokio::test]
+async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
+    assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
+}
+
+fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
+    let mut request = incarnation_bound_mrf_fallback(incarnation);
+    if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
+        *id = version_id.map(str::to_owned);
+    } else {
+        unreachable!("fallback fixture must be an object request");
+    }
+    request
+}
+
+#[tokio::test]
+async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
+    for version in [Uuid::nil(), Uuid::new_v4()] {
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+        let incarnation = Uuid::new_v4();
+        let version_id = version.to_string();
+        let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
+        let original_id = original.id.clone();
+        let accepted = manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first queued fallback");
+        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
+        let merged = manager
+            .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
+            .await
+            .expect("duplicate queued fallback");
+        assert_eq!(merged.result, HealAdmissionResult::Merged);
+        assert_eq!(merged.task_id, original_id);
+        assert_eq!(manager.heal_queue.lock().await.len(), 1);
+    }
+}
+
+#[tokio::test]
+async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let original = incarnation_bound_mrf_fallback(incarnation);
+    let original_id = original.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("first fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let same = manager
+        .submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
+        .await
+        .expect("same-incarnation fallback");
+    assert_eq!(same.result, HealAdmissionResult::Merged);
+    assert_eq!(same.task_id, original_id);
+
+    let mut versioned = incarnation_bound_mrf_fallback(incarnation);
+    versioned.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "object".to_owned(),
+        version_id: Some(Uuid::new_v4().to_string()),
+    };
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(versioned)
+            .await
+            .expect("different version")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
+    other_scope.options.pool_index = Some(1);
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(other_scope)
+            .await
+            .expect("different pool")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    same_id_other_incarnation.id = original_id;
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(same_id_other_incarnation)
+            .await
+            .expect("same-ID conflict must be decided")
+            .result,
+        HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
+    );
+}
+
+#[test]
+fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
+    let incarnation = Uuid::new_v4();
+    let bound = incarnation_bound_mrf_fallback(incarnation);
+    let mut unbound = bound.clone();
+    unbound.expected_mrf_bucket_incarnation_id = None;
+    let mut non_mrf = bound.clone();
+    non_mrf.source = HealRequestSource::ReadRepair;
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key(&non_mrf),
+        PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
+        "an unrelated source retains its previous scope key even if the MRF-only field is set"
+    );
+    assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
+
+    let mut left = bound.clone();
+    left.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part:a".to_owned(),
+        version_id: Some("b".to_owned()),
+    };
+    let mut right = bound.clone();
+    right.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: "part".to_owned(),
+        version_id: Some("a:b".to_owned()),
+    };
+    assert_eq!(
+        PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
+        PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
+        "the legacy delimiter-only key demonstrates the collision"
+    );
+    assert_ne!(
+        PriorityHealQueue::make_dedup_key(&left),
+        PriorityHealQueue::make_dedup_key(&right),
+        "length-prefixed object and version components must not alias"
+    );
+    let mut forged = unbound;
+    forged.heal_type = HealType::Object {
+        bucket: "mrf-recreated-bucket".to_owned(),
+        object: PriorityHealQueue::make_dedup_key(&bound),
+        version_id: None,
+    };
+    assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
+}
+
+#[test]
+fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
+    let incarnation = Uuid::new_v4();
+    for decode in [false, true] {
+        let make_request = |object: &str, version_id: Option<&str>| {
+            let mut request = incarnation_bound_mrf_fallback(incarnation);
+            let bucket = "mrf-recreated-bucket".to_owned();
+            let object = object.to_owned();
+            let version_id = version_id.map(str::to_owned);
+            request.heal_type = if decode {
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            } else {
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                }
+            };
+            request
+        };
+        let absent = make_request("object", None);
+        let empty = make_request("object", Some(""));
+        let forged_object = make_request("object4:some0:", None);
+        let forged_version = make_request("object", Some("4:none"));
+        let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
+        for (index, key) in keys.iter().enumerate() {
+            assert!(keys.iter().skip(index + 1).all(|other| key != other));
+        }
+
+        let mut queue = PriorityHealQueue::new();
+        assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
+        assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
+        assert_eq!(queue.len(), 2);
+
+        for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
+            let mut legacy_absent = absent.clone();
+            let mut legacy_empty = empty.clone();
+            legacy_absent.source = source;
+            legacy_empty.source = source;
+            if source == HealRequestSource::Mrf {
+                legacy_absent.expected_mrf_bucket_incarnation_id = None;
+                legacy_empty.expected_mrf_bucket_incarnation_id = None;
+            }
+            assert_eq!(
+                PriorityHealQueue::make_dedup_key(&legacy_absent),
+                PriorityHealQueue::make_dedup_key(&legacy_empty),
+                "unbound and non-MRF work retain the legacy empty-version key"
+            );
+        }
+    }
+}
+
+#[test]
+fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
+    let mut queue = PriorityHealQueue::new();
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let old_id = old.id.clone();
+    let old_key = PriorityHealQueue::make_dedup_key(&old);
+    let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    let new_id = new.id.clone();
+    let new_key = PriorityHealQueue::make_dedup_key(&new);
+    let mut forced_old = old.clone();
+    forced_old.id = Uuid::new_v4().to_string();
+    forced_old.force_start = true;
+    let forced_id = forced_old.id.clone();
+
+    assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    queue
+        .remove_request_id(&old_id)
+        .expect("remove the old incarnation representative");
+    assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
+    assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
+    assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
+    assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
+    assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
+}
+
+#[tokio::test]
+async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let incarnation = Uuid::new_v4();
+    let request = incarnation_bound_mrf_fallback(incarnation);
+    let request_id = request.id.clone();
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from("object"),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let receipt = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(request, target)
+        .await
+        .expect("durable admission must be decided");
+    assert_eq!(
+        receipt.result,
+        HealAdmissionResult::Accepted,
+        "a durable new incarnation needs its own owner"
+    );
+    assert_eq!(receipt.task_id, request_id);
+    assert!(
+        manager
+            .heal_queue
+            .lock()
+            .await
+            .requests()
+            .any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
+    );
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
+#[tokio::test]
+async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
+    use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
+
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let incarnation = Uuid::new_v4();
+    let object = format!("same-generation-{}", Uuid::new_v4());
+    let make_request = || {
+        let mut request = incarnation_bound_mrf_fallback(incarnation);
+        request.heal_type = HealType::Object {
+            bucket: "mrf-recreated-bucket".to_owned(),
+            object: object.clone(),
+            version_id: None,
+        };
+        request
+    };
+    let old = make_request();
+    let old_id = old.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request_with_receipt(old)
+            .await
+            .expect("old fallback")
+            .result,
+        HealAdmissionResult::Accepted
+    );
+    let mut intent = MrfIntent {
+        bucket: Arc::from("mrf-recreated-bucket"),
+        object: Arc::from(object.as_str()),
+        version_id: None,
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(MrfScope {
+            pool_index: 0,
+            set_index: 0,
+        }),
+        lease: None,
+        enqueued_at_ms: 1,
+        attempts: 0,
+    };
+    assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
+    let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
+    let target = MrfRepairNoticeTarget {
+        bucket: Arc::clone(&intent.bucket),
+        object: Arc::clone(&intent.object),
+        version_id: intent.version_id,
+        kind: intent.kind,
+        scope: intent.scope,
+        delete_marker_purge: None,
+        lease: intent.lease,
+        durable_anchor: Some(anchor.clone()),
+    };
+    let merged = manager
+        .submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
+        .await
+        .expect("durable same-incarnation admission");
+    assert_eq!(merged.result, HealAdmissionResult::Merged);
+    assert_eq!(merged.task_id, old_id);
+    assert!(
+        manager
+            .try_in_flight_durable_mrf_anchors()
+            .expect("stable owner snapshot")
+            .contains(&anchor)
+    );
+}
+
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
         task_id: "fixture-token".to_string(),

crates/heal/src/heal/pacing.rs

@@ -7,8 +7,14 @@ use rustfs_concurrency::{
     workload::{ForegroundPressure, foreground_pressure},
 };
 use rustfs_heal_contracts::heal_channel::HealRequestSource;
-use std::{sync::Arc, time::Duration};
+use std::{
-use tokio::{sync::Mutex, time::Instant};
+    sync::{
+        Arc,
+        atomic::{AtomicU32, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{sync::Mutex as AsyncMutex, time::Instant};
 use tokio_util::sync::CancellationToken;
 
 #[derive(Default)]
@@ -22,17 +28,19 @@ pub(crate) enum PacingDecision {
     Wait(Option<ForegroundPressure>),
 }
 
-/// Cooperative pacing for one heal execution, not a storage admission permit.
+/// Foreground watermarks for one heal execution. Constructed once per request;
-pub(crate) struct MainlinePacer {
+/// disabled when watermarks are zero or no workload provider is configured.
+#[derive(Clone)]
+pub(crate) struct ForegroundPacing {
     provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
     read_high: usize,
     write_high: usize,
     pause: Duration,
     source: HealRequestSource,
-    state: Mutex<PacingState>,
+    state: Arc<AsyncMutex<PacingState>>,
 }
 
-impl MainlinePacer {
+impl ForegroundPacing {
     pub(crate) fn new(
         provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
         read_high: usize,
@@ -49,7 +57,7 @@ impl MainlinePacer {
             write_high: write_high.min(100),
             pause: pause.min(Duration::from_secs(1)),
             source,
-            state: Mutex::new(PacingState::default()),
+            state: Arc::new(AsyncMutex::new(PacingState::default())),
         })
     }
 
@@ -125,6 +133,142 @@ impl MainlinePacer {
     }
 }
 
+/// Single owner of "how hard may this heal pass push storage?".
+///
+/// One instance spans the whole erasure-set scan (all buckets and pages):
+/// - foreground admission pressure (`ForegroundPacing`)
+/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
+///
+/// Call `observe_object_outcome` exactly once per finished object so a single
+/// SlowDown cannot double-reduce concurrency.
+pub(crate) struct HealPageRate {
+    base_concurrency: usize,
+    /// 0 = full base concurrency; each SlowDown increases the reduction.
+    slow_down_step: AtomicU32,
+    slow_down_until: std::sync::Mutex<Option<Instant>>,
+    foreground: Option<ForegroundPacing>,
+}
+
+impl HealPageRate {
+    /// Always-on controller. Foreground yield is optional; SlowDown control is not.
+    pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
+        Self {
+            base_concurrency: base_concurrency.max(1),
+            slow_down_step: AtomicU32::new(0),
+            slow_down_until: std::sync::Mutex::new(None),
+            foreground,
+        }
+    }
+
+    /// Concurrency after applying the current SlowDown reduction. Always ≥ 1
+    /// so maintenance can still make progress under sustained rate pressure.
+    pub(crate) fn effective_concurrency(&self) -> usize {
+        let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
+        (self.base_concurrency >> step).max(1)
+    }
+
+    /// Fold one finished object into the rate state. Call once per outcome.
+    /// `saw_slow_down` is true when any attempt of this object hit SlowDown,
+    /// including attempts that later succeeded on retry.
+    pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
+        if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
+            self.observe_slow_down();
+            return;
+        }
+        if result.is_ok() {
+            self.observe_success();
+        }
+    }
+
+    /// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
+    fn observe_slow_down(&self) {
+        // fetch_add returns the previous step; +1 so the first SlowDown uses
+        // exponent 1 (2s), not exponent 0 (1s).
+        let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
+        let delay = Self::backoff(step);
+        let ready = Instant::now() + delay;
+        if let Ok(mut until) = self.slow_down_until.lock()
+            && until.is_none_or(|current| current < ready)
+        {
+            *until = Some(ready);
+        }
+    }
+
+    /// Restore one concurrency step only after the SlowDown cool-down expires.
+    /// A same-page success must not undo the reduction while the pass is still
+    /// in the rate-limited window (the page semaphore is fixed until the next page).
+    fn observe_success(&self) {
+        let cooling = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .is_some_and(|ready| ready > Instant::now());
+        if cooling {
+            return;
+        }
+        self.slow_down_step
+            .try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
+            .ok();
+    }
+
+    /// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
+    pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
+        self.wait_slow_down(cancel).await?;
+        if let Some(foreground) = &self.foreground {
+            foreground.wait(cancel).await?;
+        }
+        Ok(())
+    }
+
+    /// Foreground admission decision while the caller owns page capacity.
+    pub(crate) fn admission_decision(&self) -> PacingDecision {
+        match &self.foreground {
+            Some(foreground) => foreground.admission_decision(),
+            None => PacingDecision::Ready,
+        }
+    }
+
+    /// Foreground wait after a rejected admission. Returns whether a pause was paid.
+    pub(crate) async fn wait_after_admission(
+        &self,
+        cancel: &CancellationToken,
+        observed: Option<ForegroundPressure>,
+    ) -> Result<bool> {
+        match &self.foreground {
+            Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
+            None => Ok(false),
+        }
+    }
+
+    async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
+        let delay = self
+            .slow_down_until
+            .lock()
+            .ok()
+            .and_then(|until| *until)
+            .map(|ready| ready.saturating_duration_since(Instant::now()));
+        let Some(delay) = delay else {
+            return Ok(());
+        };
+        if delay.is_zero() {
+            return Ok(());
+        }
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            _ = tokio::time::sleep(delay) => Ok(()),
+        }
+    }
+
+    fn backoff(step: u32) -> Duration {
+        // step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
+        // rate with exponential backoff, and heal must do the same to itself.
+        let exponent = step.clamp(1, 5);
+        Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
+    }
+}
+
 #[cfg(test)]
 pub(crate) struct TestPressure {
     pub(crate) active: std::sync::atomic::AtomicUsize,
@@ -164,12 +308,87 @@ mod tests {
     use rustfs_concurrency::WorkloadClass;
     use std::sync::atomic::Ordering;
 
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_first_slow_down_waits_two_seconds() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("cool-down wait");
+        assert_eq!(Instant::now() - before, Duration::from_secs(2));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_reduces_concurrency_and_backs_off() {
+        let rate = HealPageRate::new(8, None);
+        assert_eq!(rate.effective_concurrency(), 8);
+        rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        assert_eq!(rate.effective_concurrency(), 2);
+        // Floor is one so maintenance never stops entirely.
+        for _ in 0..8 {
+            rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
+        }
+        assert_eq!(rate.effective_concurrency(), 1);
+
+        let cancel = CancellationToken::new();
+        assert!(rate.wait_before_page(&cancel).await.is_ok());
+
+        // After cool-down, successes restore concurrency one step at a time.
+        for _ in 0..12 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert!(rate.effective_concurrency() > 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
+        let rate = HealPageRate::new(8, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        assert_eq!(rate.effective_concurrency(), 4);
+        // Same-page successes while cool-down is armed must not restore fan-out.
+        for _ in 0..4 {
+            rate.observe_object_outcome(&Ok(true), false);
+        }
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
+        let rate = HealPageRate::new(8, None);
+        // Replacement retry: SlowDown on attempt 1, success on attempt 2.
+        rate.observe_object_outcome(&Ok(true), true);
+        assert_eq!(rate.effective_concurrency(), 4);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_wait_is_cancellable() {
+        let rate = HealPageRate::new(4, None);
+        rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
+        let cancel = CancellationToken::new();
+        cancel.cancel();
+        assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
+        let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
+            .expect("foreground pacer");
+        let rate = HealPageRate::new(4, Some(foreground));
+        let cancel = CancellationToken::new();
+        let before = Instant::now();
+        rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
+        assert_eq!(Instant::now() - before, Duration::from_millis(250));
+    }
+
     #[tokio::test(start_paused = true)]
     async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
         for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
             let provider = Arc::new(TestPressure::new(class, 0));
             let pause = Duration::from_millis(250);
-            let pacer = MainlinePacer::new(
+            let pacer = ForegroundPacing::new(
                 provider.clone(),
                 if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
                 if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
@@ -197,9 +416,9 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
-        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
+        assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
-        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
+        assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
-        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
+        let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
         let before = Instant::now();
         pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
         assert_eq!(Instant::now() - before, Duration::from_secs(1));
@@ -209,7 +428,7 @@ mod tests {
     async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
         let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
         let paced = Arc::new(
-            MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
+            ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
         );
         let cancel_first = CancellationToken::new();
         let first = tokio::spawn({
@@ -228,7 +447,7 @@ mod tests {
         cancel_second.cancel();
         assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
         provider.active.store(0, Ordering::SeqCst);
-        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
+        let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
             .expect("independent task");
         let before = Instant::now();
         other_task

crates/heal/src/heal/resume.rs

@@ -35,7 +35,9 @@ mod replacement;
 mod utils;
 
 pub(crate) use checkpoint::AdminErasureCheckpoint;
-pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
+pub use checkpoint::{
+    CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
+};
 pub(crate) use gc::ResumeGc;
 pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
 pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};

crates/heal/src/heal/resume/checkpoint.rs

@@ -152,6 +152,42 @@ impl AdminErasureCheckpoint {
     }
 }
 
+/// Lightweight progress view for resume decisions. Omits the per-page object
+/// ledgers so callers do not clone a million-object checkpoint just to read
+/// counters and positions.
+#[derive(Debug, Clone, Copy)]
+pub struct CheckpointProgressSnapshot {
+    pub current_bucket_index: usize,
+    pub current_object_index: usize,
+    pub successful_objects: u64,
+    pub failed_object_count: u64,
+    pub skipped_object_count: u64,
+    pub skipped_new_versions: u64,
+    pub skipped_ilm_expired: u64,
+    pub processed_bytes: u64,
+    pub total_objects: u64,
+    pub total_bytes: u64,
+    pub baseline_generation: Option<u64>,
+    pub baseline_known: bool,
+    pub counter_unknown: bool,
+}
+
+impl CheckpointProgressSnapshot {
+    pub fn has_progress(&self) -> bool {
+        self.baseline_known
+            || self.successful_objects > 0
+            || self.failed_object_count > 0
+            || self.skipped_object_count > 0
+            || self.skipped_new_versions > 0
+            || self.skipped_ilm_expired > 0
+            || self.processed_bytes > 0
+            || self.total_objects > 0
+            || self.total_bytes > 0
+            || self.baseline_generation.is_some()
+            || self.counter_unknown
+    }
+}
+
 /// resume checkpoint
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ResumeCheckpoint {
@@ -715,6 +751,75 @@ impl CheckpointManager {
         self.checkpoint.read().await.clone()
     }
 
+    /// True when a page object is already settled in the current ledger.
+    /// Membership is checked under the lock without cloning the checkpoint,
+    /// which used to copy every page's dedup sets on the small-object path.
+    pub async fn is_object_settled(&self, key: &str) -> bool {
+        let checkpoint = self.checkpoint.read().await;
+        checkpoint.processed_objects.contains(key)
+            || checkpoint.failed_objects.contains(key)
+            || checkpoint.skipped_objects.contains(key)
+    }
+
+    /// True when the administrator checkpoint already finished.
+    pub async fn admin_completed(&self) -> bool {
+        self.checkpoint
+            .read()
+            .await
+            .admin
+            .as_ref()
+            .is_some_and(|admin| admin.completed)
+    }
+
+    /// Current resume position without cloning the object ledgers.
+    pub async fn resume_position(&self) -> (usize, usize) {
+        let checkpoint = self.checkpoint.read().await;
+        (checkpoint.current_bucket_index, checkpoint.current_object_index)
+    }
+
+    /// Dangling-delete grace summary without cloning the object ledgers.
+    pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
+        let checkpoint = self.checkpoint.read().await;
+        (
+            checkpoint.dangling_delete_grace_objects,
+            checkpoint.dangling_delete_retry_not_before,
+            checkpoint.counter_unknown,
+        )
+    }
+
+    /// Aggregate progress counters without cloning the page dedup sets.
+    pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
+        let checkpoint = self.checkpoint.read().await;
+        CheckpointProgressSnapshot {
+            current_bucket_index: checkpoint.current_bucket_index,
+            current_object_index: checkpoint.current_object_index,
+            successful_objects: checkpoint.successful_objects,
+            failed_object_count: checkpoint.failed_object_count,
+            skipped_object_count: checkpoint.skipped_object_count,
+            skipped_new_versions: checkpoint.skipped_new_versions,
+            skipped_ilm_expired: checkpoint.skipped_ilm_expired,
+            processed_bytes: checkpoint.processed_bytes,
+            total_objects: checkpoint.total_objects,
+            total_bytes: checkpoint.total_bytes,
+            baseline_generation: checkpoint.baseline_generation,
+            baseline_known: checkpoint.baseline_known,
+            counter_unknown: checkpoint.counter_unknown,
+        }
+    }
+
+    /// Finish the administrator outcome in place and persist it.
+    pub async fn complete_admin(&self) -> Result<()> {
+        let mut checkpoint = self.checkpoint.write().await;
+        let admin = checkpoint
+            .admin
+            .as_mut()
+            .ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
+        admin.completed = true;
+        admin.outcome.finish(None);
+        drop(checkpoint);
+        self.save_checkpoint().await
+    }
+
     /// update position
     pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
         let mut checkpoint = self.checkpoint.write().await;

crates/heal/src/heal/task.rs

@@ -485,7 +485,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
-    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+    mainline_pacer: Option<super::pacing::ForegroundPacing>,
 }
 
 impl HealTask {
@@ -586,7 +586,7 @@ impl HealTask {
         task
     }
 
-    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
         self.mainline_pacer = pacer;
         self
     }

crates/heal/src/heal/task/heal_object.rs

@@ -480,8 +480,9 @@ impl HealTask {
                 message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
             })?;
         if current_bucket_incarnation_id != bucket_incarnation_id {
-            return Err(Error::TaskExecutionFailed {
+            return Err(Error::StaleBucketIncarnation {
-                message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
+                bucket: bucket.to_owned(),
+                expected: Some(bucket_incarnation_id),
             });
         }
         expected.bucket_incarnation_id = Some(bucket_incarnation_id);

crates/heal/src/heal/task/tests.rs

@@ -3375,10 +3375,67 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
 
     let result = task.heal_object("bucket", "object", None).await;
 
-    assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
+    assert!(matches!(
+        result,
+        Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
+    ));
     assert!(storage.heal_object_calls.lock().unwrap().is_empty());
 }
 
+#[tokio::test]
+async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
+    for object in ["timed out", "network error", "ordinary"] {
+        let original_incarnation = Uuid::new_v4();
+        let recreated_incarnation = Uuid::new_v4();
+        let storage = Arc::new(MockStorage {
+            bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
+            ..Default::default()
+        });
+        let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
+        request.source = HealRequestSource::Mrf;
+        request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
+        request.options.pool_index = Some(0);
+        request.options.set_index = Some(1);
+        let task = HealTask::from_request(request, storage.clone());
+
+        let error = task
+            .execute()
+            .await
+            .expect_err("a recreated bucket must reject the old MRF responsibility");
+        assert!(
+            !error.is_recoverable_heal(),
+            "user-controlled object names must not make a stale incarnation retryable: {object}"
+        );
+        assert!(
+            matches!(
+                &error,
+                Error::StaleBucketIncarnation { bucket, expected }
+                    if bucket == "bucket" && *expected == Some(original_incarnation)
+            ),
+            "the failure must retain the original bucket incarnation: {error:?}"
+        );
+        assert!(
+            matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
+            "stale responsibility must be classified independently of the object name"
+        );
+        assert!(
+            storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
+            "an old responsibility must not reach storage repair in the recreated bucket"
+        );
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
+        assert!(
+            outcome.objects.iter().all(|item| !matches!(
+                &item.disposition,
+                HealObjectDisposition::Repaired
+                    | HealObjectDisposition::VerifiedHealthy
+                    | HealObjectDisposition::AuthoritativelyAbsent
+            )),
+            "the stale task must not produce a positive repair proof"
+        );
+    }
+}
+
 #[tokio::test(start_paused = true)]
 async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
     let storage = Arc::new(MockStorage {

crates/iam/src/oidc/provider.rs

@@ -16,9 +16,10 @@ use super::{
     transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
 };
 use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
-use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
+use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
 use openidconnect::{
-    AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
+    AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
+    ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
 };
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -212,6 +213,35 @@ impl DiscoveredProviderMetadata {
     }
 }
 
+pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
+    token: &'a CoreIdToken,
+    verifier: &CoreIdTokenVerifier<'_>,
+    nonce_verifier: N,
+) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
+    let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
+        CoreJwsSigningAlgorithm::HmacSha256 => 32,
+        CoreJwsSigningAlgorithm::HmacSha384 => 48,
+        CoreJwsSigningAlgorithm::HmacSha512 => 64,
+        _ => return token.claims(verifier, nonce_verifier),
+    };
+    // RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
+    // openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
+    let invalid_length = || {
+        ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
+            "invalid HMAC signature length".to_owned(),
+        ))
+    };
+    let compact = token.to_string();
+    let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
+    let signature = base64_simd::URL_SAFE_NO_PAD
+        .decode_to_vec(signature)
+        .map_err(|_| invalid_length())?;
+    if signature.len() != expected_length {
+        return Err(invalid_length());
+    }
+    token.claims(verifier, nonce_verifier)
+}
+
 // This adapter is used only for discovery/JWKS fetches, never token exchange.
 struct JwksAcceptClient<'a> {
     inner: &'a ReqwestHttpClient,

crates/iam/src/oidc/provider_tests.rs

@@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
     let now = time::OffsetDateTime::now_utc().unix_timestamp();
     let payload = serde_json::json!({
         "iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
-        "iat": now, "exp": now + 300,
+        "iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
     });
 
     for (algorithm, name, signature_length) in [
@@ -275,22 +275,28 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
                 .expect("parse configured provider metadata");
             let verifier = metadata.verifier(&config);
             assert!(
-                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "valid {name} token must pass for workload={workload}"
             );
+            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
+            assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
 
             for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
                 let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
                 let malformed: CoreIdToken = format!("{signing_input}.{signature}")
                     .parse()
-                    .expect("invalid signature length must still reach cryptographic verification");
+                    .expect("invalid signature length must remain parseable before verification");
                 assert!(
-                    malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
+                    verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
                     "{name} signature length {length} must return an error for workload={workload}"
                 );
+                assert!(
+                    verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
+                    "{name} signature length {length} must return an error with a nonce for workload={workload}"
+                );
             }
             assert!(
-                valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
+                verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
                 "invalid inputs must not prevent later valid {name} verification"
             );
         }

crates/iam/src/oidc/runtime.rs

@@ -16,7 +16,7 @@ use super::{
         OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
         SourcedOidcProviderConfig, load_oidc_config_snapshot,
     },
-    provider::{ProviderRuntime, discover_provider_for_recovery},
+    provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
     state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
     transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
 };
@@ -535,7 +535,7 @@ impl OidcSys {
         let verifier = client
             .id_token_verifier()
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
+        let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
         if let Err(e) = verified {
             let verification_error = e.to_string();
             warn!(
@@ -583,8 +583,7 @@ impl OidcSys {
             let verifier = client
                 .id_token_verifier()
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            id_token
+            verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
-                .claims(&verifier, &Nonce::new(session.nonce.clone()))
                 .map_err(|retry_err| {
                     error!(
                         event = EVENT_OIDC_DIAGNOSTICS,
@@ -767,7 +766,7 @@ impl OidcSys {
             .metadata
             .verifier(&config)
             .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-        if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
+        if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
             state = self
                 .provider_runtime
                 .refresh_provider_state(&provider_id, &config)
@@ -780,8 +779,7 @@ impl OidcSys {
                 .metadata
                 .verifier(&config)
                 .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
-            id_token
+            verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
-                .claims(&verifier, |_: Option<&Nonce>| Ok(()))
                 .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
         }
 

crates/io-metrics/src/internode_metrics.rs

@@ -1411,6 +1411,28 @@ pub fn record_peer_reachable(addr: &str) {
     publish_offline_gauge(&peers);
 }
 
+/// Record a successful interaction with a cluster peer, but only when the peer is currently
+/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
+/// and lock-path successes that do not go through a fresh dial.
+///
+/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
+/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
+/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
+pub fn record_peer_reachable_if_offline(addr: &str) {
+    let key = normalize_peer_key(addr);
+    // Fast path: read lock to check whether the peer is currently offline.
+    {
+        let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
+        match peers.get(key) {
+            None => return,
+            Some(entry) if entry.online => return,
+            Some(_) => {} // offline: fall through to the write path
+        }
+    }
+    // Slow path: the peer is offline, so flip it back online and republish the gauge.
+    record_peer_reachable(addr);
+}
+
 /// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
 /// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
 pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
@@ -2188,6 +2210,34 @@ mod tests {
         assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
     }
 
+    #[test]
+    fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
+        let addr = "http://cluster-peer-recover-if-offline-test:9000";
+
+        // Mark the peer offline.
+        record_peer_unreachable(addr, 1);
+        assert!(cluster_peer_is_offline(addr));
+
+        // A successful cached-channel RPC (no fresh dial) clears the offline state.
+        record_peer_reachable_if_offline(addr);
+        assert!(!cluster_peer_is_offline(addr));
+        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
+    }
+
+    #[test]
+    fn record_peer_reachable_if_offline_no_op_for_online_peer() {
+        let addr = "http://cluster-peer-recover-noop-test:9000";
+
+        record_peer_reachable(addr);
+        // Should not panic or change state for an already-online peer.
+        record_peer_reachable_if_offline(addr);
+        assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
+
+        // Should also be a no-op for unknown peers.
+        record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
+        assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
+    }
+
     #[test]
     fn cluster_peer_should_bypass_is_self_healing() {
         let addr = "http://cluster-peer-bypass-selfheal-test:9000";

crates/protocols/Cargo.toml

@@ -30,6 +30,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-tls-runtime?/fips"]
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/protocols/src/ftps/server.rs

@@ -110,6 +110,9 @@ where
             ftps_required = self.config.ftps_required,
             "FTPS server starting"
         );
+        if self.config.tls_enabled && self.config.cert_dir.is_some() {
+            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
+        }
         let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
 
         let storage_clone = self.storage.clone();
@@ -191,8 +194,6 @@ where
                 );
 
                 // Build ServerConfig with SNI support
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
                 let server_config = rustls::ServerConfig::builder()
                     .with_no_client_auth()
                     .with_cert_resolver(resolver);

crates/protocols/src/lib.rs

@@ -43,3 +43,70 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
 
 #[cfg(feature = "sftp")]
 pub use sftp::{SftpConfig, SftpInitError, SftpServer};
+
+#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
+mod fips_tests {
+    use crate::common::dummy_storage::DummyBackend;
+    use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
+
+    #[test]
+    fn protocols_reject_incompatible_host_provider() {
+        const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
+        if std::env::var_os(CHILD).is_some() {
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            provider.install_default().expect("install incompatible host provider");
+            let cert_dir = tempfile::tempdir().expect("certificate directory");
+            let cert_dir = cert_dir.path().to_string_lossy().into_owned();
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("test runtime");
+            runtime.block_on(async {
+                let ftps_config = FtpsConfig {
+                    cert_dir: Some(cert_dir.clone()),
+                    ..Default::default()
+                };
+                let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
+                    .await
+                    .expect("FTPS config");
+                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
+                let error = ftps
+                    .start(receiver)
+                    .await
+                    .expect_err("FTPS must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+
+                let webdav_config = WebDavConfig {
+                    bind_addr: ([127, 0, 0, 1], 0).into(),
+                    cert_dir: Some(cert_dir),
+                    ..Default::default()
+                };
+                let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
+                    .await
+                    .expect("WebDAV config");
+                let (_sender, receiver) = tokio::sync::broadcast::channel(1);
+                let error = webdav
+                    .start(receiver)
+                    .await
+                    .expect_err("WebDAV must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            });
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("protocols_reject_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .env("NO_PROXY", "localhost,127.0.0.1,::1")
+            .env("no_proxy", "localhost,127.0.0.1,::1")
+            .output()
+            .expect("start isolated protocol TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+}

crates/protocols/src/webdav/server.rs

@@ -118,6 +118,10 @@ where
             "WebDAV server starting"
         );
 
+        if self.config.tls_enabled && self.config.cert_dir.is_some() {
+            rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
+        }
+
         let listener = TcpListener::bind(self.config.bind_addr).await?;
         self.serve(listener, shutdown_rx).await
     }
@@ -155,8 +159,6 @@ where
                     reload_shutdown_rx.clone(),
                 );
 
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-
                 let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
 
                 Some(TlsAcceptor::from(Arc::new(server_config)))

crates/s3-client/Cargo.toml

@@ -25,6 +25,10 @@ keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
 categories = ["web-programming", "development-tools", "network-programming"]
 documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
 
+[features]
+default = []
+fips = ["rustfs-tls-runtime/fips"]
+
 [dependencies]
 rustfs-checksums.workspace = true
 rustfs-config.workspace = true

crates/s3-client/src/transition_api.rs

@@ -81,7 +81,7 @@ use time::Duration;
 use time::OffsetDateTime;
 use tokio::io::BufReader;
 use tokio::io::{AsyncRead, AsyncReadExt};
-use tracing::{debug, error, trace, warn};
+use tracing::{error, trace, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
@@ -448,16 +448,7 @@ impl TransitionClient {
         tier_type: &str,
         timeouts: TransitionClientTimeouts,
     ) -> Result<TransitionClient, std::io::Error> {
-        if rustls::crypto::CryptoProvider::get_default().is_none() {
+        rustfs_tls_runtime::ensure_crypto_provider().map_err(std::io::Error::other)?;
-            // No default provider is set yet; try to install aws-lc-rs.
-            // `install_default` can only fail if another thread races us and installs a provider
-            // between our check and this call, which is still safe to ignore.
-            if rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() {
-                debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install");
-            }
-        } else {
-            debug!("rustls crypto provider already installed, skipping aws-lc-rs install");
-        }
 
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
         let timeouts = timeouts.validate()?;
@@ -1862,21 +1853,36 @@ mod tests {
         assert!(outcome.is_ok(), "TLS config creation should not panic");
     }
 
-    /// Installing the rustls crypto provider when one is already set must not panic or return
+    #[cfg(feature = "fips")]
-    /// an error that surfaces to callers (the race-safe `get_default` check guards the install).
     #[test]
-    fn provider_install_is_idempotent() {
+    fn fips_transition_rejects_incompatible_host_provider() {
-        // Install once (may already be set by another test in this binary — that's fine).
+        const CHILD: &str = "RUSTFS_TEST_TRANSITION_FIPS_PROVIDER";
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+        if std::env::var_os(CHILD).is_some() {
-
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
-        // A second install attempt on an already-set provider must not panic.
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
-        let outcome = std::panic::catch_unwind(|| {
+            provider.install_default().expect("install incompatible host provider");
-            if rustls::crypto::CryptoProvider::get_default().is_none() {
+            let runtime = tokio::runtime::Builder::new_current_thread()
-                let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+                .enable_all()
-            }
+                .build()
-            // If a default is already present, the branch above is simply skipped.
+                .expect("test runtime");
-        });
+            let error = runtime
-        assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
+                .block_on(super::TransitionClient::new("127.0.0.1:0", super::Options::default(), ""))
+                .expect_err("transition client must reject the incompatible provider");
+            assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("fips_transition_rejects_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated transition TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
     }
 
     #[test]

crates/s3select-api/src/object_store.rs

@@ -640,12 +640,12 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
         GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
         GetRange::Offset(start) => HTTPRangeSpec {
             is_suffix_length: false,
-            start: *start as i64,
+            start: i64::try_from(*start).unwrap_or(i64::MAX),
             end: -1,
         },
         GetRange::Suffix(length) => HTTPRangeSpec {
             is_suffix_length: true,
-            start: *length as i64,
+            start: i64::try_from(*length).unwrap_or(i64::MAX),
             end: -1,
         },
     }
@@ -654,15 +654,15 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
 fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
     HTTPRangeSpec {
         is_suffix_length: false,
-        start: range.start as i64,
+        start: i64::try_from(range.start).unwrap_or(i64::MAX),
-        end: range.end.saturating_sub(1) as i64,
+        end: i64::try_from(range.end.saturating_sub(1)).unwrap_or(i64::MAX),
     }
 }
 
 fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
     HTTPRangeSpec {
         is_suffix_length: false,
-        start: start as i64,
+        start: i64::try_from(start).unwrap_or(i64::MAX),
         end: -1,
     }
 }
@@ -2476,6 +2476,48 @@ mod test {
         assert_eq!(range.end, 19);
     }
 
+    #[test]
+    fn test_get_range_conversion_for_suffix_ranges() {
+        // Normal suffix: last 100 bytes.
+        let range = http_range_spec_from_get_range(&GetRange::Suffix(100));
+        assert!(range.is_suffix_length);
+        assert_eq!(range.start, 100);
+        assert_eq!(range.end, -1);
+
+        // Overflowing suffix: `u64::MAX` previously wrapped to `-1` via `as i64`
+        // and was misread as "last 1 byte". It now clamps to `i64::MAX`, which
+        // `get_length` treats as "the whole object".
+        let range = http_range_spec_from_get_range(&GetRange::Suffix(u64::MAX));
+        assert!(range.is_suffix_length);
+        assert_eq!(range.start, i64::MAX);
+        assert_eq!(range.end, -1);
+    }
+
+    #[test]
+    fn test_get_range_conversion_for_offset_overflow() {
+        // Normal offset.
+        let range = http_range_spec_from_get_range(&GetRange::Offset(42));
+        assert!(!range.is_suffix_length);
+        assert_eq!(range.start, 42);
+        assert_eq!(range.end, -1);
+
+        // Overflowing offset clamps to `i64::MAX`, which is then rejected as
+        // out-of-range (416) by `get_offset_length`.
+        let range = http_range_spec_from_get_range(&GetRange::Offset(u64::MAX));
+        assert!(!range.is_suffix_length);
+        assert_eq!(range.start, i64::MAX);
+        assert_eq!(range.end, -1);
+    }
+
+    #[test]
+    fn test_get_range_conversion_for_bounded_overflow() {
+        // Overflowing bounded range clamps start/end to `i64::MAX`.
+        let range = http_range_spec_from_get_range(&GetRange::Bounded(u64::MAX - 10..u64::MAX));
+        assert!(!range.is_suffix_length);
+        assert_eq!(range.start, i64::MAX);
+        assert_eq!(range.end, i64::MAX);
+    }
+
     #[tokio::test]
     async fn test_scan_range_output_can_convert_field_delimiter() {
         let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);

crates/targets/Cargo.toml

@@ -13,6 +13,7 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
 
 [features]
 default = []
+fips = ["rustfs-tls-runtime/fips"]
 # Exposes the builder-style `testkit::MockTarget` to downstream test suites. Never enable this
 # from a production `[dependencies]` entry; in-crate unit tests get the module via cfg(test).
 test-support = []

crates/targets/src/check.rs

@@ -162,7 +162,7 @@ pub async fn check_mysql_server_available(args: &crate::target::mysql::MySqlArgs
         .db_name(Some(dsn.database.clone()));
 
     if dsn.tls {
-        ensure_rustls_provider_installed();
+        ensure_rustls_provider_installed()?;
         let mut ssl_opts = SslOpts::default();
         if !args.tls_ca.is_empty() {
             ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);

crates/targets/src/target/amqp.rs

@@ -293,6 +293,10 @@ fn map_lapin_error(err: lapin::Error, context: &str) -> TargetError {
 
 pub async fn connect_amqp(args: &AMQPArgs) -> Result<AMQPConnection, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    if args.url.scheme() == "amqps" {
+        super::ensure_rustls_provider_installed()?;
+    }
     tokio::time::timeout(Duration::from_secs(5), async {
         let url = connection_url(args)?;
         // Reconnect explicitly so every new channel enables publisher confirms below.

crates/targets/src/target/mod.rs

@@ -27,7 +27,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::thread_local;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
-use tracing::{debug, warn};
+use tracing::warn;
 
 pub mod amqp;
 pub mod kafka;
@@ -947,16 +947,12 @@ pub(crate) fn encode_failed_entry(
 
 /// Ensures a rustls crypto provider is installed before any TLS operation.
 ///
-/// Multiple target modules (MySQL, Redis, Postgres, MQTT) need this because
+/// Target modules may be the first to construct a TLS client. In a FIPS build,
-/// each may be the first to perform a TLS handshake. Idempotent: if a
+/// an incompatible provider installed by the host is rejected.
-/// provider is already registered, returns immediately.
+pub(crate) fn ensure_rustls_provider_installed() -> Result<(), TargetError> {
-pub(crate) fn ensure_rustls_provider_installed() {
+    rustfs_tls_runtime::ensure_crypto_provider()
-    if rustls::crypto::CryptoProvider::get_default().is_some() {
+        .map(|_| ())
-        return;
+        .map_err(|err| TargetError::Configuration(err.to_string()))
-    }
-    if let Err(err) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
-        debug!("rustls provider already installed or unavailable: {err:?}");
-    }
 }
 
 #[cfg(test)]

crates/targets/src/target/mqtt.rs

@@ -241,7 +241,7 @@ fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::Roo
 }
 
 fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transport, TargetError> {
-    super::ensure_rustls_provider_installed();
+    super::ensure_rustls_provider_installed()?;
 
     let client_config = match tls
         .policy

crates/targets/src/target/mysql.rs

@@ -881,7 +881,7 @@ async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<(Pool, bool), Ta
         .db_name(Some(dsn.database.clone()));
 
     if dsn.tls {
-        super::ensure_rustls_provider_installed();
+        super::ensure_rustls_provider_installed()?;
         let mut ssl_opts = SslOpts::default();
         if !args.tls_ca.is_empty() {
             let _ =

crates/targets/src/target/nats/mod.rs

@@ -204,6 +204,8 @@ fn nats_sends_credentials_without_tls(args: &NATSArgs) -> bool {
 
 pub async fn connect_nats(args: &NATSArgs) -> Result<async_nats::Client, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    super::ensure_rustls_provider_installed()?;
 
     let mut options = async_nats::ConnectOptions::new().require_tls(args.tls_required);
 

crates/targets/src/target/postgres.rs

@@ -462,7 +462,7 @@ pub fn table_probe_sql(schema: &str, table: &str) -> String {
 /// When `tls_client_cert` and `tls_client_key` are both set the connection
 /// uses mTLS authentication; otherwise no client cert is sent.
 pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, TargetError> {
-    super::ensure_rustls_provider_installed();
+    super::ensure_rustls_provider_installed()?;
 
     let mut root_store = rustls::RootCertStore::empty();
 
@@ -939,6 +939,32 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "fips")]
+    #[test]
+    fn fips_postgres_rejects_incompatible_host_provider() {
+        const CHILD: &str = "RUSTFS_TEST_POSTGRES_FIPS_PROVIDER";
+        if std::env::var_os(CHILD).is_some() {
+            let mut provider = rustfs_tls_runtime::default_crypto_provider();
+            provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            provider.install_default().expect("install incompatible host provider");
+            let error = build_tls_config(&base_args()).expect_err("Postgres must reject the incompatible provider");
+            assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("fips_postgres_rejects_incompatible_host_provider")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated Postgres TLS test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+
     #[test]
     fn validate_disabled_skips_all_checks() {
         let args = PostgresArgs {

crates/targets/src/target/pulsar.rs

@@ -90,7 +90,7 @@ impl PulsarArgs {
             return Ok(());
         }
 
-        validate_pulsar_broker(&self.broker)?;
+        let parsed = validate_pulsar_broker(&self.broker)?;
 
         if self.topic.trim().is_empty() {
             return Err(TargetError::Configuration("Pulsar topic cannot be empty".to_string()));
@@ -116,8 +116,6 @@ impl PulsarArgs {
             return Err(TargetError::Configuration("Pulsar queue directory must be an absolute path".to_string()));
         }
 
-        let parsed = Url::parse(&self.broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid Pulsar broker URL: {e} (value: '{}')", self.broker)))?;
         let tls_enabled = parsed.scheme() == "pulsar+ssl";
         // A CA bundle is TLS trust material a plaintext `pulsar://` broker
         // silently ignores, so treat it as a genuine misconfiguration. The
@@ -173,6 +171,10 @@ fn pulsar_producer_name(target_id: &TargetID, target_type: TargetType, nonce: Uu
 
 pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>, TargetError> {
     args.validate()?;
+    #[cfg(feature = "fips")]
+    if Url::parse(&args.broker).is_ok_and(|url| url.scheme() == "pulsar+ssl") {
+        super::ensure_rustls_provider_installed()?;
+    }
 
     let mut builder = Pulsar::builder(args.broker.clone(), TokioExecutor);
 

crates/targets/src/target/redis.rs

@@ -766,7 +766,7 @@ pub(crate) fn build_redis_client(args: &RedisArgs) -> Result<Client, TargetError
 
     let secure_scheme = matches!(args.url.scheme(), "rediss" | "valkeys");
     if secure_scheme {
-        super::ensure_rustls_provider_installed();
+        super::ensure_rustls_provider_installed()?;
         let tls_certs = TlsCertificates {
             client_tls: read_client_tls(&args.tls)?,
             root_cert: read_root_cert(&args.tls)?,

crates/targets/src/target/webhook.rs

@@ -305,6 +305,8 @@ where
         args: &WebhookArgs,
         resolver: impl reqwest::dns::Resolve + 'static,
     ) -> Result<Client, TargetError> {
+        #[cfg(feature = "fips")]
+        super::ensure_rustls_provider_installed()?;
         let mut client_builder = Client::builder()
             .no_proxy()
             .dns_resolver(resolver)
@@ -848,6 +850,51 @@ mod tests {
         }
     }
 
+    #[test]
+    fn webhook_provider_policy_isolated() {
+        const CHILD: &str = "RUSTFS_TEST_WEBHOOK_PROVIDER_POLICY";
+        if std::env::var_os(CHILD).is_some() {
+            if cfg!(feature = "fips") {
+                let mut provider = rustfs_tls_runtime::default_crypto_provider();
+                provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+                provider.install_default().expect("install incompatible host provider");
+            } else {
+                assert!(rustls::crypto::CryptoProvider::get_default().is_none());
+            }
+            let args = WebhookArgs {
+                skip_tls_verify: true,
+                ..base_args()
+            };
+            let result = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(
+                &args,
+                StaticResolver("127.0.0.1".parse().expect("loopback IP")),
+            );
+            if cfg!(feature = "fips") {
+                let error = result.expect_err("FIPS webhook must reject the incompatible provider");
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+            } else {
+                result.expect("default webhook client should build");
+                assert!(
+                    rustls::crypto::CryptoProvider::get_default().is_none(),
+                    "building a default webhook must not install a process-wide provider"
+                );
+            }
+            return;
+        }
+        let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+            .arg("webhook_provider_policy_isolated")
+            .arg("--nocapture")
+            .env(CHILD, "1")
+            .output()
+            .expect("start isolated webhook provider test");
+        assert!(
+            output.status.success(),
+            "stdout={} stderr={}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+
     async fn http_status_url(status: u16) -> Url {
         let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
         let address = listener.local_addr().expect("test server address");

crates/tls-runtime/src/certs.rs

@@ -610,10 +610,12 @@ pub fn validate_certified_key(
             }
         }
     }
-    let certified = CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &rustls::crypto::aws_lc_rs::default_provider())
+    let certified =
-        .map_err(|source| TlsRuntimeError::InvalidCertifiedKey {
+        CertifiedKey::from_der(certs.to_vec(), key.clone_key(), &crate::default_crypto_provider()).map_err(|source| {
-            name: name.to_string(),
+            TlsRuntimeError::InvalidCertifiedKey {
-            source,
+                name: name.to_string(),
+                source,
+            }
         })?;
     certified
         .keys_match()

crates/tls-runtime/src/lib.rs

@@ -21,6 +21,7 @@ pub mod fingerprint;
 pub mod material;
 pub mod metrics;
 pub mod outbound;
+pub mod provider;
 pub mod server;
 pub mod source;
 pub mod state;
@@ -45,6 +46,7 @@ pub use outbound::{
     GlobalOutboundTlsStateSummary, GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation,
     load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state,
 };
+pub use provider::{CryptoProviderError, CryptoProviderStatus, default_crypto_provider, ensure_crypto_provider};
 pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop};
 pub use source::{TlsFileLayout, TlsSource, TlsSourceKind, TlsSourceSnapshot};
 pub use state::OutboundOnlySnapshotArgs;

crates/tls-runtime/src/provider.rs

@@ -0,0 +1,108 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustls::crypto::CryptoProvider;
+
+/// The outcome of ensuring the process-wide rustls provider.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CryptoProviderStatus {
+    Installed,
+    AlreadyInstalled,
+}
+
+/// An incompatible process-wide rustls provider prevents FIPS startup.
+#[derive(Debug, thiserror::Error)]
+#[error(
+    "FIPS TLS requested, but the process-wide rustls crypto provider is not FIPS compatible; configure the host to install a FIPS provider before starting RustFS"
+)]
+pub struct CryptoProviderError {
+    _private: (),
+}
+
+/// Construct the provider selected by this crate's feature configuration.
+pub fn default_crypto_provider() -> CryptoProvider {
+    #[cfg(feature = "fips")]
+    {
+        rustls::crypto::default_fips_provider()
+    }
+    #[cfg(not(feature = "fips"))]
+    {
+        rustls::crypto::aws_lc_rs::default_provider()
+    }
+}
+
+/// Install or reuse the process-wide rustls provider.
+///
+/// # Errors
+/// Returns an error in a FIPS build if the provider actually installed in the
+/// process is absent or does not satisfy rustls's FIPS checks.
+pub fn ensure_crypto_provider() -> Result<CryptoProviderStatus, CryptoProviderError> {
+    // Rustls's process-wide provider is write-once; check again after a race.
+    let status = if CryptoProvider::get_default().is_some() || default_crypto_provider().install_default().is_err() {
+        CryptoProviderStatus::AlreadyInstalled
+    } else {
+        CryptoProviderStatus::Installed
+    };
+    #[cfg(feature = "fips")]
+    if !CryptoProvider::get_default().is_some_and(|provider| provider.fips()) {
+        return Err(CryptoProviderError { _private: () });
+    }
+    Ok(status)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{CryptoProviderStatus, default_crypto_provider, ensure_crypto_provider};
+
+    #[test]
+    fn provider_policy_isolated() {
+        const SCENARIO: &str = "RUSTFS_TLS_TEST_PROVIDER_SCENARIO";
+        if let Ok(scenario) = std::env::var(SCENARIO) {
+            match scenario.as_str() {
+                "fresh" => {
+                    assert_eq!(ensure_crypto_provider().expect("initial install"), CryptoProviderStatus::Installed);
+                    assert_eq!(ensure_crypto_provider().expect("provider reuse"), CryptoProviderStatus::AlreadyInstalled);
+                }
+                "incompatible" => {
+                    let mut provider = default_crypto_provider();
+                    provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+                    provider.install_default().expect("install incompatible provider");
+                    let result = ensure_crypto_provider();
+                    if cfg!(feature = "fips") {
+                        assert!(result.is_err(), "FIPS mode must reject the installed provider");
+                    } else {
+                        assert_eq!(result.expect("reuse existing provider"), CryptoProviderStatus::AlreadyInstalled);
+                    }
+                }
+                _ => panic!("unknown provider scenario"),
+            }
+            return;
+        }
+
+        for scenario in ["fresh", "incompatible"] {
+            let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+                .arg("provider_policy_isolated")
+                .arg("--nocapture")
+                .env(SCENARIO, scenario)
+                .output()
+                .expect("start isolated provider test");
+            assert!(
+                output.status.success(),
+                "{scenario}: stdout={} stderr={}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+    }
+}

crates/utils/Cargo.toml

@@ -26,6 +26,7 @@ categories = ["web-programming", "development-tools", "cryptography"]
 
 [dependencies]
 hotpath.workspace = true
+aws-lc-rs = { workspace = true, optional = true }
 base64-simd = { workspace = true, optional = true }
 blake2 = { workspace = true, optional = true }
 brotli = { workspace = true, optional = true }
@@ -89,6 +90,7 @@ path = []  # path manipulation features
 compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
 string = ["dep:regex"]
 crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1", "dep:sha2"]
+fips = ["crypto", "dep:aws-lc-rs", "aws-lc-rs/fips"]
 hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"]
 os = ["dep:rustix", "dep:tempfile", "dep:windows"]  # operating system utilities
 integration = []  # integration test features

crates/utils/src/crypto.rs

@@ -13,12 +13,77 @@
 // limitations under the License.
 
 use hex_simd::{AsOut, AsciiCase};
-use hmac::{Hmac, KeyInit, Mac};
+use hmac::Hmac;
+#[cfg(not(feature = "fips"))]
+pub use hmac::{KeyInit, Mac};
 use hyper::body::Bytes;
+#[cfg(not(feature = "fips"))]
 use sha1::Sha1;
 use sha2::{Digest, Sha256};
 use std::mem::MaybeUninit;
 
+#[cfg(not(feature = "fips"))]
+pub type HmacSha256 = Hmac<Sha256>;
+
+#[cfg(feature = "fips")]
+pub struct HmacSha256Output([u8; 32]);
+
+#[cfg(feature = "fips")]
+impl HmacSha256Output {
+    #[must_use]
+    pub fn into_bytes(self) -> hmac::digest::Output<Hmac<Sha256>> {
+        self.0.into()
+    }
+}
+
+#[cfg(feature = "fips")]
+pub trait KeyInit: Sized {
+    fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength>;
+}
+
+#[cfg(feature = "fips")]
+pub trait Mac: Sized {
+    fn update(&mut self, data: &[u8]);
+    fn finalize(self) -> HmacSha256Output;
+    fn verify_slice(self, tag: &[u8]) -> Result<(), hmac::digest::MacError>;
+}
+
+#[cfg(feature = "fips")]
+#[derive(Clone)]
+pub struct HmacSha256(aws_lc_rs::hmac::Context);
+
+#[cfg(feature = "fips")]
+impl HmacSha256 {
+    pub fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength> {
+        let signing_key = aws_lc_rs::hmac::Key::new(aws_lc_rs::hmac::HMAC_SHA256, key);
+        Ok(Self(aws_lc_rs::hmac::Context::with_key(&signing_key)))
+    }
+}
+
+#[cfg(feature = "fips")]
+impl KeyInit for HmacSha256 {
+    fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength> {
+        Self::new_from_slice(key)
+    }
+}
+
+#[cfg(feature = "fips")]
+impl Mac for HmacSha256 {
+    fn update(&mut self, data: &[u8]) {
+        self.0.update(data);
+    }
+
+    fn finalize(self) -> HmacSha256Output {
+        let tag = self.0.sign();
+        HmacSha256Output(tag.as_ref().try_into().expect("HMAC-SHA256 tag is 32 bytes"))
+    }
+
+    fn verify_slice(self, tag: &[u8]) -> Result<(), hmac::digest::MacError> {
+        let computed = self.0.sign();
+        aws_lc_rs::constant_time::verify_slices_are_equal(computed.as_ref(), tag).map_err(|_| hmac::digest::MacError)
+    }
+}
+
 /// Base64 URL safe encoding without padding
 /// `base64_encode_url_safe_no_pad(input)`
 ///
@@ -84,9 +149,18 @@ pub fn is_sha256_checksum(s: &str) -> bool {
 /// A 20-byte array containing the HMAC-SHA1 hash of the input data using the provided key
 ///
 pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
-    let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).expect("operation should succeed");
+    #[cfg(feature = "fips")]
-    m.update(data.as_ref());
+    {
-    m.finalize().into_bytes().into()
+        let signing_key = aws_lc_rs::hmac::Key::new(aws_lc_rs::hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, key.as_ref());
+        let tag = aws_lc_rs::hmac::sign(&signing_key, data.as_ref());
+        tag.as_ref().try_into().expect("HMAC-SHA1 tag is 20 bytes")
+    }
+    #[cfg(not(feature = "fips"))]
+    {
+        let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).expect("operation should succeed");
+        m.update(data.as_ref());
+        m.finalize().into_bytes().into()
+    }
 }
 
 /// HMAC-SHA256 hashing
@@ -100,7 +174,7 @@ pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
 /// A 32-byte array containing the HMAC-SHA256 hash of the input data using the provided key
 ///
 pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
-    let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).expect("operation should succeed");
+    let mut m = HmacSha256::new_from_slice(key.as_ref()).expect("operation should succeed");
     m.update(data.as_ref());
     m.finalize().into_bytes().into()
 }
@@ -167,3 +241,20 @@ fn sha256_checksum_rejects_wrong_length_or_non_lowercase_hex() {
     assert!(!is_sha256_checksum("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdeg"));
     assert!(!is_sha256_checksum("0123456789ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef"));
 }
+
+#[test]
+fn test_hmac_sha256_rfc4231() {
+    #[cfg(feature = "fips")]
+    aws_lc_rs::try_fips_mode().expect("aws-lc-rs FIPS mode must be active when `rustfs-utils/fips` is enabled");
+
+    let key = [0x0b_u8; 20];
+    let data = b"Hi There";
+    assert_eq!(hex(hmac_sha1(key, data)), "b617318655057264e28bc0b6fb378c8ef146be00");
+    let mac = hmac_sha256(key, data);
+    assert_eq!(hex(mac), "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7");
+
+    let mut verifier = <HmacSha256 as KeyInit>::new_from_slice(&key).expect("valid key");
+    verifier.update(data);
+    assert!(verifier.clone().verify_slice(&mac).is_ok());
+    assert!(verifier.verify_slice(&[0_u8; 32]).is_err());
+}

crates/zstd-compat/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zstd"
-version = "0.13.3"
+version = "0.13.4"
 publish = false
 edition.workspace = true
 license.workspace = true
@@ -33,10 +33,10 @@ zstd-safe = { version = "8.2.0", features = ["seekable"] }
 toml = "1.1.8"
 semver = "1.0.28"
 apache-avro.workspace = true
-arrow-array = "59.3.2"
+arrow-array = "60.1.0"
-arrow-ipc = { version = "59.3.2", features = ["zstd"] }
+arrow-ipc = { version = "60.1.0", features = ["zstd"] }
-arrow-schema = "59.3.2"
+arrow-schema = "60.1.0"
-parquet = { version = "59.3.2", default-features = false, features = ["arrow", "zstd"] }
+parquet = { version = "60.1.0", default-features = false, features = ["arrow", "zstd"] }
 kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
 bytes.workspace = true
 

crates/zstd-compat/README.md

@@ -6,6 +6,6 @@ This private package satisfies the version constraints of Avro, Arrow IPC, Parqu
 
 The fixed implementation closes RUSTSEC-2026-0391, RUSTSEC-2026-0398, and RUSTSEC-2026-0403. A successful advisory scan alone is insufficient evidence: cargo-deny skips local path packages. The dependency-graph test therefore requires this adapter's sole runtime dependency to be the exact fixed registry implementation and rejects vulnerable registry codec versions. Security tests exercise abandoned stream contexts, malformed legacy frames, and seekable offsets past EOF. Consumer tests retain zstandard support in all four upstream formats.
 
-Run `cargo nextest run --locked -p zstd@0.13.3` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production.
+Run `cargo nextest run --locked -p zstd@0.13.4` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production.
 
 Remove this package and the root Cargo patch when all consumers accept the fixed zstd release line directly. The cleanup marker is registered in the architecture compatibility register.

crates/zstd-compat/tests/dependency_graph.rs

@@ -57,7 +57,7 @@ fn legacy_constraints_resolve_only_to_the_fixed_registry_implementation() {
         let version = Version::parse(package["version"].as_str().expect("package version")).expect("valid package semver");
         if name == "zstd" && package.get("source").is_none() {
             adapters += 1;
-            assert_eq!(version, Version::new(0, 13, 3));
+            assert_eq!(version, Version::new(0, 13, 4));
             let runtime = package["dependencies"].as_array().expect("adapter dependency edges");
             assert!(runtime.iter().any(|value| value.as_str() == Some("zstd 0.14.3")));
         } else {

deny.toml

@@ -36,10 +36,6 @@ allow-git = [
     # Official s3s repository pinned for Iceberg SigV4 path encoding support.
     # owner: GatewayJ  review: 2026-10
     "https://github.com/s3s-project/s3s",
-    # Upstream OIDC/OAuth2 crypto migrations, pinned to reviewed PR commits.
-    # owner: rustfs-maintainers  review: 2026-10
-    "https://github.com/ramosbugs/openidconnect-rs",
-    "https://github.com/ramosbugs/oauth2-rs",
     # Completion ownership patch fixed to rustfs/uring#20.
     # owner: overtrue  review: 2026-10
     "https://github.com/rustfs/uring.git",

docs/architecture/compat-cleanup-register.md

@@ -11,6 +11,8 @@
 
 ## Open Items
 
+- `openidconnect-hmac-signature-width` ID token verification guard: openidconnect 4.1.0 converts decoded HMAC signatures to fixed-size arrays, which panic on unexpected lengths. Check HS256, HS384, and HS512 signature widths before delegating all cryptographic, issuer, audience, expiry, and nonce checks to the upstream verifier. Remove the guard after a supported upstream release returns verification errors for arbitrary HMAC signature lengths.
+
 - `zstd-013-fixed-backend` dependency constraint bridge: Avro, Arrow IPC, Parquet, and Kafka still require zstd 0.13. The local adapter re-exports the fixed registry zstd 0.14.3 implementation and its zstd-safe types. Remove the adapter and Cargo patch after every dependency accepts the fixed zstd line without the 0.13 bridge.
 
 - `8565` historical rebalance replacement recovery: older schema-seven intents can exhaust their retry budget before format repair when retained rebalance metadata describes removed pools. After validating blank targets, terminal historical topology, and ownership, grant one durable handoff and one rebuild attempt. Remove after supported direct upgrades no longer admit snapshots lacking replacement_preformat_handoff_retry_done and all retained affected intents have been resolved.

docs/architecture/s3-tables-support-matrix.md

@@ -64,7 +64,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
 | Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
 | Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
 | Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
-| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. |
+| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. Object-backed recovery reports a staged request as SUPERSEDED only when a reachable finalized same-base winner proves it lost CAS and its idempotency binding is intact. The original staged receipt is retained; its old request still conflicts. The winning history must be retained with the losing receipt; missing or contradictory history requires manual review. |
 | Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
 | Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
 | LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. |

docs/operations/kms-cryptographic-compliance.md

@@ -1,29 +1,30 @@
 # Cryptographic compliance positioning
 
 **Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs` / `rustfs-tls-runtime` / `rustfs-crypto` `fips` features and algorithm deprecation.
-**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `rustfs/Cargo.toml` and `crates/tls-runtime/Cargo.toml` (`fips` feature); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
+**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); the `fips` features in `rustfs/Cargo.toml` and the affected library manifests; `crates/tls-runtime/src/provider.rs` (provider policy); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes).
 
 This document records where RustFS stands on cryptographic module validation and what may and may not be said about it, so the question is answered once, from the code. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md).
 
 ## Status: not FIPS 140-3 validated
 
-**RustFS is not FIPS 140-3 (or 140-2) validated as a product, and in the default build no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS itself, or the libraries it uses in the default shipped configuration. An opt-in `--features fips` build flag on `rustfs` switches the `rustls` / `aws-lc-rs` provider to `aws-lc-fips-sys` for TLS, X.509, and JWT operations, while object data-path and local KMS operations remain on unvalidated RustCrypto software implementations. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS as a whole has or pursues.
+**RustFS is not FIPS 140-3 (or 140-2) validated as a product, and in the default build no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS itself, or the libraries it uses in the default shipped configuration. An opt-in `--features fips` build flag on `rustfs` enables the `aws-lc-fips-sys` backend in the applicable dependency graph; it does not establish that every TLS, X.509, or JWT operation uses an approved service. Object data-path and local KMS operations remain on unvalidated RustCrypto software implementations. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS as a whole has or pursues.
 
 ### What the process actually links
 
 | Layer | Where | Implementation | Primitives | Validated module |
 | --- | --- | --- | --- | --- |
-| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider (`default_provider` by default; `default_fips_provider` when built with `--features fips`) | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange (restricted to FIPS-approved suites and groups when `--features fips` is enabled) | No in the default build (`aws-lc-sys`); opt-in `--features fips` links `aws-lc-fips-sys` and asserts `provider.fips()` at startup |
+| TLS using the process-wide rustls provider | Provider installed or checked by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the process-wide provider (AWS-LC is installed when none exists) | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange (restricted to FIPS-approved suites and groups when `--features fips` is enabled) | No in the default build (`aws-lc-sys`); opt-in `--features fips` enables `aws-lc-fips-sys` and checks the installed provider at startup; module validation depends on the deployment |
+| Internode RPC HMAC authentication | `crates/ecstore/src/cluster/rpc/http_auth.rs` via `rustfs_utils::crypto::HmacSha256` | RustCrypto `hmac` + `sha2` in the default build; `aws-lc-rs` when `fips` is enabled | HMAC-SHA256 (`x-rustfs-signature`, `x-rustfs-rpc-signature-v2`, `x-rustfs-rpc-signature-v3`) | No in the default build; opt-in `--features fips` enables `rustfs-utils/fips` (`aws-lc-fips-sys`) |
 | Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
 | DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
 | Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
 | Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `rustfs-crypto` `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
-| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS `aws-lc-sys` build by default; unified with `aws-lc-fips-sys` when `rustfs` is built with `--features fips` | No in the default build; links `aws-lc-fips-sys` only when built with `-p rustfs --features fips` |
+| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS `aws-lc-sys` in the standalone default build; the FIPS backend can be enabled by Cargo feature unification | No in the standalone default build; linking the FIPS backend does not establish validation of each operation |
 
 Two consequences are commonly assumed the other way:
 
-- **AWS-LC being present does not imply FIPS in the default build.** `aws-lc-rs` has a FIPS variant (`aws-lc-fips-sys`); default builds do not enable it. Building `rustfs` with `--features fips` enables `rustls/fips`, `tokio-rustls/fips`, and `rustfs-tls-runtime/fips`, which links `aws-lc-fips-sys` and installs `rustls::crypto::default_fips_provider()` (see [below](#the-rustfs-and-rustfs-tls-runtime-fips-feature)).
+- **AWS-LC being present does not imply FIPS in the default build.** `aws-lc-rs` has a FIPS variant (`aws-lc-fips-sys`); default builds do not enable it. Building `rustfs` with `--features fips` enables `rustls/fips`, `tokio-rustls/fips`, and `rustfs-tls-runtime/fips`, which enables `aws-lc-fips-sys` and installs or checks the process-wide rustls provider (see [below](#the-rustfs-and-rustfs-tls-runtime-fips-feature)).
-- **The object data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations, even when `rustfs` is built with `--features fips`. Enabling the TLS FIPS provider does not change the object encryption data path; see route 1 below.
+- **The object encryption data path uses RustCrypto.** When RustFS encrypts object data, it uses RustCrypto software implementations, even when `rustfs` is built with `--features fips`. Enabling the TLS FIPS provider does not change the object encryption data path; see route 1 below.
 
 ## Terminology red lines for external material
 
@@ -44,7 +45,7 @@ These rules apply to the README, CHANGELOG, release notes, marketing pages, sale
 
 Boilerplate when the topic cannot be avoided:
 
-> RustFS encrypts object data with AES-256-GCM and supports ChaCha20-Poly1305. These are FIPS-approved algorithms, but the implementations are not FIPS 140-3 validated cryptographic modules and RustFS makes no FIPS validation claim.
+> RustFS supports AES-256-GCM and ChaCha20-Poly1305 for object encryption. AES-256-GCM can be a FIPS-approved algorithm when used under the applicable conditions; ChaCha20-Poly1305 is not FIPS-approved. These implementations are not FIPS 140-3 validated cryptographic modules and RustFS makes no FIPS validation claim.
 
 ### Guard
 
@@ -52,13 +53,25 @@ Boilerplate when the topic cannot be avoided:
 
 ## The `rustfs` and `rustfs-tls-runtime` `fips` feature
 
-`rustfs/Cargo.toml` (`fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]`) and `crates/tls-runtime/Cargo.toml` (`fips = ["rustls/fips"]`) define an opt-in Cargo feature (disabled by default) for builds that require the `aws-lc-fips-sys` cryptographic module on the `aws-lc-rs` / `rustls` path:
+`rustfs/Cargo.toml` and the TLS runtime, protocols, targets, S3 client, and ECStore library manifests define opt-in `fips` features (disabled by default) for the `aws-lc-fips-sys` / `rustls` path. The RustFS feature propagates the policy to these libraries; when enabling a protocol library independently, also enable the required protocol feature.
 
-- **What it enables:** Activates `rustls/fips` and `tokio-rustls/fips`, which pull in `aws-lc-rs/fips` (`aws-lc-fips-sys`) and `rustls-webpki/aws-lc-rs-fips`. Because Cargo unifies features across the dependency graph, other `aws-lc-rs` consumers in the binary (`jsonwebtoken`, `rcgen`, `google-cloud-auth`, `async-nats`) also link against `aws-lc-fips-sys`.
+Cargo's `--all-features` also enables `fips`, including for these individual libraries. Such builds inherit the FIPS backend's target and native-tool requirements; select supported feature combinations deliberately rather than assuming every all-features build is supported.
-- **Runtime provider installation:** In `rustfs/src/startup_runtime_hooks.rs`, `build_default_crypto_provider()` selects `rustls::crypto::default_fips_provider()` (instead of `rustls::crypto::aws_lc_rs::default_provider()`) and asserts `provider.fips()` on startup, restricting TLS negotiation to FIPS-approved cipher suites (e.g., `TLS_AES_256_GCM_SHA384`, `TLS_AES_128_GCM_SHA256`, `TLS_ECDHE_*_AES_*_GCM_*`) and key exchange groups (excluding `TLS_CHACHA20_POLY1305_SHA256` and `X25519`).
+
-- **Build requirements:** Compiling `aws-lc-fips-sys` requires CMake, Go, and Clang/`libclang` in the build environment (unlike the default `aws-lc-sys` build).
+```bash
+cargo build --release --locked -p rustfs --bin rustfs --features fips
+```
+
+- **What it enables:** Activates `rustls/fips` and `tokio-rustls/fips`, which pull in `aws-lc-rs/fips` (`aws-lc-fips-sys`) and `rustls-webpki/aws-lc-rs-fips`. Cargo unifies features for the same dependency version in a build graph; consumers of that `aws-lc-rs` instance use its FIPS backend, but this does not establish approval for every operation or cover independent dependency versions.
+- **Runtime provider installation:** `rustfs-tls-runtime` installs or reuses the process-wide provider and checks the actual provider in FIPS builds. RustFS performs this check before observability initialization in standalone and embedded startup. FTPS, WebDAV, transition clients, and relevant notification TLS clients also invoke the shared policy before using the process-wide provider. An incompatible host provider causes a configuration error. The check does not inspect clients that use their own provider or a different TLS implementation.
+- **Build requirements:** The FIPS backend requires a C/C++ toolchain, CMake, and Go; targets without suitable pre-generated bindings also require bindgen and Clang/`libclang`. Check the locked dependency version and target platform for their exact requirements.
 - **What it does not change:** It does not alter the object data-path AEAD (`crates/kms`, `crates/rio`, `crates/rio-v2`) or `rustfs-crypto` config/IAM blob encryption, which continue to use pure-Rust `RustCrypto` crates.
 
+The RustFS server TLS configuration uses the process-wide rustls provider. The Kafka notification client and remote replication SDK explicitly select AWS-LC, so their providers are independent of the process default; for the locked rustls version, the unified `rustls/fips` feature restricts that provider's suites and groups. ECStore's insecure replication-client path still uses the process default: an application using ECStore independently must directly depend on `rustfs-tls-runtime` with its `fips` feature and call `rustfs_tls_runtime::ensure_crypto_provider()` before constructing that client when it requires the FIPS policy. Skipping TLS certificate verification does not authenticate the remote target, regardless of provider selection. Feature unification applies only to the same dependency version in a build graph; independent builds and third-party TLS stacks require separate verification. No complete outbound TLS or platform validation matrix is asserted here.
+
+NATS can upgrade a connection to TLS based on the server's response, so the targets library checks the process provider before connecting in a FIPS build, including for a plaintext-looking NATS URL. Webhook targets construct a TLS-capable HTTP client even for an HTTP endpoint and also check the provider when built with `fips`. An incompatible host provider therefore prevents these initializations even if a particular connection would not use TLS.
+
+A provider check or linked FIPS backend does not establish CMVP validation for a deployment or approval of every operation. Verify the specific module version, certificate status, security policy, build, operating environment, and approved services against the [CMVP validated modules list](https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules). The feature is opt-in; before enabling it, test client algorithms, certificates, TLS versions, rolling upgrades, and rollback in the target environment.
+
 ## The `rustfs-crypto` `fips` feature: what it actually does
 
 `crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
@@ -80,8 +93,8 @@ Selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src
 
 | Route | Scope | What it buys | Verdict |
 | --- | --- | --- | --- |
-| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) across the data path | The TLS/X.509/JWT provider swap is wired via the opt-in `-p rustfs --features fips` build flag. Extending `aws-lc-fips-sys` to the data path would require re-implementing every AEAD/KDF call in `crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, Clang/`libclang`, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Data-path rewrite remains heavy and re-opens platform-support constraints; TLS/X.509/JWT is opt-in via `--features fips` |
+| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) across the data path | The opt-in `-p rustfs --features fips` flag enables the FIPS backend for the relevant dependency graph; individual TLS, X.509, and JWT services still require verification. Extending `aws-lc-fips-sys` to the data path would require re-implementing every AEAD/KDF call in `crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs a C toolchain, CMake, Go, and target-dependent bindings tooling, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | A claim that particular operations use a validated module requires verified module and deployment boundaries, certificate status, approved-mode operation, and service coverage; it does not establish product-level validation | Data-path rewrite remains heavy and re-opens platform-support constraints; the FIPS backend is opt-in via `--features fips` |
-| 2. Let an externally validated KMS carry key operations | Mostly built: the `VaultTransit` backend never lets key-encryption key material leave Vault, so master key generation, wrapping, unwrapping, and rotation happen inside whatever module Vault's seal/HSM is validated against. Remaining work is configuration guidance and a supported-deployment statement | A defensible partial answer to "where do keys live and who validated that". The object data path stays RustCrypto and TLS stays non-FIPS AWS-LC: "key management operations are performed by an externally validated module; the object data path is not validated" | Nearest partial step, no code rewrite. Point customers here when the requirement is key custody rather than a certificate covering the storage layer |
+| 2. Let an externally validated KMS carry key operations | Mostly built: the `VaultTransit` backend never lets key-encryption key material leave Vault, so master key generation, wrapping, unwrapping, and rotation happen inside whatever module Vault's seal/HSM is validated against. Remaining work is configuration guidance and a supported-deployment statement | A defensible partial answer to "where do keys live and who validated that". The object data path stays RustCrypto; TLS backend selection remains a separate decision: "key management operations are performed by an externally validated module; the object data path is not validated" | Nearest partial step, no code rewrite. Point customers here when the requirement is key custody rather than a certificate covering the storage layer |
 | 3. Make no validation claim (current default) | This document plus the grep guard | Nothing further | The current decision. FIPS 140-3 validation is not a roadmap target; adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI |
 
 ## Algorithm disablement and migration policy

docs/operations/site-replication-operations.md

@@ -131,7 +131,10 @@ persist until the refresh settles.
 Control-plane retry begins on the 30-second drain, while heavyweight snapshots,
 pending topology operations, and bucket wiring are revisited on the 600-second
 pass. Object MRF entries are persisted every 10 seconds by default and target
-health is probed every 5 seconds. These are scheduling bounds, not delivery
+health is probed every 5 seconds. The persisted MRF journal is replayed at
+startup and then once every 5 minutes across the cluster, so a delete-marker purge or an overflowed
+replication whose target was down converges within one replay interval after
+the target returns. These are scheduling bounds, not delivery
 SLAs: network timeouts and the amount of queued work add to them.
 
 Objects that must be rediscovered by the scanner have this conservative upper

docs/testing/README.md

@@ -14,7 +14,7 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 | Outbound target matrix | Replication of every object shape (empty, plain, retention, legal hold, multipart, checksummed, SSE-S3) against every remote-target failure mode the fake target models; an explicit expectation table pins known-red cells to an open issue | `python3 scripts/e2e_binary.py build`, then `python3 scripts/e2e_binary.py run -- cargo nextest run -p e2e_test -E 'test(/^replication_target_matrix_test::/)'` | With `e2e-repl-nightly`; required locally for any change to outbound client defaults (SOP: [`docs/postmortems/2026-09-03-replication-checksum-default-regression.md`](../postmortems/2026-09-03-replication-checksum-default-regression.md)) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | Non-documentation PRs, required through `Test and Lint` (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | Non-documentation PRs: implemented and lifecycle S3 lanes required through `Test and Lint`; the separate s3-tests workflow provides additional PR reporting and a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos nightly via `e2e-distributed` |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |
 | Conditional reads | Paired ordinary/hit/miss GET latency, throughput and read-lock measurements | `scripts/run_conditional_read_abba.py` | Local conditional-read acceptance; [guide and scope](conditional-read-benchmark.md) |
@@ -88,7 +88,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
-| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
+| `e2e-distributed` | Nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |
 
 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):

docs/testing/ci-gates.md

@@ -35,7 +35,7 @@ gh api repos/rustfs/rustfs/rulesets/6436880 \
   --jq '.rules[] | select(.type == "required_status_checks") | .parameters'
 ```
 
-The aggregate requires the validation lanes already selected by `ci.yml`; this closes the gap where a failing critical lane left the required workspace check green. Independent workflows remain report-only unless separately required. Before adding a new expensive lane or moving existing PR coverage to a schedule, collect representative execution and regression evidence, establish ownership and a working scheduled replacement, and update this reference with the resulting policy.
+The aggregate requires the validation lanes already selected by `ci.yml`; this closes the gap where a failing critical lane left the required workspace check green. Independent workflows remain report-only unless separately required. Before adding a new expensive lane or moving existing PR coverage to a schedule, collect representative execution and regression evidence, establish ownership and a working scheduled replacement, and update this reference with the resulting policy. `e2e-distributed.yml` and `e2e-upgrade.yml` are that kind of replacement: the 4-node lane runs on its nightly schedule, and the single-node upgrade lane runs on its weekly Monday schedule, on version tags, and on manual dispatch. Neither is part of `Test and Lint`. Dispatch the workflow against a candidate branch when that signal is needed before merge.
 
 Superseded PR attempts are cancelled. A running main validation finishes, with only the newest pending main run retained, so frequent merges cannot continuously cancel the full baseline. Having a `merge_group` trigger does not itself require use of the merge queue.
 
@@ -66,7 +66,6 @@ The four site-replication state-writer concurrency proofs reserve the available
 | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
 | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety`, `Startup Smoke` | `windows-filesystem.yml` `rename-safety`, `startup-smoke` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows; `cargo build --locked -p rustfs --bin rustfs`, then `rustfs.exe --version` and a single-volume server polled on `/health/ready` |
 | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
-| PR touching `paths` in `e2e-upgrade.yml` | `Upgrade compatibility` | `e2e-upgrade.yml` `upgrade` | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
 | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
 | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
 | PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` |
@@ -90,7 +89,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 | `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | yes | dispatch `cache-warm.yml` |
 | `build.yml` (nightly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--features e2e-test-hooks` on both steps with `--profile e2e-nightly`; `--features ftps,webdav,sftp` on both steps with `-j 1 --profile e2e-protocols` |
-| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
+| `e2e-distributed.yml` (nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes (disabled) | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -100,7 +99,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
-| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | see the PR row |
+| `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
 | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |

fuzz/Cargo.lock

@@ -273,9 +273,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.156.0"
+version = "1.157.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
+checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1837,18 +1837,18 @@ dependencies = [
 
 [[package]]
 name = "hotpath"
-version = "0.30.2"
+version = "0.30.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
+checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935"
 dependencies = [
  "hotpath-macros",
 ]
 
 [[package]]
 name = "hotpath-macros"
-version = "0.30.2"
+version = "0.30.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
+checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a"
 
 [[package]]
 name = "http"
@@ -2418,9 +2418,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.34"
+version = "0.4.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
 
 [[package]]
 name = "lru"
@@ -4134,8 +4134,7 @@ dependencies = [
 [[package]]
 name = "rustfs-uring"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/rustfs/uring.git?rev=eb8fc160e71d804427de1e905f5c226d580f71bf#eb8fc160e71d804427de1e905f5c226d580f71bf"
-checksum = "f7deda770b7981ee3bfaa202ecd18625fc794c962b8d58cd4f436952b7e66dcf"
 dependencies = [
  "io-uring",
  "libc",

rustfs/Cargo.toml

@@ -58,7 +58,7 @@ required-features = ["swift"]
 
 [features]
 default = ["ftps", "webdav", "gcs"]
-fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]
+fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
 gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]

rustfs/src/app/object/delete.rs

@@ -295,16 +295,32 @@ fn enrich_delete_replication_state_if_needed(
     }
 }
 
-fn should_schedule_replica_delete_replication(
+/// The current peer set is read only when a rule has the derived shape. If it
+/// cannot be read, no rule is skipped: a redundant echo is safer than a chained
+/// target missing the delete.
+async fn replica_delete_fanout_state(
     snapshot: &DeleteReplicationConfigSnapshot,
+    bucket: &str,
     replication_source: &ObjectInfo,
     version_id: Option<Uuid>,
-) -> bool {
+) -> Option<ReplicationState> {
-    let Some(config) = snapshot.replication_config() else {
+    let config = snapshot.replication_config()?;
-        return false;
+    let site_peer_deployment_ids = if has_site_replication_shaped_rule(config) {
+        match crate::site_replication::site_replication_remote_peer_deployment_ids().await {
+            Ok(peers) => peers,
+            Err(error) => {
+                warn!(
+                    bucket = %bucket,
+                    error = %error,
+                    "failed to load site replication peers; forwarding replica delete along every rule"
+                );
+                HashSet::new()
+            }
+        }
+    } else {
+        HashSet::new()
     };
-
+    replica_delete_fanout_state_from_config(config, replication_source, version_id, &site_peer_deployment_ids)
-    delete_replication_state_from_config(config, replication_source, version_id, true).is_some()
 }
 
 fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> S3Result<()> {
@@ -1347,12 +1363,21 @@ impl DefaultObjectUsecase {
         let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker);
 
         let delete_replication_version_id = delete_replication_version_id(deleted_object_source, deleted_delete_marker_version);
-        let schedule_delete_replication = if opts.replication_request && replica {
+        let replica_fanout_state = if opts.replication_request && replica {
-            should_schedule_replica_delete_replication(
+            Some(
-                &delete_config_snapshot,
+                replica_delete_fanout_state(
-                replication_state_source,
+                    &delete_config_snapshot,
-                delete_replication_version_id,
+                    &bucket,
+                    replication_state_source,
+                    delete_replication_version_id,
+                )
+                .await,
             )
+        } else {
+            None
+        };
+        let schedule_delete_replication = if let Some(fanout_state) = &replica_fanout_state {
+            fanout_state.is_some()
         } else {
             should_schedule_delete_replication(
                 &opts,
@@ -1381,8 +1406,12 @@ impl DefaultObjectUsecase {
                 replication_state: None,
                 ..Default::default()
             };
-            set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
+            if let Some(Some(fanout_state)) = &replica_fanout_state {
-            enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
+                set_deleted_object_replication_state(&mut deleted_object, fanout_state);
+            } else {
+                set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
+                enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
+            }
             schedule_replication_delete(deleted_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
         }
 
@@ -3583,6 +3612,83 @@ mod tests {
         assert!(state.purge_targets.contains_key(&arn));
     }
 
+    #[test]
+    fn replica_delete_fanout_skips_site_replication_peers_but_keeps_chain_targets() {
+        let peer_arn = "arn:rustfs:replication::peer-dep:bucket";
+        let chain_arn = "arn:aws:s3:::downstream-bucket".to_string();
+        let site_rule = crate::site_replication::build_site_replication_rule(peer_arn, 10, "site-repl-peer-dep");
+        let mut chain_rule = site_rule.clone();
+        chain_rule.id = Some("chain".to_string());
+        chain_rule.priority = Some(1);
+        chain_rule.destination.bucket = chain_arn.clone();
+        let site_only = ReplicationConfiguration {
+            role: String::new(),
+            rules: vec![site_rule.clone()],
+        };
+        let site_and_chain = ReplicationConfiguration {
+            role: String::new(),
+            rules: vec![site_rule, chain_rule],
+        };
+        let marker = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            delete_marker: true,
+            replication_status: ReplicationStatusType::Replica,
+            ..Default::default()
+        };
+        let version = ObjectInfo {
+            delete_marker: false,
+            ..marker.clone()
+        };
+        // An ordinary bucket target ARN carries its target id, so an operator
+        // rule named `site-repl-<target id>` has the derived shape.
+        let shaped_operator_arn = "arn:minio:replication:us-east-1:target-1:downstream".to_string();
+        let mut shaped_operator_rule =
+            crate::site_replication::build_site_replication_rule(&shaped_operator_arn, 1, "site-repl-target-1");
+        shaped_operator_rule.priority = Some(1);
+        let shaped_operator_only = ReplicationConfiguration {
+            role: String::new(),
+            rules: vec![shaped_operator_rule],
+        };
+        assert!(has_site_replication_shaped_rule(&shaped_operator_only));
+        let site_peers = HashSet::from(["peer-dep".to_string()]);
+        let no_site_peers = HashSet::new();
+        let removed_peer_only = HashSet::from(["other-dep".to_string()]);
+        let purged_version = Some(Uuid::new_v4());
+        let fanout_targets = |state: ReplicationState, version_id: Option<Uuid>| {
+            if version_id.is_some() {
+                state.purge_targets.keys().cloned().collect::<Vec<_>>()
+            } else {
+                state.targets.keys().cloned().collect::<Vec<_>>()
+            }
+        };
+
+        for (source, version_id) in [(&marker, None), (&marker, purged_version), (&version, purged_version)] {
+            assert!(
+                delete_replication_state_from_config(&site_only, source, version_id, true).is_some(),
+                "the site-replication rule alone would forward the replica delete"
+            );
+            assert!(
+                replica_delete_fanout_state_from_config(&site_only, source, version_id, &site_peers).is_none(),
+                "a replica delete must not echo back across the site-replication mesh"
+            );
+
+            let state = replica_delete_fanout_state_from_config(&site_and_chain, source, version_id, &site_peers)
+                .expect("a replica delete must still follow a chained operator rule");
+            assert!(!state.replicate_decision_str.contains(peer_arn));
+            assert_eq!(fanout_targets(state, version_id), vec![chain_arn.clone()]);
+
+            for peers in [&no_site_peers, &removed_peer_only, &site_peers] {
+                let state = replica_delete_fanout_state_from_config(&shaped_operator_only, source, version_id, peers)
+                    .expect("an operator rule naming no current site peer must keep forwarding replica deletes");
+                assert_eq!(fanout_targets(state, version_id), vec![shaped_operator_arn.clone()]);
+            }
+            let state = replica_delete_fanout_state_from_config(&site_only, source, version_id, &no_site_peers)
+                .expect("with site replication disabled a site-repl rule is an operator rule");
+            assert_eq!(fanout_targets(state, version_id), vec![peer_arn.to_string()]);
+        }
+    }
+
     #[test]
     fn replica_delete_enrichment_must_not_reuse_upstream_targets() {
         let upstream_state = ReplicationState {

rustfs/src/app/object/mod.rs

@@ -29,8 +29,9 @@ use super::storage_api::object_usecase::access::{
 #[cfg(test)]
 use super::storage_api::object_usecase::bucket::quota::BucketQuota;
 use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker;
+use super::storage_api::object_usecase::bucket::replication::ReplicationState;
 #[cfg(test)]
-use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map};
+use super::storage_api::object_usecase::bucket::replication::replication_statuses_map;
 use super::storage_api::object_usecase::bucket::{
     VersioningConfigExt as _,
     lifecycle::{
@@ -49,10 +50,11 @@ use super::storage_api::object_usecase::bucket::{
     replication::{
         DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent,
         delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
-        force_delete_target_set, get_read_proxy_targets, has_active_delete_rule, load_delete_config_snapshot,
+        force_delete_target_set, get_read_proxy_targets, has_active_delete_rule, has_site_replication_shaped_rule,
-        must_replicate_object, persist_force_delete_intent, record_replication_proxy, schedule_object_replication,
+        load_delete_config_snapshot, must_replicate_object, persist_force_delete_intent, record_replication_proxy,
-        schedule_replication_delete, schedule_replication_deletes, set_deleted_object_replication_state,
+        replica_delete_fanout_state_from_config, schedule_object_replication, schedule_replication_delete,
-        should_schedule_delete_replication, should_use_existing_delete_replication_info,
+        schedule_replication_deletes, set_deleted_object_replication_state, should_schedule_delete_replication,
+        should_use_existing_delete_replication_info,
     },
     tagging::decode_tags,
     validate_restore_request,

rustfs/src/app/storage_api.rs

@@ -708,7 +708,6 @@ pub(crate) mod bucket {
         pub(crate) type DeleteReplicationConfigSnapshot =
             crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
         pub(crate) type ReplicateDecision = replication_contracts::ReplicateDecision;
-        #[cfg(test)]
         pub(crate) type ReplicationState = replication_contracts::ReplicationState;
         pub(crate) type ReplicationStatusType = replication_contracts::ReplicationStatusType;
         pub(crate) type ReplicationTargetValidationError = replication_contracts::ReplicationTargetValidationError;
@@ -959,6 +958,52 @@ pub(crate) mod bucket {
             replication_contracts::delete_replication_state_from_config(config, &source)
         }
 
+        /// Replication state for forwarding a replica delete (an inbound
+        /// replication request) to this site's own targets.
+        ///
+        /// Operator rules with ReplicaModifications keep forwarding, so chained
+        /// replication A -> B -> C propagates deletes (rustfs#2609). Derived
+        /// site-replication rules never do: site replication is a full mesh
+        /// where the origin delivers, and MRF-retries, every delete to each
+        /// peer itself. Forwarding along a `site-repl-<deploymentId>` rule only
+        /// echoes the delete back to the origin, where it removes the origin's
+        /// pending-purge version before its own delivery state is written back,
+        /// and duplicates it to every other peer (rustfs#9121). MinIO never
+        /// re-replicates a replication request at all.
+        ///
+        /// Rule ids and target ARN shapes are not reserved: an ordinary bucket
+        /// target ARN also carries a target id, so an operator rule can match
+        /// the derived shape. Only a rule naming a current remote peer in
+        /// `site_peer_deployment_ids` is skipped; with site replication
+        /// disabled the set is empty and every rule forwards.
+        pub(crate) fn replica_delete_fanout_state_from_config(
+            config: &s3s::dto::ReplicationConfiguration,
+            obj_info: &crate::storage::storage_api::StorageObjectInfo,
+            version_id: Option<Uuid>,
+            site_peer_deployment_ids: &HashSet<String>,
+        ) -> Option<replication_contracts::ReplicationState> {
+            let is_site_peer_rule = |rule: &s3s::dto::ReplicationRule| {
+                replication_contracts::site_replication_rule_deployment_id(rule)
+                    .is_some_and(|deployment_id| site_peer_deployment_ids.contains(deployment_id))
+            };
+            if !config.rules.iter().any(is_site_peer_rule) {
+                return delete_replication_state_from_config(config, obj_info, version_id, true);
+            }
+
+            let mut operator_config = config.clone();
+            operator_config.rules.retain(|rule| !is_site_peer_rule(rule));
+            delete_replication_state_from_config(&operator_config, obj_info, version_id, true)
+        }
+
+        /// Whether `config` has a rule with the derived site-replication shape,
+        /// so the replica-delete fan-out needs the current peer set.
+        pub(crate) fn has_site_replication_shaped_rule(config: &s3s::dto::ReplicationConfiguration) -> bool {
+            config
+                .rules
+                .iter()
+                .any(|rule| replication_contracts::site_replication_rule_deployment_id(rule).is_some())
+        }
+
         pub(crate) fn replication_target_arns(config: &s3s::dto::ReplicationConfiguration) -> HashSet<String> {
             replication_contracts::replication_target_arns(config)
         }

rustfs/src/site_replication/state.rs

@@ -368,6 +368,18 @@ pub(crate) async fn site_replication_deployment_ids() -> S3Result<HashSet<String
     Ok(state.peers.values().map(|peer| peer.deployment_id.clone()).collect())
 }
 
+/// Deployment ids of the remote peers this site's reconciler derives a
+/// `site-repl-<id>` rule for; empty when site replication is not enabled.
+/// Read on the replica-delete path so only rules naming a current peer are
+/// treated as reconciler-owned.
+pub(crate) async fn site_replication_remote_peer_deployment_ids() -> S3Result<HashSet<String>> {
+    let state = load_site_replication_state().await?;
+    if !state.enabled() {
+        return Ok(HashSet::new());
+    }
+    Ok(remote_peer_deployment_ids(&state, &current_local_runtime_peer(&state)))
+}
+
 pub(crate) async fn load_site_replication_state_no_lock(store: Arc<ECStore>) -> S3Result<SiteReplicationState> {
     match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
         Ok(data) => parse_site_replication_state(&data),

rustfs/src/startup_preflight.rs

@@ -63,6 +63,9 @@ pub(crate) async fn init_startup_server_preflight(
     config: &Config,
     env_compat_report: &ExternalEnvCompatReport,
 ) -> std::result::Result<(), StartupServerPreflightError> {
+    #[cfg(feature = "fips")]
+    crate::startup_runtime_hooks::install_default_crypto_provider().map_err(StartupServerPreflightError::Other)?;
+
     crate::config::init_config_snapshot(config);
     init_license(config.license.clone());
     init_startup_observability(config.obs_endpoint.clone()).await?;

rustfs/src/startup_runtime.rs

@@ -25,7 +25,7 @@ pub(crate) async fn init_startup_runtime_foundation(config: &Config) -> Result<(
     log_startup_runtime_diagnostics();
     init_profiling_runtime().await;
     rustfs_trusted_proxies::init();
-    install_default_crypto_provider();
+    install_default_crypto_provider()?;
     init_object_max_versions_config()?;
     init_outbound_tls_material(config).await
 }

rustfs/src/startup_runtime_hooks.rs

@@ -14,10 +14,7 @@
 
 use crate::license::license_status;
 use crate::startup_runtime_sources;
-#[cfg(not(feature = "fips"))]
+use rustfs_tls_runtime::{CryptoProviderStatus, ensure_crypto_provider};
-use rustls::crypto::aws_lc_rs::default_provider;
-#[cfg(feature = "fips")]
-use rustls::crypto::default_fips_provider as default_provider;
 use std::future::Future;
 use std::io::Result;
 use tracing::{debug, info, warn};
@@ -108,31 +105,24 @@ where
     shutdown();
 }
 
-fn build_default_crypto_provider() -> rustls::crypto::CryptoProvider {
+pub(crate) fn install_default_crypto_provider() -> Result<()> {
-    let provider = default_provider();
+    if ensure_crypto_provider().map_err(std::io::Error::other)? == CryptoProviderStatus::AlreadyInstalled {
-    #[cfg(feature = "fips")]
-    assert!(
-        provider.fips(),
-        "FIPS crypto provider requested via `fips` feature, but AWS-LC FIPS mode is not active"
-    );
-    provider
-}
-
-pub(crate) fn install_default_crypto_provider() {
-    if build_default_crypto_provider().install_default().is_err() {
         debug!(
             target: "rustfs::main",
             event = EVENT_CRYPTO_PROVIDER_STATE,
             component = LOG_COMPONENT_MAIN,
             subsystem = LOG_SUBSYSTEM_STARTUP,
-            provider = "aws_lc_rs",
+            provider = "process_default",
             state = "already_installed",
             "Rustls crypto provider state checked"
         );
     }
+    Ok(())
 }
 
 pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<()> {
+    install_embedded_default_crypto_provider()?;
+
     // Observability is a process-global tracing subscriber and can only be
     // installed once. A second embedded server in the same process reuses the
     // first server's subscriber — try to publish anyway (first server wins),
@@ -161,37 +151,228 @@ pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<
         }
     }
 
-    install_embedded_default_crypto_provider();
     rustfs_trusted_proxies::init();
 
     Ok(())
 }
 
-fn install_embedded_default_crypto_provider() {
+fn install_embedded_default_crypto_provider() -> Result<()> {
-    if let Err(err) = build_default_crypto_provider().install_default() {
+    if ensure_crypto_provider().map_err(std::io::Error::other)? == CryptoProviderStatus::AlreadyInstalled {
         debug!(
+            event = EVENT_CRYPTO_PROVIDER_STATE,
             component = LOG_COMPONENT_EMBEDDED,
             subsystem = LOG_SUBSYSTEM_EMBEDDED,
-            event = EVENT_CRYPTO_PROVIDER_STATE,
             state = "already_installed",
-            error = ?err,
             "Embedded crypto provider state changed"
         );
     }
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{build_default_crypto_provider, init_profiling_runtime_with, shutdown_profiling_runtime_with};
+    use super::{
+        init_profiling_runtime_with, install_default_crypto_provider, install_embedded_default_crypto_provider,
+        shutdown_profiling_runtime_with,
+    };
+    use rustfs_tls_runtime::default_crypto_provider as build_default_crypto_provider;
     use std::sync::{
         Arc,
         atomic::{AtomicBool, Ordering},
     };
 
+    #[cfg(feature = "fips")]
     #[test]
     fn default_crypto_provider_matches_fips_feature() {
         let provider = build_default_crypto_provider();
-        assert_eq!(provider.fips(), cfg!(feature = "fips"));
+        assert!(provider.fips());
+    }
+
+    #[cfg(feature = "fips")]
+    #[test]
+    fn fips_provider_builds_fips_tls_configs() {
+        let provider = Arc::new(build_default_crypto_provider());
+        assert!(provider.cipher_suites.iter().all(|suite| suite.fips()));
+        assert!(provider.kx_groups.iter().all(|group| group.fips()));
+        assert!(
+            !provider
+                .cipher_suites
+                .iter()
+                .any(|suite| { suite.suite() == rustls::CipherSuite::TLS13_CHACHA20_POLY1305_SHA256 })
+        );
+        assert!(
+            !provider
+                .kx_groups
+                .iter()
+                .any(|group| group.name() == rustls::NamedGroup::X25519)
+        );
+
+        let client = rustls::ClientConfig::builder_with_provider(provider.clone())
+            .with_safe_default_protocol_versions()
+            .expect("supported protocol versions")
+            .with_root_certificates(rustls::RootCertStore::empty())
+            .with_no_client_auth();
+        let server = rustls::ServerConfig::builder_with_provider(provider)
+            .with_safe_default_protocol_versions()
+            .expect("supported protocol versions")
+            .with_no_client_auth()
+            .with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new()));
+        assert!(client.fips());
+        assert!(server.fips());
+    }
+
+    #[cfg(feature = "fips")]
+    fn tls_handshake_with_provider(
+        client_provider: rustls::crypto::CryptoProvider,
+        version: &'static rustls::SupportedProtocolVersion,
+    ) -> std::result::Result<(), rustls::Error> {
+        let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).expect("generate certificate");
+        let cert_der = cert.cert.der().clone();
+        let private_key = rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der());
+        let mut roots = rustls::RootCertStore::empty();
+        roots.add(cert_der.clone()).expect("trust certificate");
+
+        let server_config = rustls::ServerConfig::builder_with_provider(Arc::new(build_default_crypto_provider()))
+            .with_protocol_versions(&[version])
+            .expect("server protocol version")
+            .with_no_client_auth()
+            .with_single_cert(vec![cert_der], private_key.into())
+            .expect("server identity");
+        assert!(server_config.fips());
+        let client_config = rustls::ClientConfig::builder_with_provider(Arc::new(client_provider))
+            .with_protocol_versions(&[version])
+            .expect("client protocol version")
+            .with_root_certificates(roots)
+            .with_no_client_auth();
+
+        let mut server = rustls::ServerConnection::new(Arc::new(server_config)).expect("server connection");
+        let mut client = rustls::ClientConnection::new(
+            Arc::new(client_config),
+            rustls::pki_types::ServerName::try_from("localhost").expect("server name"),
+        )
+        .expect("client connection");
+        for _ in 0..8 {
+            let mut client_bytes = Vec::new();
+            client.write_tls(&mut client_bytes).expect("write client TLS");
+            if !client_bytes.is_empty() {
+                server
+                    .read_tls(&mut std::io::Cursor::new(client_bytes))
+                    .expect("read server TLS");
+                server.process_new_packets()?;
+            }
+
+            let mut server_bytes = Vec::new();
+            server.write_tls(&mut server_bytes).expect("write server TLS");
+            if !server_bytes.is_empty() {
+                client
+                    .read_tls(&mut std::io::Cursor::new(server_bytes))
+                    .expect("read client TLS");
+                client.process_new_packets()?;
+            }
+            if !client.is_handshaking() && !server.is_handshaking() {
+                return Ok(());
+            }
+        }
+        panic!("TLS handshake did not complete within eight exchanges");
+    }
+
+    #[cfg(feature = "fips")]
+    #[test]
+    fn fips_provider_tls_handshake_accepts_and_rejects_algorithms() {
+        for version in [&rustls::version::TLS12, &rustls::version::TLS13] {
+            tls_handshake_with_provider(build_default_crypto_provider(), version).expect("FIPS TLS handshake");
+            let mut client_provider = build_default_crypto_provider();
+            client_provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+            assert!(tls_handshake_with_provider(client_provider, version).is_err());
+        }
+        let mut client_provider = build_default_crypto_provider();
+        client_provider.cipher_suites = vec![rustls::crypto::aws_lc_rs::cipher_suite::TLS13_CHACHA20_POLY1305_SHA256];
+        assert!(tls_handshake_with_provider(client_provider, &rustls::version::TLS13).is_err());
+    }
+
+    #[test]
+    fn provider_installation_isolated() {
+        const SCENARIO: &str = "RUSTFS_TEST_PROVIDER_SCENARIO";
+        if let Ok(scenario) = std::env::var(SCENARIO) {
+            let incompatible = || {
+                let mut provider = build_default_crypto_provider();
+                provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
+                assert!(!provider.fips());
+                provider
+            };
+            match scenario.as_str() {
+                "fresh" => {}
+                "compatible" => {
+                    build_default_crypto_provider().install_default().expect("install provider");
+                }
+                "incompatible" => {
+                    incompatible().install_default().expect("install provider");
+                }
+                "race" => {
+                    let barrier = Arc::new(std::sync::Barrier::new(2));
+                    let other_barrier = barrier.clone();
+                    let other = std::thread::spawn(move || {
+                        other_barrier.wait();
+                        let _ = incompatible().install_default();
+                    });
+                    barrier.wait();
+                    let result = install_default_crypto_provider();
+                    other.join().expect("provider installer thread");
+                    let actual_fips = rustls::crypto::CryptoProvider::get_default()
+                        .expect("provider installed")
+                        .fips();
+                    assert_eq!(result.is_ok(), !cfg!(feature = "fips") || actual_fips);
+                    assert_eq!(install_embedded_default_crypto_provider().is_ok(), !cfg!(feature = "fips") || actual_fips);
+                    return;
+                }
+                _ => panic!("unknown provider scenario"),
+            }
+            let result = install_default_crypto_provider();
+            let actual_fips = rustls::crypto::CryptoProvider::get_default()
+                .expect("provider installed")
+                .fips();
+            assert_eq!(result.is_ok(), !cfg!(feature = "fips") || actual_fips);
+            if cfg!(feature = "fips") && scenario == "incompatible" {
+                assert!(
+                    result
+                        .unwrap_err()
+                        .to_string()
+                        .contains("process-wide rustls crypto provider")
+                );
+                let runtime = tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .expect("embedded runtime");
+                let error = runtime
+                    .block_on(super::init_embedded_runtime_hooks(String::new()))
+                    .unwrap_err();
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+                let config = crate::config::Config::new("127.0.0.1:9000", Vec::new());
+                let report = rustfs_utils::ExternalEnvCompatReport::default();
+                let error = runtime
+                    .block_on(crate::startup_preflight::init_startup_server_preflight(&config, &report))
+                    .unwrap_err();
+                assert!(error.to_string().contains("process-wide rustls crypto provider"));
+                assert!(rustfs_obs::get_global_guard().is_err(), "observability must not initialize first");
+            }
+            assert_eq!(install_embedded_default_crypto_provider().is_ok(), !cfg!(feature = "fips") || actual_fips);
+            return;
+        }
+
+        for scenario in ["fresh", "compatible", "incompatible", "race"] {
+            let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
+                .arg("provider_installation_isolated")
+                .arg("--nocapture")
+                .env(SCENARIO, scenario)
+                .output()
+                .expect("start isolated test");
+            assert!(
+                output.status.success(),
+                "{scenario}: stdout={} stderr={}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
     }
 
     #[tokio::test]

rustfs/src/storage/rpc/http_service.rs

@@ -90,8 +90,83 @@ static PUT_FILE_AUTH_STRICT: LazyLock<bool> = LazyLock::new(|| {
         rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
     )
 });
-static PUT_FILE_TARGET_LOCKS: LazyLock<parking_lot::Mutex<HashMap<String, Weak<Mutex<()>>>>> =
+const PUT_FILE_TARGET_LOCK_SHARDS: usize = 64;
-    LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
+const PUT_FILE_TARGET_LOCK_INITIAL_GC_THRESHOLD: usize = 64;
+
+#[derive(Clone, Copy, Default)]
+struct Xxh3TargetKeyHasher(u64);
+
+impl std::hash::Hasher for Xxh3TargetKeyHasher {
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) {
+        self.0 = xxhash_rust::xxh3::xxh3_64(bytes);
+    }
+
+    #[inline]
+    fn write_usize(&mut self, _i: usize) {}
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.0
+    }
+}
+
+type Xxh3TargetKeyBuildHasher = std::hash::BuildHasherDefault<Xxh3TargetKeyHasher>;
+
+struct PutFileTargetLockShard {
+    entries: HashMap<Box<[u8]>, Weak<Mutex<()>>, Xxh3TargetKeyBuildHasher>,
+    next_gc_threshold: usize,
+}
+
+impl Default for PutFileTargetLockShard {
+    fn default() -> Self {
+        Self {
+            entries: HashMap::default(),
+            next_gc_threshold: PUT_FILE_TARGET_LOCK_INITIAL_GC_THRESHOLD,
+        }
+    }
+}
+
+struct ShardedPutFileTargetLockRegistry {
+    shards: [parking_lot::Mutex<PutFileTargetLockShard>; PUT_FILE_TARGET_LOCK_SHARDS],
+}
+
+impl ShardedPutFileTargetLockRegistry {
+    fn new() -> Self {
+        Self {
+            shards: std::array::from_fn(|_| parking_lot::Mutex::new(PutFileTargetLockShard::default())),
+        }
+    }
+
+    #[inline]
+    fn get_or_create(&self, key: &[u8]) -> Arc<Mutex<()>> {
+        let hash = xxhash_rust::xxh3::xxh3_64(key);
+        let shard_idx = ((hash >> 32) as usize) & (PUT_FILE_TARGET_LOCK_SHARDS - 1);
+        let mut shard = self.shards[shard_idx].lock();
+        if let Some(existing) = shard.entries.get(key).and_then(Weak::upgrade) {
+            return existing;
+        }
+        let created = Arc::new(Mutex::new(()));
+        let weak = Arc::downgrade(&created);
+        if let Some(slot) = shard.entries.get_mut(key) {
+            *slot = weak;
+            return created;
+        }
+        if shard.entries.len() >= shard.next_gc_threshold {
+            shard.entries.retain(|_, lock| lock.strong_count() > 0);
+            shard.next_gc_threshold = shard
+                .entries
+                .len()
+                .saturating_add(1)
+                .saturating_mul(2)
+                .max(PUT_FILE_TARGET_LOCK_INITIAL_GC_THRESHOLD);
+        }
+        shard.entries.insert(key.into(), weak);
+        created
+    }
+}
+
+static PUT_FILE_TARGET_LOCKS: LazyLock<ShardedPutFileTargetLockRegistry> = LazyLock::new(ShardedPutFileTargetLockRegistry::new);
 
 macro_rules! log_internode_rpc_response_failure {
     ($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{
@@ -1437,16 +1512,31 @@ where
 }
 
 fn put_file_target_lock(disk: &DiskStore, query: &PutFileQuery) -> Arc<Mutex<()>> {
-    let key = format!("{:p}\0{}\0{}", Arc::as_ptr(disk), query.volume, query.path);
+    let disk_addr = (Arc::as_ptr(disk) as usize).to_ne_bytes();
-    let mut locks = PUT_FILE_TARGET_LOCKS.lock();
+    let vol_bytes = query.volume.as_bytes();
-    locks.retain(|_, lock| lock.strong_count() > 0);
+    let vol_len = (vol_bytes.len() as u64).to_ne_bytes();
-    if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
+    let path_bytes = query.path.as_bytes();
-        return lock;
+    let total_len = disk_addr.len() + vol_len.len() + vol_bytes.len() + path_bytes.len();
+
+    let mut stack_buf = [0u8; 512];
+    if total_len <= stack_buf.len() {
+        let mut pos = 0;
+        stack_buf[pos..pos + disk_addr.len()].copy_from_slice(&disk_addr);
+        pos += disk_addr.len();
+        stack_buf[pos..pos + vol_len.len()].copy_from_slice(&vol_len);
+        pos += vol_len.len();
+        stack_buf[pos..pos + vol_bytes.len()].copy_from_slice(vol_bytes);
+        pos += vol_bytes.len();
+        stack_buf[pos..pos + path_bytes.len()].copy_from_slice(path_bytes);
+        return PUT_FILE_TARGET_LOCKS.get_or_create(&stack_buf[..total_len]);
     }
 
-    let lock = Arc::new(Mutex::new(()));
+    let mut heap_key = Vec::with_capacity(total_len);
-    locks.insert(key, Arc::downgrade(&lock));
+    heap_key.extend_from_slice(&disk_addr);
-    lock
+    heap_key.extend_from_slice(&vol_len);
+    heap_key.extend_from_slice(vol_bytes);
+    heap_key.extend_from_slice(path_bytes);
+    PUT_FILE_TARGET_LOCKS.get_or_create(&heap_key)
 }
 
 async fn remove_put_file_staging(disk: &DiskStore, volume: &str, path: &str) -> Result<(), BoxError> {
@@ -3747,4 +3837,51 @@ mod tests {
             assert_eq!(data, expected);
         }
     }
+
+    #[tokio::test]
+    async fn sharded_put_file_target_lock_separates_tuples_and_handles_long_keys() {
+        let (disk_a, _dir_a) = new_put_file_test_disk().await;
+        let (disk_b, _dir_b) = new_put_file_test_disk().await;
+
+        let make_query = |volume: &str, path: &str| PutFileQuery {
+            disk: "disk-a".to_string(),
+            volume: volume.to_string(),
+            path: path.to_string(),
+            append: false,
+            size: 0,
+            put_file_auth: None,
+            put_file_nonce: None,
+            put_file_server_epoch: None,
+        };
+
+        let q1 = make_query("ab", "c");
+        let q2 = make_query("a", "bc");
+        let lock_q1_a = put_file_target_lock(&disk_a, &q1);
+        let lock_q1_b = put_file_target_lock(&disk_a, &q1);
+        let lock_q2 = put_file_target_lock(&disk_a, &q2);
+        let lock_other_disk = put_file_target_lock(&disk_b, &q1);
+        assert!(std::sync::Arc::ptr_eq(&lock_q1_a, &lock_q1_b));
+        assert!(!std::sync::Arc::ptr_eq(&lock_q1_a, &lock_q2));
+        assert!(!std::sync::Arc::ptr_eq(&lock_q1_a, &lock_other_disk));
+
+        let long_path = "segment/".repeat(80);
+        let q_long = make_query("bucket", &long_path);
+        let lock_long_a = put_file_target_lock(&disk_a, &q_long);
+        let lock_long_b = put_file_target_lock(&disk_a, &q_long);
+        assert!(std::sync::Arc::ptr_eq(&lock_long_a, &lock_long_b));
+
+        let registry = super::ShardedPutFileTargetLockRegistry::new();
+        let held = registry.get_or_create(b"persistent-target");
+        drop(registry.get_or_create(b"reused-target"));
+        drop(registry.get_or_create(b"reused-target"));
+        for i in 0..(super::PUT_FILE_TARGET_LOCK_SHARDS * super::PUT_FILE_TARGET_LOCK_INITIAL_GC_THRESHOLD * 3) {
+            let key = format!("ephemeral-{i}");
+            drop(registry.get_or_create(key.as_bytes()));
+        }
+        assert!(std::sync::Arc::ptr_eq(&held, &registry.get_or_create(b"persistent-target")));
+        for shard in &registry.shards {
+            let guard = shard.lock();
+            assert!(guard.entries.len() <= guard.next_gc_threshold);
+        }
+    }
 }

rustfs/src/table_catalog/iceberg/commit.rs

@@ -44,45 +44,58 @@ pub(crate) fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitL
 pub(crate) struct TableCommitHistoryIndex<'a> {
     table_id: &'a str,
     reachable_states: BTreeSet<(&'a str, &'a str)>,
+    reachable_version_tokens: BTreeSet<&'a str>,
+    committed_successors: BTreeMap<(&'a str, &'a str), &'a CommitLogEntry>,
     ambiguous_states: BTreeSet<(&'a str, &'a str)>,
     cycle_detected: bool,
+    version_token_reused: bool,
 }
 
 impl<'a> TableCommitHistoryIndex<'a> {
     pub(crate) fn new(table: &'a TableEntry, commits: impl IntoIterator<Item = &'a CommitLogEntry>) -> Self {
-        let mut by_new_state = BTreeMap::<(&str, &str), Option<(&str, &str)>>::new();
+        let mut by_new_state = BTreeMap::<(&str, &str), Option<&CommitLogEntry>>::new();
         let mut ambiguous_states = BTreeSet::new();
         for commit in commits
             .into_iter()
             .filter(|commit| commit.table_id == table.table_id && !matches!(commit.status, CommitLogStatus::Failed))
         {
             let key = (commit.new_metadata_location.as_str(), commit.new_version_token.as_str());
-            let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str());
             by_new_state
                 .entry(key)
                 .and_modify(|candidate| {
                     *candidate = None;
                     ambiguous_states.insert(key);
                 })
-                .or_insert(Some(previous));
+                .or_insert(Some(commit));
         }
 
         let mut reachable_states = BTreeSet::new();
+        let mut reachable_version_tokens = BTreeSet::new();
+        let mut committed_successors = BTreeMap::new();
+        let mut version_token_reused = false;
         let mut state = (table.metadata_location.as_str(), table.version_token.as_str());
         let cycle_detected = loop {
             if !reachable_states.insert(state) {
                 break true;
             }
-            let Some(Some(previous)) = by_new_state.get(&state) else {
+            version_token_reused |= !reachable_version_tokens.insert(state.1);
+            let Some(commit) = by_new_state.get(&state).copied().flatten() else {
                 break false;
             };
-            state = *previous;
+            let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str());
+            if matches!(commit.status, CommitLogStatus::Committed) {
+                committed_successors.insert(previous, commit);
+            }
+            state = previous;
         };
         Self {
             table_id: &table.table_id,
             reachable_states,
+            reachable_version_tokens,
+            committed_successors,
             ambiguous_states,
             cycle_detected,
+            version_token_reused,
         }
     }
 
@@ -97,6 +110,49 @@ impl<'a> TableCommitHistoryIndex<'a> {
                 .reachable_states
                 .contains(&(target.new_metadata_location.as_str(), target.new_version_token.as_str()))
     }
+
+    fn proven_superseding_commit(&self, target: &CommitLogEntry) -> Option<&CommitLogEntry> {
+        if self.cycle_detected
+            || self.version_token_reused
+            || !self.ambiguous_states.is_empty()
+            || self.table_id != target.table_id
+            || !matches!(target.status, CommitLogStatus::Staged)
+            || self.reachable_version_tokens.contains(target.new_version_token.as_str())
+        {
+            return None;
+        }
+        let base = (target.previous_metadata_location.as_str(), target.expected_version_token.as_str());
+        self.committed_successors
+            .get(&base)
+            .copied()
+            .filter(|winner| winner.commit_id != target.commit_id)
+    }
+}
+
+/// Classifies an object-backed staged loser without rewriting its receipt.
+///
+/// A reachable finalized successor from the same pointer/token base proves that
+/// the old request lost CAS. Its STAGED record and idempotency binding remain
+/// available, and replay of that old request must still conflict. Missing or
+/// contradictory proof remains a manual-review state.
+pub(crate) fn table_commit_recovery_entry_from_history(
+    table: &TableEntry,
+    commit_log: &CommitLogEntry,
+    idempotency_commit: Option<&CommitLogEntry>,
+    history: &TableCommitHistoryIndex<'_>,
+) -> TableCommitRecoveryEntry {
+    let mut entry = table_commit_recovery_entry(table, commit_log, idempotency_commit, history.proves_committed(commit_log));
+    if entry.recovery_state == TableCommitRecoveryState::ManualReview
+        && matches!(
+            entry.idempotency_index_status,
+            TableCommitIdempotencyIndexStatus::NotRequired | TableCommitIdempotencyIndexStatus::Matches
+        )
+        && let Some(winner) = history.proven_superseding_commit(commit_log)
+    {
+        entry.recovery_state = TableCommitRecoveryState::Superseded;
+        entry.reason = format!("staged request lost CAS to reachable committed successor {}", winner.commit_id);
+    }
+    entry
 }
 
 pub(crate) fn table_catalog_recovery_summary(

rustfs/src/table_catalog/model.rs

@@ -1415,6 +1415,7 @@ pub(crate) struct TableCatalogDiagnosticsReport {
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub(crate) enum TableCommitRecoveryState {
     Committed,
+    Superseded,
     StagedBeforeTableUpdate,
     FinalizationRequired,
     IdempotencyIndexRepairRequired,

rustfs/src/table_catalog/store/object.rs

@@ -3065,11 +3065,11 @@ where
                 }
                 None => None,
             };
-            commits.push(table_commit_recovery_entry(
+            commits.push(table_commit_recovery_entry_from_history(
                 entry,
                 commit_log,
                 idempotency_commit.as_ref(),
-                history.proves_committed(commit_log),
+                &history,
             ));
         }
         commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id));
@@ -3168,12 +3168,8 @@ where
                 }
                 None => None,
             };
-            let recovery_entry = table_commit_recovery_entry(
+            let recovery_entry =
-                &entry,
+                table_commit_recovery_entry_from_history(&entry, commit_log, idempotency_commit.as_ref(), &history);
-                commit_log,
-                idempotency_commit.as_ref(),
-                history.proves_committed(commit_log),
-            );
             if matches!(
                 recovery_entry.recovery_state,
                 TableCommitRecoveryState::FinalizationRequired | TableCommitRecoveryState::IdempotencyIndexRepairRequired

rustfs/src/table_catalog/tests.rs

@@ -20,6 +20,7 @@ use std::sync::Arc;
 
 mod strong_maintenance_planning;
 mod strong_maintenance_planning_paths;
+mod superseded_commit_recovery;
 
 const TABLE_CATALOG_TEST_TIMEOUT: StdDuration = StdDuration::from_secs(30);
 

rustfs/src/table_catalog/tests/superseded_commit_recovery.rs

@@ -0,0 +1,487 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+struct StagedLoserFixture {
+    backend: TestCatalogObjectBackend,
+    store: ObjectTableCatalogStore<TestCatalogObjectBackend>,
+    loser: TableCommitRequest,
+    winner: TableCommitRequest,
+    committed: TableCommitResult,
+}
+
+async fn staged_loser_fixture(interrupt_winner_finalization: bool) -> StagedLoserFixture {
+    let backend = TestCatalogObjectBackend::default();
+    let store = ObjectTableCatalogStore::new(backend.clone());
+    let bucket = "analytics";
+    let namespace = Namespace::parse("sales").unwrap();
+    let table = IdentifierSegment::parse("orders").unwrap();
+    let initial = test_table_entry(
+        bucket,
+        &namespace,
+        &table,
+        default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
+    );
+    store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
+    store
+        .create_namespace(test_namespace_entry(bucket, &namespace))
+        .await
+        .unwrap();
+    store.create_table(initial.clone()).await.unwrap();
+    for name in ["00001", "loser", "winner", "reloaded"] {
+        let location = default_table_metadata_file_path(&namespace, &table, &format!("{name}.metadata.json"));
+        let mut metadata = super::super::test_support::table_metadata_json(&initial.table_uuid, &initial.warehouse_location);
+        metadata["properties"] = serde_json::json!({"writer": name});
+        backend
+            .seed_object(bucket, &location, serde_json::to_vec(&metadata).unwrap())
+            .await;
+    }
+    let paths = TableCatalogObjectPaths::default();
+    let table_path = paths.table_entry_path(bucket, &namespace, &table);
+    backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
+    let loser = TableCommitRequest {
+        table_bucket: bucket.to_string(),
+        namespace: namespace.public_name(),
+        table: table.as_str().to_string(),
+        commit_id: "paused-writer-a0".to_string(),
+        idempotency_key: Some("a0-request".to_string()),
+        operation: "set-properties".to_string(),
+        expected_version_token: initial.version_token.clone(),
+        expected_metadata_location: initial.metadata_location.clone(),
+        new_metadata_location: default_table_metadata_file_path(&namespace, &table, "loser.metadata.json"),
+        requirements: Vec::new(),
+        writer: None,
+    };
+    assert_matches!(store.commit_table(loser.clone()).await, Err(TableCatalogStoreError::Internal(_)));
+    let winner = TableCommitRequest {
+        commit_id: "winner-c1".to_string(),
+        idempotency_key: Some("c1-request".to_string()),
+        new_metadata_location: default_table_metadata_file_path(&namespace, &table, "winner.metadata.json"),
+        ..loser.clone()
+    };
+    if interrupt_winner_finalization {
+        let winner_log = paths.commit_log_entry_path(bucket, &initial.table_id, &winner.commit_id);
+        backend.fail_put_attempt(RUSTFS_META_BUCKET, &winner_log, 2).await;
+    }
+    let committed = store.commit_table(winner.clone()).await.unwrap();
+    StagedLoserFixture {
+        backend,
+        store,
+        loser,
+        winner,
+        committed,
+    }
+}
+
+#[tokio::test]
+async fn superseded_staged_commit_recovers_without_rewriting_receipts_or_pointer() {
+    let StagedLoserFixture {
+        backend,
+        store,
+        loser,
+        winner,
+        committed,
+    } = staged_loser_fixture(false).await;
+    let bucket = loser.table_bucket.as_str();
+    let before = backend.state.lock().await.objects.clone();
+
+    let recovered = store.recover_table_commits(bucket, "sales", "orders").await.unwrap();
+    assert_eq!(recovered.manual_review_count, 0, "a durable same-base winner proves the staged loser");
+    assert_eq!(recovered.finalized_count, 0, "a losing writer must never be finalized as committed");
+    let lost = recovered
+        .commits
+        .iter()
+        .find(|entry| entry.commit_id == loser.commit_id)
+        .unwrap();
+    assert_eq!(lost.status, CommitLogStatus::Staged);
+    assert_eq!(serde_json::to_value(&lost.recovery_state).unwrap(), "SUPERSEDED");
+    assert_eq!(lost.idempotency_index_status, TableCommitIdempotencyIndexStatus::Matches);
+    let after = backend.state.lock().await.objects.clone();
+    assert_eq!(after.len(), before.len());
+    for (path, original) in &before {
+        let actual = after.get(path).unwrap();
+        assert_eq!(actual.data, original.data, "recovery changed bytes at {path:?}");
+        assert_eq!(actual.etag, original.etag, "recovery rewrote {path:?}");
+    }
+
+    let restarted = ObjectTableCatalogStore::new(backend.clone());
+    let diagnostics = restarted.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap();
+    assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Healthy);
+    assert_matches!(restarted.commit_table(loser.clone()).await, Err(TableCatalogStoreError::Conflict(_)));
+    let mut stale_request = loser.clone();
+    stale_request.commit_id = "stale-new-request".to_string();
+    stale_request.idempotency_key = Some("stale-new-key".to_string());
+    assert_matches!(restarted.commit_table(stale_request).await, Err(TableCatalogStoreError::Conflict(_)));
+    let mut different_payload = loser.clone();
+    different_payload.operation = "replace".to_string();
+    assert_matches!(
+        restarted.commit_table(different_payload.clone()).await,
+        Err(TableCatalogStoreError::Conflict(_))
+    );
+    assert_eq!(
+        restarted.load_table(bucket, "sales", "orders").await.unwrap(),
+        Some(committed.table.clone())
+    );
+    assert_eq!(restarted.commit_table(winner).await.unwrap().table, committed.table);
+
+    let reloaded = TableCommitRequest {
+        commit_id: "reloaded-writer-a1".to_string(),
+        idempotency_key: Some("a1-request".to_string()),
+        expected_version_token: committed.table.version_token.clone(),
+        expected_metadata_location: committed.table.metadata_location.clone(),
+        new_metadata_location: default_table_metadata_file_path(
+            &Namespace::parse("sales").unwrap(),
+            &IdentifierSegment::parse("orders").unwrap(),
+            "reloaded.metadata.json",
+        ),
+        ..different_payload
+    };
+    let next = restarted.commit_table(reloaded.clone()).await.unwrap();
+    assert_eq!(next.table.generation, committed.table.generation + 1);
+    assert_eq!(restarted.commit_table(reloaded).await.unwrap().table, next.table);
+    let restarted = ObjectTableCatalogStore::new(backend);
+    assert_eq!(
+        restarted
+            .diagnose_table_catalog(bucket, "sales", "orders", 0)
+            .await
+            .unwrap()
+            .recovery_status,
+        TableCatalogRecoveryStatus::Healthy
+    );
+}
+
+#[tokio::test]
+async fn superseded_staged_commit_waits_for_durable_winner_finalization() {
+    let StagedLoserFixture {
+        backend,
+        store,
+        loser,
+        committed,
+        ..
+    } = staged_loser_fixture(true).await;
+    let paths = TableCatalogObjectPaths::default();
+    let log_path = paths.commit_log_entry_path(&loser.table_bucket, &committed.table.table_id, &loser.commit_id);
+    let before = backend.read_object(RUSTFS_META_BUCKET, &log_path).await.unwrap().unwrap();
+    let planned = store
+        .plan_table_commit_recovery("analytics", "sales", "orders")
+        .await
+        .unwrap();
+    assert_eq!(planned.manual_review_count, 1, "a merely staged winner is insufficient proof");
+    assert_eq!(planned.finalization_required_count, 1);
+
+    let recovered = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
+    assert_eq!(recovered.finalized_count, 1);
+    assert_eq!(recovered.manual_review_count, 0);
+    let loser_state = recovered
+        .commits
+        .iter()
+        .find(|entry| entry.commit_id == loser.commit_id)
+        .unwrap();
+    assert_eq!(loser_state.recovery_state, TableCommitRecoveryState::Superseded);
+    assert_eq!(loser_state.status, CommitLogStatus::Staged);
+    let after = backend.read_object(RUSTFS_META_BUCKET, &log_path).await.unwrap().unwrap();
+    assert_eq!(after.data, before.data);
+    assert_eq!(after.etag, before.etag);
+    assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(committed.table));
+    assert_eq!(
+        store
+            .diagnose_table_catalog("analytics", "sales", "orders", 0)
+            .await
+            .unwrap()
+            .recovery_status,
+        TableCatalogRecoveryStatus::Healthy
+    );
+}
+
+#[tokio::test]
+async fn superseded_staged_commit_requires_retained_winner_proof_after_restart() {
+    let StagedLoserFixture {
+        backend,
+        store,
+        loser,
+        winner,
+        committed,
+    } = staged_loser_fixture(false).await;
+    let healthy = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
+    assert_eq!(healthy.manual_review_count, 0);
+    let winner_path =
+        TableCatalogObjectPaths::default().commit_log_entry_path("analytics", &committed.table.table_id, &winner.commit_id);
+    assert!(
+        backend
+            .state
+            .lock()
+            .await
+            .objects
+            .remove(&(RUSTFS_META_BUCKET.to_string(), winner_path))
+            .is_some()
+    );
+    let restarted = ObjectTableCatalogStore::new(backend);
+    let before = restarted.load_table("analytics", "sales", "orders").await.unwrap();
+    let report = restarted.recover_table_commits("analytics", "sales", "orders").await.unwrap();
+    assert_eq!(report.manual_review_count, 1);
+    assert_eq!(report.finalized_count, 0);
+    assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::ManualReview);
+    assert_eq!(report.commits[0].status, CommitLogStatus::Staged);
+    assert_matches!(restarted.commit_table(loser).await, Err(TableCatalogStoreError::Conflict(_)));
+    assert_eq!(restarted.load_table("analytics", "sales", "orders").await.unwrap(), before);
+    let diagnostics = restarted
+        .diagnose_table_catalog("analytics", "sales", "orders", 0)
+        .await
+        .unwrap();
+    assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::ManualReviewRequired);
+    assert!(
+        diagnostics
+            .recommended_actions
+            .contains(&TableCatalogRecoveryAction::ReviewCommitLog)
+    );
+}
+
+#[tokio::test]
+async fn superseded_staged_commit_does_not_repair_a_conflicting_index() {
+    let StagedLoserFixture {
+        backend,
+        store,
+        loser,
+        committed,
+        ..
+    } = staged_loser_fixture(false).await;
+    let mut conflicting = store
+        .get_commit_by_id("analytics", &committed.table.table_id, &loser.commit_id)
+        .await
+        .unwrap()
+        .unwrap();
+    conflicting.new_version_token = "contradictory-token".to_string();
+    let path = TableCatalogObjectPaths::default().commit_idempotency_entry_path(
+        "analytics",
+        &committed.table.table_id,
+        loser.idempotency_key.as_deref().unwrap(),
+    );
+    backend
+        .seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&conflicting).unwrap())
+        .await;
+    let before = backend.state.lock().await.objects.clone();
+    let report = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
+    assert_eq!(report.manual_review_count, 1);
+    assert_eq!(report.finalized_count, 0);
+    let loser_state = report
+        .commits
+        .iter()
+        .find(|entry| entry.commit_id == loser.commit_id)
+        .unwrap();
+    assert_eq!(loser_state.recovery_state, TableCommitRecoveryState::ManualReview);
+    assert_eq!(loser_state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting);
+    let after = backend.state.lock().await.objects.clone();
+    assert_eq!(before.len(), after.len());
+    for (key, original) in &before {
+        let actual = after.get(key).unwrap();
+        assert_eq!(actual.data, original.data);
+        assert_eq!(actual.etag, original.etag);
+    }
+    assert_matches!(store.commit_table(loser).await, Err(TableCatalogStoreError::Conflict(_)));
+}
+
+fn history_fixture() -> (TableEntry, CommitLogEntry, CommitLogEntry) {
+    let namespace = Namespace::parse("sales").unwrap();
+    let table_name = IdentifierSegment::parse("orders").unwrap();
+    let metadata = |name| default_table_metadata_file_path(&namespace, &table_name, name);
+    let mut table = test_table_entry("analytics", &namespace, &table_name, metadata("winner.metadata.json"));
+    table.version_token = "winner-token".to_string();
+    table.generation = 2;
+    let loser = CommitLogEntry {
+        version: TABLE_CATALOG_ENTRY_VERSION,
+        commit_id: "loser".to_string(),
+        idempotency_key: None,
+        table_id: table.table_id.clone(),
+        operation: "set-properties".to_string(),
+        expected_version_token: "base-token".to_string(),
+        new_version_token: "loser-token".to_string(),
+        previous_metadata_location: metadata("base.metadata.json"),
+        new_metadata_location: metadata("loser.metadata.json"),
+        requirements: Vec::new(),
+        status: CommitLogStatus::Staged,
+        writer: None,
+        created_at: None,
+        updated_at: None,
+    };
+    let winner = CommitLogEntry {
+        commit_id: "winner".to_string(),
+        new_version_token: table.version_token.clone(),
+        new_metadata_location: table.metadata_location.clone(),
+        status: CommitLogStatus::Committed,
+        ..loser.clone()
+    };
+    (table, loser, winner)
+}
+
+fn classify_history(
+    table: &TableEntry,
+    target: &CommitLogEntry,
+    commits: &[CommitLogEntry],
+    indexed: Option<&CommitLogEntry>,
+) -> TableCommitRecoveryEntry {
+    let history = TableCommitHistoryIndex::new(table, commits.iter());
+    table_commit_recovery_entry_from_history(table, target, indexed, &history)
+}
+
+#[test]
+fn superseded_history_requires_a_reachable_finalized_same_base_winner() {
+    for case in [
+        "staged-winner",
+        "failed-winner",
+        "wrong-base-token",
+        "wrong-base-location",
+        "foreign-table",
+        "disconnected-winner",
+        "same-commit-id",
+    ] {
+        let (mut table, loser, mut winner) = history_fixture();
+        match case {
+            "staged-winner" => winner.status = CommitLogStatus::Staged,
+            "failed-winner" => winner.status = CommitLogStatus::Failed,
+            "wrong-base-token" => winner.expected_version_token = "other-base-token".to_string(),
+            "wrong-base-location" => winner.previous_metadata_location = "other/base.metadata.json".to_string(),
+            "foreign-table" => winner.table_id = "other-table-id".to_string(),
+            "disconnected-winner" => {
+                table.metadata_location = "other/current.metadata.json".to_string();
+                table.version_token = "other-current-token".to_string();
+            }
+            "same-commit-id" => winner.commit_id.clone_from(&loser.commit_id),
+            _ => unreachable!(),
+        }
+        let recovery = classify_history(&table, &loser, &[loser.clone(), winner], None);
+        assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview, "{case}");
+    }
+}
+
+#[test]
+fn superseded_history_is_stable_after_later_commits() {
+    let (mut table, loser, winner) = history_fixture();
+    let immediate = classify_history(&table, &loser, &[loser.clone(), winner.clone()], None);
+    assert_eq!(immediate.recovery_state, TableCommitRecoveryState::Superseded);
+    assert!(immediate.reason.contains(&winner.commit_id));
+    let later = CommitLogEntry {
+        commit_id: "later".to_string(),
+        previous_metadata_location: table.metadata_location.clone(),
+        expected_version_token: table.version_token.clone(),
+        new_metadata_location: "later/current.metadata.json".to_string(),
+        new_version_token: "later-token".to_string(),
+        ..winner.clone()
+    };
+    table.metadata_location.clone_from(&later.new_metadata_location);
+    table.version_token.clone_from(&later.new_version_token);
+    table.generation += 1;
+    let after = classify_history(&table, &loser, &[loser.clone(), winner, later], None);
+    assert_eq!(after.recovery_state, TableCommitRecoveryState::Superseded);
+    assert_eq!(after.reason, immediate.reason);
+}
+
+#[test]
+fn superseded_history_rejects_failed_or_committed_losers() {
+    for status in [CommitLogStatus::Failed, CommitLogStatus::Committed] {
+        let (table, mut loser, winner) = history_fixture();
+        loser.status = status;
+        let recovery = classify_history(&table, &loser, &[loser.clone(), winner], None);
+        assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview);
+    }
+}
+
+#[test]
+fn superseded_history_rejects_ambiguity_cycles_and_reused_tokens() {
+    for case in [
+        "ambiguous-current",
+        "cycle",
+        "reused-history-token",
+        "loser-token-in-history",
+        "off-history-ambiguity",
+    ] {
+        let (mut table, mut loser, winner) = history_fixture();
+        let mut extra = CommitLogEntry {
+            commit_id: "extra".to_string(),
+            ..winner.clone()
+        };
+        let mut commits = vec![loser.clone(), winner.clone()];
+        match case {
+            "ambiguous-current" => commits.push(extra),
+            "cycle" => {
+                extra.previous_metadata_location.clone_from(&winner.new_metadata_location);
+                extra.expected_version_token.clone_from(&winner.new_version_token);
+                extra.new_metadata_location.clone_from(&winner.previous_metadata_location);
+                extra.new_version_token.clone_from(&winner.expected_version_token);
+                commits.push(extra);
+            }
+            "reused-history-token" => {
+                extra.previous_metadata_location.clone_from(&winner.new_metadata_location);
+                extra.expected_version_token.clone_from(&winner.new_version_token);
+                extra.new_metadata_location = "later/current.metadata.json".to_string();
+                extra.new_version_token.clone_from(&winner.expected_version_token);
+                table.metadata_location.clone_from(&extra.new_metadata_location);
+                table.version_token.clone_from(&extra.new_version_token);
+                commits.push(extra);
+            }
+            "loser-token-in-history" => {
+                loser.new_version_token.clone_from(&winner.expected_version_token);
+                commits[0] = loser.clone();
+            }
+            "off-history-ambiguity" => {
+                extra.new_metadata_location = "unknown/metadata.json".to_string();
+                extra.new_version_token = "unknown-token".to_string();
+                commits.push(extra.clone());
+                extra.commit_id = "another-extra".to_string();
+                commits.push(extra);
+            }
+            _ => unreachable!(),
+        }
+        assert_eq!(
+            classify_history(&table, &loser, &commits, None).recovery_state,
+            TableCommitRecoveryState::ManualReview,
+            "{case}"
+        );
+    }
+}
+
+#[test]
+fn superseded_history_requires_an_intact_idempotency_binding() {
+    let (table, mut loser, winner) = history_fixture();
+    loser.idempotency_key = Some("loser-request".to_string());
+    let commits = [loser.clone(), winner];
+    assert_eq!(
+        classify_history(&table, &loser, &commits, Some(&loser)).recovery_state,
+        TableCommitRecoveryState::Superseded
+    );
+    let missing = classify_history(&table, &loser, &commits, None);
+    assert_eq!(missing.recovery_state, TableCommitRecoveryState::ManualReview);
+    assert_eq!(missing.idempotency_index_status, TableCommitIdempotencyIndexStatus::Missing);
+    let mut stale = loser.clone();
+    stale.status = CommitLogStatus::Committed;
+    let state = classify_history(&table, &loser, &commits, Some(&stale));
+    assert_eq!(state.recovery_state, TableCommitRecoveryState::ManualReview);
+    assert_eq!(state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Stale);
+    let mut conflicting = loser.clone();
+    conflicting.commit_id = "another-request".to_string();
+    let state = classify_history(&table, &loser, &commits, Some(&conflicting));
+    assert_eq!(state.recovery_state, TableCommitRecoveryState::ManualReview);
+    assert_eq!(state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting);
+}
+
+#[test]
+fn superseded_history_never_relabels_a_committed_staged_ancestor() {
+    let (table, mut ancestor, winner) = history_fixture();
+    ancestor.new_metadata_location.clone_from(&winner.previous_metadata_location);
+    ancestor.new_version_token.clone_from(&winner.expected_version_token);
+    ancestor.previous_metadata_location = "older/base.metadata.json".to_string();
+    ancestor.expected_version_token = "older-base-token".to_string();
+    let recovery = classify_history(&table, &ancestor, &[ancestor.clone(), winner], None);
+    assert_eq!(recovery.recovery_state, TableCommitRecoveryState::FinalizationRequired);
+}

rustfs/tests/connect_perf_object.rs

@@ -540,7 +540,7 @@ struct CleanupObservation {
     elapsed: Duration,
 }
 
-async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation {
+async fn observe_cleanup(replies: [CleanupReply; 2], measurement_delay: Duration) -> CleanupObservation {
     use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
 
     const PRIVATE_BODY: &str = "private-cleanup-response-body";
@@ -583,6 +583,12 @@ async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation {
                 .to_owned();
             let reply = if index < 2 {
                 assert!(request_line.starts_with("PUT "));
+                if index == 1 && !measurement_delay.is_zero() {
+                    // Model descheduling during the measured PUT without a wall-clock sleep.
+                    tokio::time::pause();
+                    tokio::time::advance(measurement_delay).await;
+                    tokio::time::resume();
+                }
                 CleanupReply::Status(200)
             } else {
                 assert!(request_line.starts_with("DELETE "));
@@ -616,6 +622,8 @@ async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation {
     )
     .expect("cleanup probe");
     let mut request = request(ObjectOperation::PutObject);
+    // Status fixtures must reach cleanup independently of short measurement deadlines.
+    request.duration = REAL_SERVER_MEASUREMENT_BUDGET;
     request.traffic_bytes = 4;
     let log = CleanupLog::default();
     let writer = log.clone();
@@ -707,7 +715,7 @@ async fn cleanup_diagnostics_identify_each_failed_http_stage_without_private_con
         ([CleanupReply::Status(204), CleanupReply::Status(403)], "bucket_delete", 403),
         ([CleanupReply::Status(204), CleanupReply::Status(404)], "bucket_delete", 404),
     ] {
-        let observed = observe_cleanup(replies).await;
+        let observed = observe_cleanup(replies, Duration::ZERO).await;
         assert_cleanup_failure(&observed, stage, "CleanupFailed", Some(status));
         assert_eq!(observed.deletes.len(), if stage == "object_delete" { 1 } else { 2 });
     }
@@ -720,7 +728,7 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() {
         ([CleanupReply::Disconnect, CleanupReply::Status(204)], "object_delete"),
         ([CleanupReply::Status(204), CleanupReply::Disconnect], "bucket_delete"),
     ] {
-        let observed = observe_cleanup(replies).await;
+        let observed = observe_cleanup(replies, Duration::ZERO).await;
         assert_cleanup_failure(&observed, stage, "ProtocolFailure", None);
     }
 }
@@ -728,13 +736,21 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() {
 #[tokio::test]
 async fn cleanup_accepts_missing_object_and_still_deletes_the_bucket() {
     let _guard = TEST_LOCK.lock().await;
-    let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)]).await;
+    for measurement_delay in [Duration::ZERO, Duration::from_secs(2)] {
-    assert_eq!(observed.measurement.result.outcome(), ObjectOutcome::Succeeded);
+        let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)], measurement_delay).await;
-    assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete);
+        assert_eq!(
-    assert_eq!(observed.deletes.len(), 2);
+            observed.measurement.result.outcome(),
-    assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1"));
+            ObjectOutcome::Succeeded,
-    assert!(observed.deletes[1].ends_with("/ HTTP/1.1"));
+            "target result: {:?}; cleanup diagnostics: {}",
-    assert!(!observed.log.contains("connect_object_cleanup_failed"));
+            observed.measurement.target,
+            observed.log
+        );
+        assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete);
+        assert_eq!(observed.deletes.len(), 2);
+        assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1"));
+        assert!(observed.deletes[1].ends_with("/ HTTP/1.1"));
+        assert!(!observed.log.contains("connect_object_cleanup_failed"));
+    }
 }
 
 #[tokio::test]
@@ -750,7 +766,7 @@ async fn cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_dead
             "bucket_delete",
         ),
     ] {
-        let observed = observe_cleanup(replies).await;
+        let observed = observe_cleanup(replies, Duration::ZERO).await;
         assert_cleanup_failure(&observed, stage, "TimedOut", None);
         assert!(observed.elapsed >= Duration::from_secs(5));
         assert!(observed.log.contains("remaining_ms=0"), "{}", observed.log);