chore(deps): bump golang.org/x/net from 0.54.0 to 0.55.0 in /scripts/compat/heal-outcome in the go_modules group across 1 directory (#7296)

chore(deps): bump golang.org/x/net

Bumps the go_modules group with 1 update in the /scripts/compat/heal-outcome directory: [golang.org/x/net](https://github.com/golang/net).


Updates `golang.org/x/net` from 0.54.0 to 0.55.0
- [Commits](https://github.com/golang/net/compare/v0.54.0...v0.55.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-version: 0.55.0
  dependency-type: indirect
  dependency-group: go_modules
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.config/e2e-distributed-selection.txt

@@ -0,0 +1,2 @@
+sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
+sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
+sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193
-sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
+sha256-linux=7c892afa4b9d1591b46bd79c976b647109a277284fddb3b98edced4b0297eda2

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
+sha256=5db88c6fec94d4f269c7d9cfc128bd2adc27b3d7021127e2fa0b1daccc5f900f

.config/ecstore-required-tests.json

@@ -40,6 +40,21 @@
       "invariant": "corrupt-part-arrays",
       "suite": "rustfs-filemeta",
       "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
+    },
+    {
+      "invariant": "odm-source-contract-s3",
+      "suite": "rustfs",
+      "name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract"
+    },
+    {
+      "invariant": "odm-source-contract-azure",
+      "suite": "rustfs",
+      "name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract"
+    },
+    {
+      "invariant": "odm-source-contract-gcs",
+      "suite": "rustfs",
+      "name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
     }
   ],
   "fixtures": [

.config/make/tests.mak

@@ -31,6 +31,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
+	./scripts/test_scanner_validation_harness.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh

.config/migration-gate-floor.txt

@@ -1,10 +1,11 @@
 # Committed floor for the number of tests selected by the migration-critical
 # CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12).
 #
-# The floor equals the exact count of rustfs-ecstore --lib tests matching the
+# The floor equals the exact count of rustfs-ecstore --lib tests, with the
-# gate filter (name substrings: data_movement, rebalance, decommission,
+# test-util feature enabled, matching the gate filter (name substrings:
-# source_cleanup, delete_marker) at the time this file was last updated.
+# data_movement, rebalance, decommission, source_cleanup, delete_marker) at
+# the time this file was last updated.
 # CI fails if the selected count drops below this number, so renames or
 # removals that thin the gate must update this file in the same PR.
 # Adding tests does not require a bump, but bumping keeps the guard tight.
-571
+946

.config/nextest.toml

@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -526,6 +533,27 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'
 
+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Storage-sensitive PR / nightly / dispatch lane owned by
+# .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly with no retries.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +614,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +639,7 @@ default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
     & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
     & !test(/^replication_extension_test::/)
     & !test(/^replication_target_matrix_test::/)
     & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)

.config/scanner-heal-required-tests.json

@@ -0,0 +1,40 @@
+{
+  "schema": 1,
+  "cases": {
+    "background-target-restart": {
+      "gate": "G14",
+      "task": "W21",
+      "lane": "e2e-nightly",
+      "suite": "e2e_test",
+      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
+      "oracle": "background-target-restart.json",
+      "min_objects": 9,
+      "max_objects": 65,
+      "topology": {"nodes": 4, "drives_per_node": 1},
+      "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
+    }
+  },
+  "release_pending": {
+    "G01": "W02/W04 complete root and quota authority coverage",
+    "G02": "W03 bounded checkpoint progress and independent version inventory",
+    "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
+    "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
+    "G05": "W06/W07 per-object outcomes and bounded terminal retention",
+    "G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
+    "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
+    "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
+    "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
+    "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
+    "G11": "W04/W19/W24 maintenance and complete producer coverage",
+    "G12": "W02/W15/W16 both quota paths during reset and settlement",
+    "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
+    "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
+    "P1": "W20 measured cold-walk share and foreground latency/throughput",
+    "P2": "W20/W24 measured post-stop convergence and cold segment reuse",
+    "P3": "W20 measured two-hour pressure/heal capacity and recovery window",
+    "P4": "W20 measured MRF scale and replay cost with retained responsibility",
+    "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
+    "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
+    "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
+  }
+}

.github/scheduled-validations.json

@@ -4,6 +4,11 @@
   { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
   { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },

.github/workflows/ci.yml

@@ -850,6 +850,11 @@ jobs:
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
 
+      - name: Install network fault-injection tools
+        run: |
+          sudo apt-get install -y iptables
+          sudo -n iptables --version
+
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:

.github/workflows/e2e-distributed.yml

@@ -0,0 +1,216 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+#
+# Isolated pool filesystems: expand/decommission/rebalance cases require
+# independent `statfs` capacity. This job runs on GitHub-hosted
+# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot
+# create filesystems: `mount -o loop` fails with ENOENT (no
+# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs
+# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason
+# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare
+# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`.
+
+name: e2e-distributed
+
+on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    # GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
+    # ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          # Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so
+          # a shared key would mix VM and ARC pod target/ artifacts.
+          cache-shared-key: ci-e2e-distributed-hosted
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Prepare isolated filesystems for pool movement
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          mkdir -p "${mount_base}"
+          roots=()
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            mkdir -p "${mountpoint}"
+            # Sized tmpfs reports a distinct st_dev and independent 1G
+            # statfs capacity. Requires a VM runner (ubuntu-latest).
+            if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then
+              echo "tmpfs mount failed on $(uname -a)" >&2
+              findmnt || true
+              grep Cap /proc/self/status || true
+              exit 1
+            fi
+            sudo chmod 1777 "${mountpoint}"
+            roots+=("${mountpoint}")
+          done
+          printf -v joined_roots '%s:' "${roots[@]}"
+          echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          FILTER: ${{ inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Unmount isolated pool filesystems
+        if: always()
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            if mountpoint --quiet "${mountpoint}"; then
+              sudo umount "${mountpoint}"
+            fi
+          done
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scheduled-validation-watchdog.yml

@@ -22,6 +22,7 @@ on:
       - "Continuous Integration"
       - "coverage"
       - "e2e-nightly"
+      - "e2e-distributed"
       - "e2e-s3tests"
       - "Fuzz"
       - "mint"

Cargo.lock

@@ -2527,18 +2527,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.16"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
+checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.7"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
 dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
@@ -2546,27 +2546,27 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.20"
+version = "0.9.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.13"
+version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
+checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.22"
+version = "0.8.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
 
 [[package]]
 name = "crunchy"
@@ -3673,9 +3673,9 @@ dependencies = [
 
 [[package]]
 name = "der"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
+checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
 dependencies = [
  "const-oid 0.10.2",
  "pem-rfc7468 1.0.0",
@@ -3946,7 +3946,7 @@ dependencies = [
  "libc",
  "option-ext",
  "redox_users 0.5.2",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4090,7 +4090,7 @@ version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
 dependencies = [
- "der 0.8.1",
+ "der 0.8.2",
  "digest 0.11.3",
  "elliptic-curve 0.14.1",
  "rfc6979 0.6.0",
@@ -4295,7 +4295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
 dependencies = [
  "io-lifetimes 3.0.1",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5734,9 +5734,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.12.1"
+version = "2.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
+checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
 dependencies = [
  "serde",
 ]
@@ -5758,7 +5758,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -6955,7 +6955,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -7933,7 +7933,7 @@ version = "0.8.0-rc.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
 dependencies = [
- "der 0.8.1",
+ "der 0.8.2",
  "spki 0.8.0",
 ]
 
@@ -7976,7 +7976,7 @@ dependencies = [
  "aes 0.9.3",
  "aes-gcm",
  "cbc 0.2.1",
- "der 0.8.1",
+ "der 0.8.2",
  "pbkdf2 0.13.0",
  "rand_core 0.10.1",
  "scrypt 0.12.0",
@@ -8000,7 +8000,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
 dependencies = [
- "der 0.8.1",
+ "der 0.8.2",
  "pkcs5 0.8.1",
  "rand_core 0.10.1",
  "spki 0.8.0",
@@ -8706,7 +8706,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -8942,9 +8942,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f"
+checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
 dependencies = [
  "arc-swap",
  "arcstr",
@@ -9326,7 +9326,7 @@ dependencies = [
  "curve25519-dalek 5.0.0",
  "data-encoding",
  "delegate",
- "der 0.8.1",
+ "der 0.8.2",
  "digest 0.11.3",
  "ecdsa 0.17.0",
  "ed25519-dalek 3.0.0",
@@ -10745,6 +10745,7 @@ dependencies = [
  "rustfs-data-usage",
  "rustfs-ecstore",
  "rustfs-filemeta",
+ "rustfs-heal",
  "rustfs-heal-contracts",
  "rustfs-lifecycle",
  "rustfs-lock",
@@ -11065,7 +11066,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -11148,7 +11149,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -11419,7 +11420,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
 dependencies = [
  "base16ct 1.0.0",
  "ctutils",
- "der 0.8.1",
+ "der 0.8.2",
  "hybrid-array",
  "subtle",
  "zeroize",
@@ -11993,7 +11994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f"
 dependencies = [
  "base64ct",
- "der 0.8.1",
+ "der 0.8.2",
 ]
 
 [[package]]
@@ -12405,10 +12406,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -13527,7 +13528,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]

Cargo.toml

@@ -256,10 +256,10 @@ clap = { version = "4.6.6" }
 const-str = { version = "1.1.0" }
 convert_case = "0.12.0"
 criterion = { version = "0.8" }
-crossbeam-queue = "0.3.13"
+crossbeam-queue = "0.3.14"
-crossbeam-channel = "0.5.16"
+crossbeam-channel = "0.5.17"
-crossbeam-deque = "0.8.7"
+crossbeam-deque = "0.8.8"
-crossbeam-utils = "0.8.22"
+crossbeam-utils = "0.8.23"
 datafusion = { default-features = false, version = "55.0.0" }
 derive_builder = "0.20.2"
 enumset = "1.1.14"
@@ -306,7 +306,7 @@ rustfs-erasure-codec = { version = "8.0.2" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.6.0" }
+redis = { version = "1.7.0" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }

crates/common/src/mrf_channel.rs

@@ -422,9 +422,9 @@ fn unix_now_ms() -> u64 {
         .unwrap_or(0)
 }
 
-/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
+/// Legacy, unverified repair notice. Its identity lacks kind, set scope,
-/// entries the journal no longer tracks (backlog#1894 axis B). The payload
+/// bucket incarnation and responsibility generation. Consumers must not use
-/// mirrors the intent identity so consumers match without re-parsing.
+/// it to discharge persisted repair responsibility.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct MrfRepairedEvent {
     pub bucket: Arc<str>,
@@ -439,8 +439,8 @@ const MRF_REPAIRED_EVENT_CAP: usize = 4096;
 
 static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
 
-/// Record that the MRF consumer landed a repair. Never blocks: the critical
+/// Record a legacy notification for compatibility. This is not an
-/// section is a deque push under a std mutex.
+/// acknowledgement of storage verification or durable repair completion.
 pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
     let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
     let Ok(mut events) = registry.lock() else {
@@ -515,6 +515,9 @@ mod tests {
         }
         coalescer_release(&key, Some(lease));
         let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
+        assert_ne!(lease, retry_lease);
+        coalescer_release(&key, Some(lease));
+        assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
         coalescer_release(&key, Some(retry_lease));
     }
 

crates/config/README.md

@@ -172,6 +172,14 @@ Drive timeout profile preset:
   - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
   - Then the profile-derived default (`default` or `high_latency`).
 
+## Admin peer probe timeout
+
+- `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS`
+  - total per-peer budget for the `server_info`/`storage_info` admin probe round; `server_info` may reconnect once and `storage_info` remains a single attempt.
+  - default is `10` seconds, preserving the previous two-attempt worst-case budget.
+  - values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
+  - the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
+
 ## Startup filesystem boundary policy
 
 - `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.

crates/config/src/constants/internode.rs

@@ -39,6 +39,15 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20;
 pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
 pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30;
 
+/// Total budget for one admin peer probe round, including any reconnect retry.
+///
+/// This is intentionally separate from the transport-level RPC timeout: admin
+/// probes may retry once, but the retry must consume the same round budget.
+pub const ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS: &str = "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS";
+pub const DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 10;
+pub const MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 60;
+const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS);
+
 // ── Client-side internode gRPC channel tuning (P0) ──
 // These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
 // on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
@@ -312,6 +321,7 @@ mod tests {
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20);
         assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30);
+        assert_eq!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS, 10);
         assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
     }
 
@@ -412,6 +422,7 @@ mod tests {
             "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
         );
         assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+        assert_eq!(ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS");
         assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
         assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
         assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");

crates/e2e_test/README.md

@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane

crates/e2e_test/build.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use std::path::Path;
+use std::process::Command;
+
+fn git(root: &Path, args: &[&str]) -> Option<String> {
+    let output = Command::new("git").args(args).current_dir(root).output().ok()?;
+    output
+        .status
+        .success()
+        .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
+}
+
+fn emit(name: &str, value: &str) {
+    let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
+    println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
+}
+
+fn main() {
+    let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
+    let root = Path::new(&manifest).join("../..");
+    // Cover dependency/common sources as well as this crate. HEAD/ref/index
+    // changes must refresh identity even when no Rust source mtime changes.
+    for path in [
+        "crates",
+        "rustfs",
+        "Cargo.toml",
+        "Cargo.lock",
+        "rust-toolchain.toml",
+        ".cargo",
+        ".config",
+    ] {
+        println!("cargo:rerun-if-changed={}", root.join(path).display());
+    }
+    let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
+    if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
+        git_paths.push(reference);
+    }
+    for path in git_paths {
+        if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
+            let path = Path::new(&path);
+            let path = if path.is_absolute() {
+                path.to_owned()
+            } else {
+                root.join(path)
+            };
+            if path.exists() {
+                println!("cargo:rerun-if-changed={}", path.display());
+            }
+        }
+    }
+    let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
+    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
+    let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
+    let mut features = std::env::vars()
+        .filter_map(|(key, _)| {
+            key.strip_prefix("CARGO_FEATURE_")
+                .map(|name| name.to_ascii_lowercase().replace('_', "-"))
+        })
+        .collect::<Vec<_>>();
+    features.sort();
+    emit("COMMIT", &revision);
+    emit("DIRTY", if dirty { "true" } else { "false" });
+    emit("LOCK", &lock);
+    emit("FEATURES", &features.join(","));
+    for name in ["TARGET", "PROFILE"] {
+        emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
+    }
+    println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
+    let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
+    let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
+    emit("RUSTFLAGS_HEX", &flags);
+}

crates/e2e_test/src/chaos.rs

@@ -55,18 +55,20 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
 /// A successful S3 GET only proves that a quorum can serve an object. Replacement
 /// tests need this lower-level record to prove that the rebuilt target holds the
 /// `xl.meta` selected for a specific version and every `part.N` it declares.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
 pub(crate) struct VersionShardCensus {
     pub version_id: Option<String>,
     pub has_xl_meta: bool,
     pub data_dir: Option<String>,
     pub erasure_index: Option<usize>,
+    pub data_blocks: Option<usize>,
+    pub parity_blocks: Option<usize>,
     pub expected_part_numbers: BTreeSet<usize>,
     pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
     pub inline_data_fingerprint: Option<PartShardFingerprint>,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
 pub(crate) struct PartShardFingerprint {
     pub size: u64,
     pub sha256: String,
@@ -88,13 +90,15 @@ impl VersionShardCensus {
             && manifest.is_complete()
             && self.data_dir == manifest.data_dir
             && self.erasure_index == manifest.erasure_index
+            && self.data_blocks == manifest.data_blocks
+            && self.parity_blocks == manifest.parity_blocks
             && self.expected_part_numbers == manifest.expected_part_numbers
             && self.present_part_fingerprints == manifest.present_part_fingerprints
             && self.inline_data_fingerprint == manifest.inline_data_fingerprint
     }
 }
 
-fn sha256_hex(data: &[u8]) -> String {
+pub(crate) fn sha256_hex(data: &[u8]) -> String {
     let digest = Sha256::digest(data);
     digest.iter().map(|byte| format!("{byte:02x}")).collect()
 }
@@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk(
             has_xl_meta: false,
             data_dir: None,
             erasure_index: None,
+            data_blocks: None,
+            parity_blocks: None,
             expected_part_numbers: BTreeSet::new(),
             present_part_fingerprints: BTreeMap::new(),
             inline_data_fingerprint: None,
@@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk(
         has_xl_meta: true,
         data_dir,
         erasure_index,
+        data_blocks: Some(file_info.erasure.data_blocks),
+        parity_blocks: Some(file_info.erasure.parity_blocks),
         expected_part_numbers,
         present_part_fingerprints,
         inline_data_fingerprint,
@@ -413,6 +421,8 @@ mod tests {
             has_xl_meta: true,
             data_dir: Some("data-dir".to_string()),
             erasure_index: Some(3),
+            data_blocks: Some(2),
+            parity_blocks: Some(2),
             expected_part_numbers: BTreeSet::from([1]),
             present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
             inline_data_fingerprint: None,

crates/e2e_test/src/common.rs

@@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment {
         Ok(())
     }
 
+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
+            self.extra_env
+                .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
+        }
+
+        Ok(new_idx)
+    }
+
     /// Gracefully stop one cluster node and wait for its process to exit.
     ///
     /// This is intentionally separate from [`Self::stop_node`]: the latter is

crates/e2e_test/src/data_usage_test.rs

@@ -35,11 +35,15 @@ where
 {
     let mut last_usage = DataUsageInfo::default();
     let mut last_query_error = None;
-    for _ in 0..45 {
+    for _ in 0..90 {
         match get_data_usage_info(env).await {
             Ok(usage) => {
                 last_query_error = None;
-                if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
+                if usage.is_complete_bucket_usage_snapshot()
+                    && usage.usage_snapshot_converged != Some(false)
+                    && usage.buckets_usage.contains_key(bucket)
+                    && predicate(&usage)
+                {
                     return Ok(usage);
                 }
                 last_usage = usage;

crates/e2e_test/src/distributed/chaos_test.rs

@@ -0,0 +1,222 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
+    wait_for_ready, wait_until,
+};
+use crate::chaos::{census_object_version_on_disk, signed_admin_post};
+use crate::common::{build_test_s3_config, init_logging};
+use crate::fault_proxy::FaultMode;
+use aws_sdk_s3::Client;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{Barrier, mpsc};
+use tokio::time::timeout;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
+    let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(
+        baseline.is_complete(),
+        "replacement target did not hold a complete baseline shard: {baseline:?}"
+    );
+    assert!(
+        !baseline.expected_part_numbers.is_empty(),
+        "replacement witness must use physical part shards: {baseline:?}"
+    );
+
+    dist.cluster.stop_node(0)?;
+    let format_path = replaced_drive.join(".rustfs.sys/format.json");
+    let format = std::fs::read(&format_path)?;
+    let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
+    std::fs::rename(&replaced_drive, &retired_drive)?;
+    std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
+    std::fs::write(&format_path, format)?;
+    let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
+
+    dist.cluster.start_node(0).await?;
+    wait_for_ready(&dist.cluster).await?;
+    let heal_body =
+        r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+    let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
+    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    wait_until(
+        Duration::from_secs(90),
+        || async {
+            let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+            Ok(healed.matches_manifest(&baseline))
+        },
+        "fresh replacement contains the original complete shard manifest",
+    )
+    .await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let worker_count = 12;
+    let release = Arc::new(Barrier::new(worker_count + 1));
+    let (started_tx, mut started_rx) = mpsc::unbounded_channel();
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let release = release.clone();
+        let started_tx = started_tx.clone();
+        handles.push(tokio::spawn(async move {
+            let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
+            if response.content_length() != Some(body.len() as i64) {
+                return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
+                    format!("worker {idx} received a wrong content length").into(),
+                );
+            }
+            started_tx.send(idx)?;
+            release.wait().await;
+            let actual = response.body.collect().await?.into_bytes();
+            if actual.as_ref() != body.as_slice() {
+                return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
+            }
+            Ok(())
+        }));
+    }
+    drop(started_tx);
+    for _ in 0..worker_count {
+        timeout(Duration::from_secs(30), started_rx.recv())
+            .await?
+            .ok_or("a streaming GET exited before reaching the kill barrier")?;
+    }
+
+    dist.cluster.stop_node(3)?;
+    release.wait().await;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
+    let proxied_url = format!("http://{}", proxy.local_addr());
+    let proxied_client = Client::from_conf(build_test_s3_config(
+        &proxied_url,
+        &dist.cluster.access_key,
+        &dist.cluster.secret_key,
+        None,
+        "distributed-network-chaos",
+    ));
+
+    let result: TestResult = async {
+        let bucket = unique_bucket("netfault");
+        dist.create_bucket(&bucket).await?;
+        let baseline = payload_for("network/baseline.bin", 1024 * 1024);
+        put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
+        assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
+
+        proxy.set_mode(FaultMode::Blackhole);
+        assert_eq!(proxy.mode(), FaultMode::Blackhole);
+        if let Ok(Ok(_)) = timeout(
+            Duration::from_secs(5),
+            proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
+        )
+        .await
+        {
+            return Err("blackholed node endpoint unexpectedly completed a GET".into());
+        }
+
+        let during = payload_for("network/during.bin", 1024 * 1024);
+        timeout(Duration::from_secs(30), async {
+            put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
+            assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
+            assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
+        })
+        .await??;
+
+        proxy.set_mode(FaultMode::Pass);
+        retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
+        Ok(())
+    }
+    .await;
+
+    proxy.set_mode(FaultMode::Pass);
+    proxy.shutdown().await;
+    result
+}

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -0,0 +1,98 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let worker_count = 24;
+    let rounds = 4;
+    let barrier = Arc::new(Barrier::new(worker_count));
+
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let writer = &clients[idx % clients.len()];
+            let reader = &clients[(idx + 1) % clients.len()];
+            let copier = &clients[(idx + 2) % clients.len()];
+            let mut retained = Vec::with_capacity(rounds);
+            for round in 0..rounds {
+                let key = format!("source/worker-{idx:02}-round-{round}.bin");
+                let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
+                let body = payload_for(&key, 64 * 1024);
+                put_object(writer, &bucket, &key, body.clone()).await?;
+
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                if head.content_length() != Some(body.len() as i64) {
+                    return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
+                }
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
+
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
+                if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
+                    return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
+                }
+                retained.push((copy_key, body));
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.extend(handle.await??);
+    }
+
+    let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
+    for (node_idx, client) in clients.iter().enumerate() {
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/concurrent_data_movement_test.rs

@@ -0,0 +1,74 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
+    unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(17));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    barrier.wait().await;
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let checker = dist.client(2)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/data_integrity_movement_test.rs

@@ -0,0 +1,156 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
+    sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    enable_versioning(&client, &bucket).await?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    let versioned_key = "history/versioned.bin";
+    let version_one = b"historical bytes before data movement".to_vec();
+    let version_two = b"current bytes before data movement".to_vec();
+    let version_one_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_one.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("historical PUT omitted version ID")?
+        .to_string();
+    let version_two_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_two.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("current PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "multipart/moved.bin";
+    let first_part = vec![0x31; 5 * 1024 * 1024];
+    let second_part = vec![0x72; 1024 * 1024];
+    let upload = client
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
+    let uploaded_one = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(first_part.clone()))
+        .send()
+        .await?;
+    let uploaded_two = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(2)
+        .body(ByteStream::from(second_part.clone()))
+        .send()
+        .await?;
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
+        let got = after_client
+            .get_object()
+            .bucket(&bucket)
+            .key(versioned_key)
+            .version_id(version_id)
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+        assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
+    }
+    let mut expected_multipart = first_part;
+    expected_multipart.extend_from_slice(&second_part);
+    let got_multipart = after_client
+        .get_object()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(
+        sha256_hex(got_multipart.as_ref()),
+        sha256_hex(&expected_multipart),
+        "multipart checksum changed after decommission"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs

@@ -0,0 +1,81 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
+    put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
+    wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    for expected_nodes in 2..=4 {
+        let new_node = dist.append_pool_and_restart().await?;
+        assert_eq!(new_node + 1, expected_nodes);
+        assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
+    }
+    assert_eq!(dist.cluster.nodes.len(), 4);
+
+    // Prove that the expanded pool map is durable, and clear any recovery
+    // latch raised while the newly-added pool replicas converged.
+    dist.restart_current_binary_gracefully().await?;
+
+    let after_expand = dist.client(0)?;
+    assert_inventory(&after_expand, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
+    assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after = dist.client(2)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/extra_test.rs

@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/mod.rs

@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;

crates/e2e_test/src/distributed/object_lock_test.rs

@@ -0,0 +1,219 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{
+    delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+    ObjectLockRule,
+};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+    let deleted_governance = peer
+        .head_object()
+        .bucket(&bucket)
+        .key(governance_key)
+        .version_id(&governance_version)
+        .send()
+        .await
+        .expect_err("GOVERNANCE bypass must remove the retained version");
+    assert_eq!(
+        deleted_governance.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
+    );
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let bucket = unique_bucket("default-lock");
+
+    writer
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+    put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
+
+    let key = "default-governance.bin";
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"default retention payload"))
+        .send()
+        .await?;
+    let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
+
+    let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
+    let default_retention = config
+        .object_lock_configuration()
+        .and_then(|configuration| configuration.rule())
+        .and_then(|rule| rule.default_retention())
+        .ok_or("GetObjectLockConfiguration omitted default retention")?;
+    assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    assert_eq!(default_retention.days(), Some(1));
+
+    let retention = reader
+        .get_object_retention()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
+    assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    let retain_until = retention
+        .retain_until_date()
+        .ok_or("default retention omitted retain-until date")?;
+    assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
+
+    let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
+    assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
+    expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
+        .await?;
+
+    let plain_bucket = unique_bucket("no-lock");
+    dist.create_bucket(&plain_bucket).await?;
+    let configuration = ObjectLockConfiguration::builder()
+        .object_lock_enabled(ObjectLockEnabled::Enabled)
+        .rule(
+            ObjectLockRule::builder()
+                .default_retention(
+                    DefaultRetention::builder()
+                        .mode(ObjectLockRetentionMode::Governance)
+                        .days(1)
+                        .build(),
+                )
+                .build(),
+        )
+        .build();
+    let error = writer
+        .put_object_lock_configuration()
+        .bucket(&plain_bucket)
+        .object_lock_configuration(configuration)
+        .send()
+        .await
+        .expect_err("an unversioned bucket must reject Object Lock enablement");
+    let service_error = error
+        .as_service_error()
+        .ok_or("non-lock bucket rejection was not an S3 service error")?;
+    assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
+    assert_eq!(
+        service_error.message(),
+        Some("Object Lock configuration cannot be enabled on existing buckets"),
+        "unexpected error: {error:?}"
+    );
+
+    Ok(())
+}

crates/e2e_test/src/distributed/observability_test.rs

@@ -0,0 +1,236 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
+use crate::common::{admin_request, init_logging, local_http_client};
+use aws_sdk_s3::operation::RequestId;
+use aws_sdk_s3::primitives::ByteStream;
+use bytes::Bytes;
+use http::Method;
+use http_body_util::{BodyExt, Empty};
+use hyper::body::Incoming;
+use hyper::service::service_fn;
+use hyper::{Request, Response};
+use hyper_util::rt::TokioIo;
+use local_ip_address::local_ip;
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
+use serde_json::Value;
+use std::convert::Infallible;
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, timeout};
+
+async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
+    let (tx, rx) = mpsc::unbounded_channel();
+    let handle = tokio::spawn(async move {
+        loop {
+            let Ok((stream, _)) = listener.accept().await else {
+                return;
+            };
+            let tx = tx.clone();
+            tokio::spawn(async move {
+                let service = service_fn(move |request: Request<Incoming>| {
+                    let tx = tx.clone();
+                    async move {
+                        let method = request.method().clone();
+                        if let Ok(body) = request.into_body().collect().await
+                            && method == Method::POST
+                            && let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
+                        {
+                            if let Some(records) = payload["Records"].as_array() {
+                                for entry in records {
+                                    let _ = tx.send(entry.clone());
+                                }
+                            } else {
+                                let _ = tx.send(payload);
+                            }
+                        }
+                        Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    Ok((endpoint, rx, handle))
+}
+
+async fn wait_for_audit_entry(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    bucket: &str,
+    key: &str,
+    request_id: &str,
+) -> TestResult<Value> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    let mut seen = Vec::new();
+    loop {
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            return Err(format!(
+                "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                seen.len()
+            )
+            .into());
+        }
+        let entry = match timeout(remaining, rx.recv()).await {
+            Ok(Some(entry)) => entry,
+            Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
+            Err(_) => {
+                return Err(format!(
+                    "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                    seen.len()
+                )
+                .into());
+            }
+        };
+        if entry["api"]["name"].as_str() == Some("s3:PutObject")
+            && entry["api"]["bucket"].as_str() == Some(bucket)
+            && entry["api"]["object"].as_str() == Some(key)
+            && entry["requestID"].as_str() == Some(request_id)
+        {
+            return Ok(entry);
+        }
+        if seen.len() < 8 {
+            seen.push(format!(
+                "api={:?} bucket={:?} object={:?} requestID={:?}",
+                entry["api"]["name"].as_str(),
+                entry["api"]["bucket"].as_str(),
+                entry["api"]["object"].as_str(),
+                entry["requestID"].as_str()
+            ));
+        }
+    }
+}
+
+#[tokio::test]
+async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
+    init_logging();
+    let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
+    let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
+    let audit_env = [
+        ("RUSTFS_AUDIT_ENABLE", "true"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
+        (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
+    ];
+    let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
+        tokio::fs::create_dir_all(&queue_dir).await?;
+        dist.cluster
+            .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        for probe in ["ready", "live"] {
+            let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
+            assert!(
+                response.status().is_success(),
+                "node {} {probe} probe failed: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    let info: Value = serde_json::from_str(&info_body)?;
+    let servers = info["info"]["servers"]
+        .as_array()
+        .ok_or_else(|| format!("admin info omitted servers: {info}"))?;
+    assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
+
+    let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: Value = serde_json::from_str(&storage_body)?;
+    let disks = storage["info"]["disks"]
+        .as_array()
+        .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
+    assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
+    assert!(
+        disks.iter().all(|disk| {
+            disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
+                && disk["runtimeState"]
+                    .as_str()
+                    .is_some_and(|state| state.eq_ignore_ascii_case("online"))
+        }),
+        "storageinfo reported a drive that was not healthy and online: {storage}"
+    );
+
+    for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
+        let (status, metrics_body) = admin_request(
+            &node.url,
+            Method::GET,
+            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            None,
+            &dist.cluster.access_key,
+            &dist.cluster.secret_key,
+        )
+        .await?;
+        assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
+        let sample: RealtimeMetrics = serde_json::from_str(
+            metrics_body
+                .lines()
+                .next()
+                .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
+        )?;
+        assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
+        assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
+        assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
+    }
+
+    let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let targets: Value = serde_json::from_str(&targets_body)?;
+    let configured = targets["audit_endpoints"]
+        .as_array()
+        .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
+        .iter()
+        .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
+    assert!(configured, "configured audit webhook was missing: {targets}");
+
+    let bucket = unique_bucket("audit");
+    dist.create_bucket(&bucket).await?;
+    let key = "correlated/audit-object.bin";
+    let put = dist
+        .client(2)?
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"distributed audit payload"))
+        .send()
+        .await?;
+    let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
+    let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
+    assert_eq!(
+        audit["api"]["status_code"].as_i64(),
+        Some(200),
+        "audit entry did not report success: {audit}"
+    );
+    assert!(
+        !audit.to_string().contains(&dist.cluster.secret_key),
+        "audit entry leaked the root secret key"
+    );
+
+    collector.abort();
+    Ok(())
+}

crates/e2e_test/src/distributed/replication_quota_test.rs

@@ -0,0 +1,191 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::time::Duration;
+
+async fn wait_for_replication_status(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    expected: &[&str],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            let head = client.head_object().bucket(bucket).key(key).send().await?;
+            Ok(head
+                .replication_status()
+                .is_some_and(|status| expected.contains(&status.as_str())))
+        },
+        &format!("replication status for {bucket}/{key} in {expected:?}"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, mut target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated/metadata-and-tags.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(key)
+        .metadata("origin", "four-node-source")
+        .tagging("suite=distributed&shape=metadata")
+        .body(ByteStream::from(body.clone()))
+        .send()
+        .await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
+    assert_eq!(
+        replica_head
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("four-node-source")
+    );
+    assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
+    let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
+    let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
+    assert_eq!(tags.get("suite"), Some(&"distributed"));
+    assert_eq!(tags.get("shape"), Some(&"metadata"));
+
+    target.cluster.stop();
+    let outage_key = "replicated/queued-during-target-outage.bin";
+    let outage_body = b"retry-after-target-restart".to_vec();
+    put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
+    wait_for_replication_status(
+        &source_client,
+        &source_bucket,
+        outage_key,
+        &["PENDING", "FAILED"],
+        Duration::from_secs(30),
+    )
+    .await?;
+
+    target.cluster.start().await?;
+    wait_for_ready(&target.cluster).await?;
+    wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value =
+                serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
+            let usage = stats
+                .get("current_usage")
+                .and_then(serde_json::Value::as_u64)
+                .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
+            Ok(usage >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let oversized_key = "too-big.bin";
+    let error = client
+        .put_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .body(vec![0u8; 16 * 1024].into())
+        .send()
+        .await
+        .expect_err("hard quota must reject the oversized PUT");
+    let service_error = error
+        .as_service_error()
+        .ok_or("quota rejection was not an S3 service error")?;
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(400),
+        "quota rejection must be HTTP 400: {error:?}"
+    );
+    assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
+    assert!(
+        service_error
+            .message()
+            .is_some_and(|message| message.starts_with("Bucket quota exceeded")),
+        "PUT must fail specifically at quota admission: {error:?}"
+    );
+
+    let missing = client
+        .head_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .send()
+        .await
+        .expect_err("an object rejected by quota must not become visible");
+    assert_eq!(
+        missing.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "quota-rejected object returned an unexpected HEAD result: {missing:?}"
+    );
+
+    let listed = client.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
+        "quota-rejected key leaked into ListObjectsV2"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/s3_basic_test.rs

@@ -0,0 +1,258 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    let deleted = writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
+    assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3matrix");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+
+    let special_key = "unicode/测试 space+percent%25.txt";
+    let special_body = b"metadata and tagging survive distributed routing".to_vec();
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .metadata("test-meta", "distributed")
+        .tagging("purpose=compatibility&scope=four-by-four")
+        .body(ByteStream::from(special_body.clone()))
+        .send()
+        .await?;
+    let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
+
+    let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
+    assert_eq!(
+        head.metadata()
+            .and_then(|metadata| metadata.get("test-meta"))
+            .map(String::as_str),
+        Some("distributed")
+    );
+    assert_eq!(head.e_tag(), Some(etag.as_str()));
+    let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
+    let actual_tags: std::collections::BTreeMap<_, _> = tags
+        .tag_set()
+        .iter()
+        .map(|tag| (tag.key().to_string(), tag.value().to_string()))
+        .collect();
+    assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
+    assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
+
+    let conditional = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .if_match(&etag)
+        .send()
+        .await?;
+    assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
+    let invalid_range = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .range("bytes=999999-1000000")
+        .send()
+        .await
+        .expect_err("an unsatisfiable range must fail");
+    assert_eq!(
+        invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidRange"),
+        "unexpected invalid-range error: {invalid_range:?}"
+    );
+
+    let upload_key = "multipart/aborted.bin";
+    let upload = writer
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
+    writer
+        .upload_part()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
+        .send()
+        .await?;
+    let pending = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
+    writer
+        .abort_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    let after_abort = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
+    let aborted_head = reader
+        .head_object()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await
+        .expect_err("aborted multipart upload must not create an object");
+    assert_eq!(
+        aborted_head.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
+    );
+
+    for index in 0..113 {
+        let key = format!("page/{index:04}.txt");
+        put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
+    }
+    let mut token = None;
+    let mut paged_keys = Vec::new();
+    loop {
+        let page = reader
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("page/")
+            .max_keys(37)
+            .set_continuation_token(token.take())
+            .send()
+            .await?;
+        paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
+        if page.is_truncated() != Some(true) {
+            break;
+        }
+        token = Some(
+            page.next_continuation_token()
+                .ok_or("truncated ListObjectsV2 page omitted next continuation token")?
+                .to_string(),
+        );
+    }
+    assert_eq!(paged_keys.len(), 113);
+    let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
+    assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
+    Ok(())
+}

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -0,0 +1,94 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
+    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let status = rebalance_status_json(&dist.cluster).await?;
+    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+        return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
+    }
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/site_replication_test.rs

@@ -0,0 +1,128 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+    wait_until,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
+use std::time::Duration;
+
+async fn site_replication_add(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    sites: &[PeerSite],
+) -> TestResult<ReplicateAddStatus> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let info = site_replication_info(cluster).await?;
+            Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
+        },
+        "site replication enabled with two synchronized sites",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "site replication add reported failure: {add_status:?}"
+    );
+    wait_for_site_replication_enabled(&site_a.cluster).await?;
+    wait_for_site_replication_enabled(&site_b.cluster).await?;
+
+    let info_a = site_replication_info(&site_a.cluster).await?;
+    let remote = info_a
+        .sites
+        .iter()
+        .find(|site| site.name == "site-b")
+        .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
+    assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
+    let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
+    assert!(
+        deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
+        "site peers must have two distinct non-empty deployment IDs: {info_a:?}"
+    );
+    assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
+    assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+
+    let reverse_key = "reverse/site-object.bin";
+    let reverse_body = b"site-b-to-site-a".to_vec();
+    put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
+    wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -0,0 +1,345 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            let rejected = code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+                || error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
+            if !rejected {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/versioning_test.rs

@@ -0,0 +1,188 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    let v1_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v1".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v1 PUT omitted version ID")?
+        .to_string();
+    let v2_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v2".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v2 PUT omitted version ID")?
+        .to_string();
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_versions: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
+    assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
+        "v1 was not the historical version: {versions:?}"
+    );
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
+        "v2 was not the latest version: {versions:?}"
+    );
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(deleted.delete_marker(), Some(true));
+    let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_markers: Vec<_> = after_delete
+        .delete_markers()
+        .iter()
+        .filter(|marker| marker.key() == Some(key))
+        .collect();
+    assert_eq!(
+        matching_markers.len(),
+        1,
+        "delete marker missing or duplicated after current-version delete: {after_delete:?}"
+    );
+    assert!(
+        matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
+        "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+
+    writer
+        .delete_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(marker_id)
+        .send()
+        .await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("suspend");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "suspended.txt";
+    let original = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"enabled-history".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("enabled PUT omitted version ID")?
+        .to_string();
+    writer
+        .put_bucket_versioning()
+        .bucket(&bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Suspended)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
+    let null_version_count = matching
+        .iter()
+        .filter(|version| {
+            matches!(
+                version.version_id(),
+                None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
+            )
+        })
+        .count();
+    assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
+
+    let historical = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(&original)
+        .send()
+        .await?;
+    assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
+    Ok(())
+}

crates/e2e_test/src/fake_s3_target/mod.rs

@@ -451,10 +451,40 @@ impl JournaledHeaders {
 struct ControlState {
     scripts: HashMap<Operation, VecDeque<FaultAction>>,
     keyed_scripts: HashMap<(Operation, String), VecDeque<FaultAction>>,
+    held_get: Option<HeldGetObject>,
     requests: VecDeque<RequestRecord>,
     next_sequence: u64,
 }
 
+#[derive(Clone)]
+struct HeldGetObject {
+    bucket: String,
+    key: String,
+    entered: watch::Sender<usize>,
+    released: watch::Receiver<bool>,
+}
+
+/// Holds every GET of one object, including retries, until this guard is dropped.
+#[must_use = "dropping the guard releases the held GET requests"]
+pub struct GetObjectGate {
+    control: Arc<Mutex<ControlState>>,
+    entered: watch::Receiver<usize>,
+    released: watch::Sender<bool>,
+}
+
+impl GetObjectGate {
+    pub async fn wait_until_entered(&mut self) -> Result<(), watch::error::RecvError> {
+        self.entered.wait_for(|count| *count > 0).await.map(|_| ())
+    }
+}
+
+impl Drop for GetObjectGate {
+    fn drop(&mut self) {
+        lock(&self.control).held_get = None;
+        self.released.send_replace(true);
+    }
+}
+
 #[derive(Default)]
 struct StoreState {
     assign_own_version_ids: bool,
@@ -936,6 +966,30 @@ impl FakeS3Target {
             .extend(std::iter::repeat_n(action, times));
     }
 
+    /// Hold one exact bucket/key before any GET response can reach the client.
+    /// The fixture supports one live gate; request and connection deadlines still apply.
+    pub fn hold_get_object(&self, bucket: &str, key: &str) -> GetObjectGate {
+        assert!(
+            bucket.len() <= MAX_RETAINED_IDENTIFIER_BYTES && key.len() <= MAX_RETAINED_IDENTIFIER_BYTES,
+            "held GET identifiers exceed the fixture limit"
+        );
+        let mut state = lock(&self.control);
+        assert!(state.held_get.is_none(), "fake target already holds a GET gate");
+        let (entered, entered_rx) = watch::channel(0);
+        let (released, released_rx) = watch::channel(false);
+        state.held_get = Some(HeldGetObject {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            entered,
+            released: released_rx,
+        });
+        GetObjectGate {
+            control: Arc::clone(&self.control),
+            entered: entered_rx,
+            released,
+        }
+    }
+
     pub fn clear_faults(&self) {
         let mut state = lock(&self.control);
         state.scripts.clear();
@@ -2243,6 +2297,17 @@ impl S3 for FakeBackend {
         let fault = request_fault(&req);
         apply_non_body_fault(fault.as_ref(), &self.control).await?;
         let input = req.input;
+        let held_get = lock(&self.control)
+            .held_get
+            .as_ref()
+            .filter(|held| held.bucket == input.bucket && held.key == input.key)
+            .cloned();
+        if let Some(mut held) = held_get {
+            held.entered.send_modify(|count| *count += 1);
+            // Keep the gate installed when a request is cancelled or times out:
+            // a retry must cross the same boundary before returning any bytes.
+            let _ = held.released.wait_for(|released| *released).await;
+        }
         let (version, versioned) = {
             let state = lock(&self.store);
             (
@@ -2894,6 +2959,81 @@ mod tests {
         aws_sdk_s3::primitives::DateTime::from_secs(4_102_444_800)
     }
 
+    #[tokio::test]
+    async fn get_object_gate_holds_retries_and_releases_on_drop() -> Result<(), BoxError> {
+        let target = FakeS3Target::start().await?;
+        let bucket = "gated-target";
+        target.create_bucket(bucket);
+        for key in ["held", "unrelated"] {
+            target.put_seed_object(bucket, key, Bytes::from_static(b"payload"), &SeedMetadata::default());
+        }
+        {
+            let gate = target.hold_get_object(bucket, "held");
+            let request = || S3Request {
+                input: GetObjectInput {
+                    bucket: bucket.to_string(),
+                    key: "held".to_string(),
+                    ..Default::default()
+                },
+                method: Method::GET,
+                uri: Uri::from_static("/gated-target/held"),
+                headers: HeaderMap::new(),
+                extensions: http::Extensions::new(),
+                credentials: None,
+                region: None,
+                service: None,
+                trailing_headers: None,
+            };
+            // Without a fault, only the gate can suspend this backend method.
+            let mut first = target.backend.get_object(request());
+            assert!(futures::poll!(first.as_mut()).is_pending(), "the first GET must wait at the gate");
+            drop(first);
+            let mut retry = target.backend.get_object(request());
+            assert!(futures::poll!(retry.as_mut()).is_pending(), "a cancelled GET must not consume the gate");
+            drop(gate);
+            let std::task::Poll::Ready(response) = futures::poll!(retry.as_mut()) else {
+                panic!("dropping the gate must release the waiting GET");
+            };
+            let mut body = response?.output.body.expect("released GET body");
+            assert_eq!(body.next().await.transpose()?, Some(Bytes::from_static(b"payload")));
+            assert!(body.next().await.is_none(), "released GET body must be complete");
+        }
+        let client = client(&target);
+        let mut gate = target.hold_get_object(bucket, "held");
+        let mut requests = tokio::task::JoinSet::new();
+        let first = client.clone();
+        requests.spawn(async move { get_bytes(&first, bucket, "held", None).await });
+        timeout(Duration::from_secs(2), gate.wait_until_entered()).await??;
+        requests.abort_all();
+        assert!(
+            requests
+                .join_next()
+                .await
+                .expect("first GET task")
+                .expect_err("cancel the first GET attempt")
+                .is_cancelled()
+        );
+
+        let retry = client.clone();
+        requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
+        timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
+        assert_eq!(
+            timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
+            Bytes::from_static(b"payload")
+        );
+        assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
+        drop(gate);
+        assert_eq!(
+            timeout(Duration::from_secs(2), requests.join_next())
+                .await?
+                .expect("retried GET task")??,
+            Bytes::from_static(b"payload")
+        );
+        assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
+        assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
+        Ok(())
+    }
+
     #[tokio::test]
     async fn object_lock_target_requires_a_checksum_on_locked_puts() -> Result<(), BoxError> {
         use aws_sdk_s3::error::ProvideErrorMetadata;

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -16,15 +16,18 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
+    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
     use crate::common::{
         FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
+        rustfs_binary_path,
     };
     use crate::storage_api::RUSTFS_META_BUCKET;
     use aws_sdk_s3::primitives::ByteStream;
     use http::Method;
+    use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
+    use std::io::{Read, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -34,6 +37,76 @@ mod tests {
 
     const POOL_METADATA_OBJECT: &str = "pool.bin";
 
+    #[derive(serde::Deserialize)]
+    struct EvidenceBuild {
+        sha256: String,
+    }
+
+    #[derive(serde::Deserialize)]
+    struct RestartEvidenceRun {
+        schema: u32,
+        run_id: String,
+        source_revision: String,
+        test_build: serde_json::Value,
+        binary: EvidenceBuild,
+        test_binary: EvidenceBuild,
+    }
+
+    fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let mut file = std::fs::File::open(path)?;
+        let mut digest = Sha256::new();
+        let mut buffer = [0_u8; 64 * 1024];
+        loop {
+            let read = file.read(&mut buffer)?;
+            if read == 0 {
+                break;
+            }
+            digest.update(&buffer[..read]);
+        }
+        Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
+    }
+
+    fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
+        let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
+            return Ok(None);
+        };
+        let directory = PathBuf::from(directory);
+        let receipt = directory.join("run.json");
+        if receipt.metadata()?.len() > 1024 * 1024 {
+            return Err("oversized scanner/heal execution receipt".into());
+        }
+        let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
+        if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
+            return Err("invalid scanner/heal execution identity".into());
+        }
+        let built = compiled_test_identity();
+        for key in ["source_revision", "dirty", "lock_blob", "features"] {
+            assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
+        }
+        assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
+        assert_eq!(
+            file_sha256(&std::env::current_exe()?)?,
+            run.test_binary.sha256,
+            "test executable must match the run receipt"
+        );
+        if directory.join("background-target-restart.json").exists() {
+            return Err("scanner/heal oracle already exists; create a new execution receipt".into());
+        }
+        Ok(Some((directory, run)))
+    }
+
+    fn compiled_test_identity() -> serde_json::Value {
+        serde_json::json!({
+            "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
+            "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
+            "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
+            "features": env!("RUSTFS_E2E_BUILD_FEATURES"),
+            "target": env!("RUSTFS_E2E_BUILD_TARGET"),
+            "profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
+            "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
+        })
+    }
+
     struct TcpPortBlackhole {
         port: u16,
         comment: String,
@@ -195,8 +268,9 @@ mod tests {
         clients: &[aws_sdk_s3::Client],
         bucket: &str,
         expected_keys: &HashSet<String>,
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+    ) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
         const PAGE_SIZE: i32 = 10;
+        let mut node_listings = Vec::with_capacity(clients.len());
         for (node_index, client) in clients.iter().enumerate() {
             let mut listed_keys = Vec::new();
             let mut continuation_token = None;
@@ -243,8 +317,10 @@ mod tests {
                 &listed_key_set, expected_keys,
                 "node {node_index} did not expose the complete recovered namespace"
             );
+            listed_keys.sort();
+            node_listings.push(listed_keys);
         }
-        Ok(())
+        Ok(node_listings)
     }
 
     fn heal_task_status_diagnostic(body: &str) -> String {
@@ -808,6 +884,13 @@ mod tests {
     }
 
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let server_binary = rustfs_binary_path();
+        let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
+            restart_evidence_run(&server_binary)?
+        } else {
+            None
+        };
+        let mut evidence_objects = Vec::new();
         let (background_enabled, interruption_node, interruption_kind) = match scenario {
             InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
             InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
@@ -855,7 +938,7 @@ mod tests {
         for node_index in 0..cluster.nodes.len() {
             cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
         }
-        cluster.start().await?;
+        cluster.start_with_binary(&server_binary).await?;
         let clients = cluster.create_all_clients()?;
 
         let bucket = "heal-restart-during-rebuild";
@@ -996,7 +1079,7 @@ mod tests {
             }
         }
 
-        cluster.start_node(1).await?;
+        cluster.start_node_from_binary(1, &server_binary).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
         let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1274,7 +1357,7 @@ mod tests {
                     }
                 }
             }
-            cluster.start_node(interruption_node).await?;
+            cluster.start_node_from_binary(interruption_node, &server_binary).await?;
             if interruption_node == 0 {
                 let target = cluster.nodes[1]
                     .process
@@ -1373,7 +1456,7 @@ mod tests {
             .map(|manifest| manifest.key.clone())
             .collect::<HashSet<_>>();
         assert!(expected_keys.insert(outage_key.to_string()));
-        assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
+        let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
 
         let target_client = cluster.create_s3_client(1)?;
         for expected in &expected_manifests {
@@ -1381,11 +1464,31 @@ mod tests {
             let actual = response.body.collect().await?.into_bytes();
             let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
             assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
+            if evidence_run.is_some() {
+                evidence_objects.push(serde_json::json!({
+                    "key": expected.key, "version_id": expected.shard_census.version_id,
+                    "expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
+                    "expected_sha256": sha256_hex(&expected_body),
+                    "actual_sha256": sha256_hex(&actual),
+                    "expected_physical": expected.shard_census,
+                    "physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
+                }));
+            }
         }
         let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
         let actual = response.body.collect().await?.into_bytes();
         let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
         assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
+        if evidence_run.is_some() {
+            evidence_objects.push(serde_json::json!({
+                "key": outage_key, "version_id": null,
+                "expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
+                "expected_sha256": sha256_hex(&expected_outage_body),
+                "actual_sha256": sha256_hex(&actual),
+                "expected_physical": null,
+                "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
+            }));
+        }
 
         let terminal_deadline = Instant::now() + Duration::from_secs(30);
         loop {
@@ -1432,6 +1535,31 @@ mod tests {
             return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
         }
 
+        if let Some((directory, run)) = evidence_run {
+            let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
+            assert_ne!(target_pid, restarted_pid, "target must be a new process");
+            assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
+            let evidence = serde_json::json!({
+                "schema": 1, "case": "background-target-restart", "evidence": "process-restart",
+                "run_id": run.run_id, "source_revision": run.source_revision,
+                "test_build": compiled_test_identity(),
+                "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
+                "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
+                "pid_before": target_pid, "pid_after": restarted_pid,
+                "objects": evidence_objects, "node_listings": node_listings,
+            });
+            let data = serde_json::to_vec(&evidence)?;
+            if data.len() > 1024 * 1024 {
+                return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
+            }
+            let mut output = std::fs::OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(directory.join("background-target-restart.json"))?;
+            output.write_all(&data)?;
+            output.sync_all()?;
+        }
+
         Ok(())
     }
 

crates/e2e_test/src/lib.rs

@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;
 
+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;

crates/e2e_test/src/on_demand_migration/interaction_test.rs

@@ -22,8 +22,8 @@
 //! local object and what the source was asked for.
 
 use super::common::{
-    AdminResponse, BoxError, OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env,
+    ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
-    start_configured_env_with,
+    OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with,
 };
 use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
 use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
@@ -32,7 +32,7 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
     ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
-    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, VersioningConfiguration,
+    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
 };
 use bytes::Bytes;
 use local_ip_address::local_ip;
@@ -733,6 +733,242 @@ async fn test_odm_disable_keeps_pulled_objects_and_stops_source_traffic() -> Tes
     Ok(())
 }
 
+/// The process switch preserves configured buckets and unfinished jobs while
+/// restoring local-only S3 behavior, including after an ordinary metadata write.
+#[tokio::test]
+async fn test_odm_global_disable_preserves_data_config_and_backfill_across_restarts() -> TestResult {
+    let bucket = "odm-global-disable";
+    let mut env = start_configured_env(bucket, SOURCE_BUCKET, |spec| spec.policy.list_through = true).await?;
+    let pulled_key = "migrated/pulled.bin";
+    let remote_key = "remote/untouched.bin";
+    let pending_key = "backfill/pending.bin";
+    let local_key = "local/kept.bin";
+    let source_body = Bytes::from_static(b"source payload");
+    let local_body = Bytes::from_static(b"client payload");
+    env.seed_source(
+        SOURCE_BUCKET,
+        &[
+            SeedObject::new(pulled_key, source_body.clone()),
+            SeedObject::new(remote_key, source_body.clone()),
+            SeedObject::new(pending_key, source_body.clone()),
+        ],
+    );
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key(local_key)
+        .body(local_body.clone().into())
+        .send()
+        .await?;
+    let pulled = env.raw_get(bucket, pulled_key).await?;
+    assert_eq!(pulled.status, 200);
+    assert_eq!(pulled.header(ODM_RESPONSE_HEADER), Some("source"));
+    assert_eq!(pulled.body, source_body);
+    let stored = env.raw_get(bucket, pulled_key).await?;
+    assert_eq!(stored.status, 200);
+    assert_eq!(stored.header(ODM_RESPONSE_HEADER), None, "the inline pull has committed locally");
+    assert_eq!(stored.body, source_body);
+    let config = env.get_config(bucket).await?;
+    assert_eq!(config.status, 200, "{}", config.body);
+    let config = config.json()?;
+
+    // Hold every attempt until the process has exited, so retries cannot commit
+    // the only backfill object before the crash. The start checkpoint exists.
+    let mut pending_get = env.source.hold_get_object(SOURCE_BUCKET, pending_key);
+    let started = env
+        .start_backfill(
+            bucket,
+            BackfillRequest {
+                prefix: Some("backfill/".to_string()),
+                ..BackfillRequest::default()
+            },
+        )
+        .await?;
+    assert_eq!(started.status, 200, "{}", started.body);
+    let job_id = started.json()?["job"]["job_id"].as_str().ok_or("missing job ID")?.to_string();
+    tokio::time::timeout(Duration::from_secs(10), pending_get.wait_until_entered())
+        .await
+        .expect("backfill never reached the held source GET")?;
+    let process = env.rustfs.process.as_mut().ok_or("missing RustFS process before crash")?;
+    assert!(process.try_wait()?.is_none(), "RustFS exited before the controlled crash");
+    process.kill()?;
+    let stopped = process.wait()?;
+    assert!(!stopped.success(), "the interrupted process must exit after being killed");
+    drop(env.rustfs.process.take());
+    drop(pending_get);
+    env.source.take_requests();
+    env.rustfs
+        .restart_server_preserving_data(vec![], &[(ODM_MODULE_SWITCH_ENV, "false"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")])
+        .await?;
+
+    let off_config = env.get_config(bucket).await?;
+    assert_eq!(off_config.status, 200, "{}", off_config.body);
+    assert_eq!(off_config.json()?, config, "the saved configuration and timestamp survive disabling");
+    let status = env.status_json(bucket).await?;
+    assert_eq!(status["configured"], true, "{status}");
+    assert_eq!(status["enabled"], true, "the bucket remains configured as enabled: {status}");
+    assert_eq!(status["module_enabled"], false, "{status}");
+    assert_eq!(status["counters"], Value::Null, "no bucket runtime is installed: {status}");
+    let checkpoint = env.backfill_job(bucket).await?.ok_or("disabled module lost the checkpoint")?;
+    assert_eq!(checkpoint["job_id"], job_id);
+    assert_eq!(checkpoint["state"], "running", "the interrupted job is retained: {checkpoint}");
+
+    for (key, body) in [(local_key, &local_body), (pulled_key, &source_body)] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 200);
+        assert_eq!(&get.body, body);
+        assert_eq!(get.header(ODM_RESPONSE_HEADER), None);
+        let head = env.client.head_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
+    }
+    for key in [remote_key, pending_key] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 404, "disabled source GET {key}: {}", String::from_utf8_lossy(&get.body));
+        let head = env.client.head_object().bucket(bucket).key(key).send().await;
+        let err = head.expect_err("a source-only object must remain absent locally");
+        assert_eq!(err.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
+
+    let replacement = Bytes::from_static(b"written while the module is off");
+    for key in [local_key, "local/deleted.bin"] {
+        env.client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(replacement.clone().into())
+            .send()
+            .await?;
+    }
+    env.client
+        .delete_object()
+        .bucket(bucket)
+        .key("local/deleted.bin")
+        .send()
+        .await?;
+    assert_eq!(env.raw_get(bucket, "local/deleted.bin").await?.status, 404);
+    assert_eq!(env.raw_get(bucket, local_key).await?.body, replacement);
+
+    // Both wire protocols must finish their local pages even though the saved
+    // configuration still requests list-through.
+    for use_v2 in [false, true] {
+        let mut cursor = None;
+        let mut listed = Vec::new();
+        for page_number in 0..2 {
+            let (keys, truncated, next) = if use_v2 {
+                let page = env
+                    .client
+                    .list_objects_v2()
+                    .bucket(bucket)
+                    .max_keys(1)
+                    .set_continuation_token(cursor)
+                    .send()
+                    .await?;
+                (
+                    page.contents()
+                        .iter()
+                        .map(|object| object.key().expect("listed key").to_string())
+                        .collect::<Vec<_>>(),
+                    page.is_truncated(),
+                    page.next_continuation_token().map(str::to_string),
+                )
+            } else {
+                let page = env
+                    .client
+                    .list_objects()
+                    .bucket(bucket)
+                    .max_keys(1)
+                    .set_marker(cursor)
+                    .send()
+                    .await?;
+                // V1 may omit NextMarker without a delimiter; clients then
+                // continue from the last returned key.
+                let next = page.next_marker().or_else(|| {
+                    if page.is_truncated() == Some(true) {
+                        page.contents().last().and_then(|object| object.key())
+                    } else {
+                        None
+                    }
+                });
+                (
+                    page.contents()
+                        .iter()
+                        .map(|object| object.key().expect("listed key").to_string())
+                        .collect::<Vec<_>>(),
+                    page.is_truncated(),
+                    next.map(str::to_string),
+                )
+            };
+            assert_eq!(keys.len(), 1, "one local key per page, V2={use_v2}");
+            assert_eq!(truncated, Some(page_number == 0), "local pagination must terminate, V2={use_v2}");
+            if page_number == 0 {
+                assert!(next.as_ref().is_some_and(|value| !value.is_empty()), "missing local cursor, V2={use_v2}");
+            }
+            cursor = next;
+            listed.extend(keys);
+        }
+        assert_eq!(listed, [local_key, pulled_key], "source-only keys must stay absent, V2={use_v2}");
+    }
+
+    let spec = env.fake_source_spec(SOURCE_BUCKET);
+    for response in [
+        env.configure_source(bucket, &spec).await?,
+        env.validate_source(bucket, &spec).await?,
+        env.backfill(bucket, BackfillOp::Start(BackfillRequest::default())).await?,
+    ] {
+        assert_eq!(response.status, 400, "{}", response.body);
+        assert!(response.body.contains("OnDemandMigrationDisabled"), "{}", response.body);
+    }
+    let tagging = Tagging::builder()
+        .tag_set(Tag::builder().key("module").value("disabled").build()?)
+        .build()?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(tagging.clone())
+        .send()
+        .await?;
+    assert_eq!(env.get_config(bucket).await?.json()?, config, "an unrelated metadata write preserves ODM");
+    assert_eq!(
+        env.backfill_job(bucket).await?,
+        Some(checkpoint),
+        "no recovery or checkpoint update while disabled"
+    );
+    assert!(
+        env.source.requests().is_empty(),
+        "disabled startup and all requests must leave the source untouched"
+    );
+
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        env.get_config(bucket).await?.json()?,
+        config,
+        "reenabling uses the persisted configuration"
+    );
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert_eq!(tags.tag_set(), tagging.tag_set(), "the ordinary metadata write also persists");
+    let resumed = env.raw_get(bucket, remote_key).await?;
+    assert_eq!(resumed.status, 200);
+    assert_eq!(resumed.header(ODM_RESPONSE_HEADER), Some("source"));
+    assert_eq!(resumed.body, source_body, "stored credentials still authenticate without reconfiguration");
+    let completed = env
+        .wait_for_backfill(bucket, SETTLE, |job| job["state"] == "completed")
+        .await?;
+    assert_eq!(completed["job_id"], job_id, "the interrupted job resumes without a new start");
+    assert_eq!(completed["failed"], 0, "{completed}");
+    for (key, body) in [
+        (local_key, &replacement),
+        (pulled_key, &source_body),
+        (pending_key, &source_body),
+    ] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 200);
+        assert_eq!(&get.body, body);
+        assert_eq!(get.header(ODM_RESPONSE_HEADER), None, "{key} remains stored locally");
+    }
+    Ok(())
+}
+
 /// Case 19: the admin surface an operator sees — the configuration read back
 /// without its secret, and a status document whose counters match the source
 /// journal exactly.

crates/ecstore/src/api/mod.rs

@@ -69,6 +69,13 @@ pub mod bucket {
             };
         }
 
+        pub mod recovery_control {
+            pub use crate::bucket::lifecycle::recovery_control::{
+                IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
+                inspect_recovery_control, list_recovery_controls,
+            };
+        }
+
         pub mod transition_transaction {
             pub use crate::bucket::lifecycle::transition_transaction::{
                 TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
@@ -89,8 +96,9 @@ pub mod bucket {
         #[allow(clippy::module_inception)]
         pub mod lifecycle {
             pub use crate::bucket::lifecycle::lifecycle::{
-                Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
+                Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
-                TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
+                ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
+                object_opts_from_object_info,
             };
         }
 
@@ -151,15 +159,17 @@ pub mod bucket {
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
             ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
-            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
+            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
-            get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
+            delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
-            get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
+            get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
-            get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
+            get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
+            get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
             init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
             update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
-            update_quota_if_incarnation, update_under_transaction_lock,
+            update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock,
+            update_under_transaction_lock_at,
         };
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -794,17 +794,24 @@ impl BucketTargetSys {
     ) -> Result<BucketTargets, BucketTargetError> {
         self.validate_target(bucket, target).await?;
 
-        let mut bucket_targets = match self.list_bucket_targets(bucket).await {
+        let mut bucket_targets = self.targets_base_for_write(bucket).await?;
-            Ok(targets) => targets,
-            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
-            Err(err) => return Err(err),
-        };
 
         Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
 
         Ok(bucket_targets)
     }
 
+    /// Ordinary writes must not turn an unreadable cached snapshot into an
+    /// empty configuration. Explicit repair belongs to the metadata transaction
+    /// that can inspect the current persisted state.
+    async fn targets_base_for_write(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
+        match self.list_bucket_targets(bucket).await {
+            Ok(targets) => Ok(targets),
+            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
+            Err(err) => Err(err),
+        }
+    }
+
     pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
         if !target.target_type.is_valid() {
             return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
@@ -863,7 +870,9 @@ impl BucketTargetSys {
         Ok(())
     }
 
-    fn upsert_target_entry(
+    /// Merge a validated target into a caller-owned snapshot. The caller must
+    /// protect that snapshot through persistence.
+    pub fn upsert_target_entry(
         bucket_targets: &mut Vec<BucketTarget>,
         target: &BucketTarget,
         update: bool,
@@ -1227,25 +1236,27 @@ impl BucketTargetSys {
             return (String::new(), false);
         };
 
-        {
+        let targets_map = self.targets_map.read().await;
-            let targets_map = self.targets_map.read().await;
+        let targets = targets_map.get(bucket).map(Vec::as_slice).unwrap_or_default();
-            if let Some(targets) = targets_map.get(bucket) {
+        Self::remote_arn_for_targets(targets, target, depl_id)
-                for tgt in targets {
+    }
-                    if tgt.target_type == target.target_type
+
-                        && tgt.target_bucket == target.target_bucket
+    /// Resolve create idempotency against the snapshot the caller will persist.
-                        && target.endpoint == tgt.endpoint
+    pub fn remote_arn_for_targets(targets: &[BucketTarget], target: &BucketTarget, depl_id: &str) -> (String, bool) {
-                        && tgt
+        for tgt in targets {
-                            .credentials
+            if tgt.target_type == target.target_type
-                            .as_ref()
+                && tgt.target_bucket == target.target_bucket
-                            .map(|c| {
+                && target.endpoint == tgt.endpoint
-                                let default_creds = Credentials::default();
+                && tgt
-                                c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
+                    .credentials
-                            })
+                    .as_ref()
-                            .unwrap_or(false)
+                    .map(|c| {
-                    {
+                        let default_creds = Credentials::default();
-                        return (tgt.arn.clone(), true);
+                        c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
-                    }
+                    })
-                }
+                    .unwrap_or(false)
+            {
+                return (tgt.arn.clone(), true);
             }
         }
 
@@ -4313,4 +4324,41 @@ mod tests {
         let window = LastMinuteLatency::new();
         assert_eq!(window.get_total().avg, Duration::from_secs(0));
     }
+
+    fn repair_target(bucket: &str, id: &str) -> BucketTarget {
+        BucketTarget {
+            source_bucket: bucket.to_string(),
+            endpoint: "remote.example.com".to_string(),
+            target_bucket: "remote".to_string(),
+            arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
+            target_type: BucketTargetType::ReplicationService,
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn an_unreadable_target_set_refuses_cached_writes() {
+        let sys = BucketTargetSys::default();
+        let bucket = "targets-repair-opt-in";
+        sys.mark_targets_unreadable(bucket).await;
+        assert!(matches!(
+            sys.targets_base_for_write(bucket).await,
+            Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn a_readable_target_set_remains_the_write_base() {
+        let sys = BucketTargetSys::default();
+        let bucket = "targets-repair-readable";
+        let existing = repair_target(bucket, "keep");
+        sys.targets_map
+            .write()
+            .await
+            .insert(bucket.to_string(), vec![existing.clone()]);
+        let base = sys.targets_base_for_write(bucket).await.expect("read targets");
+        assert_eq!(base.targets.len(), 1);
+        assert_eq!(base.targets[0].arn, existing.arn);
+    }
 }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -150,6 +150,18 @@ static XXHASH_SEED: u64 = 0;
 static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
 static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
 
+#[cfg(test)]
+#[derive(Default)]
+struct FreeVersionPostRemoteDeleteTestBarrier {
+    arrived: Notify,
+    release: Notify,
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc<FreeVersionPostRemoteDeleteTestBarrier>;
+}
+
 pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
 #[allow(
     dead_code,
@@ -910,6 +922,11 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
             })??;
         }
     }
+    #[cfg(test)]
+    if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
     if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
         // Remote DELETE is idempotent, but a changed fence makes the local
         // outcome ambiguous. Keep every marker for a fully fenced retry.
@@ -5831,7 +5848,7 @@ mod tests {
     #[cfg(feature = "test-util")]
     use crate::services::tier::test_util::register_mock_tier;
     #[cfg(feature = "test-util")]
-    use crate::services::tier::tier::TierConfigMgr;
+    use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory};
     #[cfg(feature = "test-util")]
     use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
     use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
@@ -7830,6 +7847,119 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial]
+    async fn tier_remove_waits_for_inflight_free_version_local_commit() {
+        let (disk_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4());
+        let object = "free-version";
+        create_test_bucket(&ecstore, &bucket).await;
+        let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await;
+        let tier_manager = ecstore.tier_config_mgr();
+        {
+            let manager = tier_manager.read().await;
+            manager
+                .save_tiering_config(Arc::clone(&ecstore))
+                .await
+                .expect("mock tier configuration should persist before removal");
+        }
+        seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await;
+        let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
+            .await
+            .expect("seeded free version should be listed");
+        let oi = page
+            .items
+            .into_iter()
+            .next()
+            .expect("seeded free version should be recoverable");
+
+        backend
+            .set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
+            .await;
+        let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM")
+            .await
+            .expect("mock tier lease should be available");
+        seed_lease
+            .put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
+            .await
+            .expect("remote free-version tuple should be seeded");
+        drop(seed_lease);
+
+        let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
+        let cleanup_barrier = Arc::clone(&barrier);
+        let cleanup_store = Arc::clone(&ecstore);
+        let cleanup_oi = oi.clone();
+        let cleanup = tokio::spawn(async move {
+            super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
+                .scope(cleanup_barrier, async move {
+                    super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
+                })
+                .await
+        });
+        tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified())
+            .await
+            .expect("free-version cleanup should pause after the remote delete");
+        assert!(!backend.contains(&oi.transitioned_object.name).await);
+
+        let remove_manager = Arc::clone(&tier_manager);
+        let remove_store = Arc::clone(&ecstore);
+        let remove_backend = backend.clone();
+        let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone())));
+        let mut remove = tokio::spawn(async move {
+            TIER_DRIVER_TEST_FACTORY
+                .scope(
+                    remove_driver_factory,
+                    TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true),
+                )
+                .await
+        });
+        let prepared = tokio::time::timeout(StdDuration::from_secs(30), async {
+            loop {
+                match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await {
+                    Ok(lease) => drop(lease),
+                    Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break,
+                    Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"),
+                }
+                tokio::task::yield_now().await;
+            }
+        });
+        tokio::select! {
+            prepared = prepared => {
+                prepared.expect("tier remove should install its prepared admission fence");
+            }
+            result = &mut remove => {
+                panic!("tier remove finished before installing its prepared admission fence: {result:?}");
+            }
+        }
+        assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit");
+
+        barrier.release.notify_one();
+        tokio::time::timeout(StdDuration::from_secs(30), cleanup)
+            .await
+            .expect("free-version cleanup should finish after release")
+            .expect("free-version cleanup task should join")
+            .expect("free-version cleanup should keep its generation current");
+        tokio::time::timeout(StdDuration::from_secs(30), remove)
+            .await
+            .expect("tier remove should finish after local cleanup")
+            .expect("tier remove task should join")
+            .expect("tier remove should pass its fresh authoritative proof");
+
+        assert!(!tier_manager.read().await.is_tier_valid("WARM"));
+        for disk_path in &disk_paths {
+            assert!(
+                !fs::try_exists(disk_path.join(&bucket).join(object))
+                    .await
+                    .expect("post-removal free-version path check should succeed")
+            );
+        }
+        ecstore
+            .delete_bucket(&bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("empty free-version test bucket should be removed");
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial]

crates/ecstore/src/bucket/lifecycle/core.rs

@@ -15,9 +15,9 @@
 use crate::object_api::ObjectInfo;
 
 pub use rustfs_lifecycle::{
-    Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
+    Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
-    TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
+    RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
-    expiration_action_has_valid_target,
+    expected_expiry_time, expiration_action_has_valid_target,
 };
 
 pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -22,7 +22,7 @@ use super::{
     bucket_lifecycle_ops::{
         ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
     },
-    manual_transition_job, tier_delete_journal, transition_transaction,
+    manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
 };
 use crate::error::{Error, Result};
 use crate::services::tier::tier_probe_intent;
@@ -41,6 +41,7 @@ pub(crate) enum DurableIlmRecordKind {
     ManualTransitionScope,
     ManualTransitionTask,
     ManualTransitionWorkerResult,
+    RecoveryControl,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -105,8 +106,14 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
+pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "recovery-control",
+    prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
+    max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
+    kind: DurableIlmRecordKind::RecoveryControl,
+};
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
     TIER_DELETE_JOURNAL_V6_NAMESPACE,
     TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
@@ -116,6 +123,7 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
     MANUAL_TRANSITION_TASK_NAMESPACE,
     MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+    RECOVERY_CONTROL_NAMESPACE,
 ];
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -241,6 +249,18 @@ pub(crate) enum DurableIlmRecordCheckpoint {
     ManualTransitionWorkerResult {
         content_sha256: String,
     },
+    RecoveryControl {
+        content_sha256: String,
+        identity_sha256: String,
+        source_generation_sha256: String,
+        first_seen_at_unix_nanos: i64,
+        revision: u64,
+        classification: recovery_control::IlmRecoveryClassification,
+        attempt_count: u64,
+        consecutive_failure_count: u32,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        owner_fence_sha256: Option<String>,
+    },
 }
 
 impl DurableIlmRecordCheckpoint {
@@ -254,7 +274,8 @@ impl DurableIlmRecordCheckpoint {
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
             | Self::ManualTransitionTask { content_sha256 }
-            | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
+            | Self::ManualTransitionWorkerResult { content_sha256 }
+            | Self::RecoveryControl { content_sha256, .. } => content_sha256,
         }
     }
 
@@ -528,6 +549,51 @@ impl DurableIlmRecordCheckpoint {
                     ..
                 },
             ) => previous_identity == next_identity && next_updated_at > previous_updated_at,
+            (
+                Self::RecoveryControl {
+                    identity_sha256: previous_identity,
+                    source_generation_sha256: previous_generation,
+                    first_seen_at_unix_nanos: previous_first_seen,
+                    revision: previous_revision,
+                    classification: previous_classification,
+                    attempt_count: previous_attempts,
+                    consecutive_failure_count: previous_failures,
+                    owner_fence_sha256: previous_owner,
+                    ..
+                },
+                Self::RecoveryControl {
+                    identity_sha256: next_identity,
+                    source_generation_sha256: next_generation,
+                    first_seen_at_unix_nanos: next_first_seen,
+                    revision: next_revision,
+                    classification: next_classification,
+                    attempt_count: next_attempts,
+                    consecutive_failure_count: next_failures,
+                    owner_fence_sha256: next_owner,
+                    ..
+                },
+            ) => {
+                let adjacent = previous_identity == next_identity
+                    && previous_first_seen == next_first_seen
+                    && previous_revision.checked_add(1) == Some(*next_revision);
+                let claim = next_owner.is_some()
+                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
+                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
+                    && previous_attempts.checked_add(1) == Some(*next_attempts)
+                    && previous_failures == next_failures;
+                let source_refresh = previous_owner.is_some()
+                    && previous_owner == next_owner
+                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
+                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
+                    && previous_attempts == next_attempts
+                    && previous_failures == next_failures
+                    && previous_generation != next_generation;
+                let completion = previous_owner.is_some()
+                    && next_owner.is_none()
+                    && previous_generation == next_generation
+                    && previous_attempts == next_attempts;
+                adjacent && (claim || source_refresh || completion)
+            }
             _ => false,
         };
 
@@ -553,6 +619,14 @@ impl DurableIlmRecordCheckpoint {
         {
             return false;
         }
+        if let Self::RecoveryControl { classification, .. } = terminal
+            && !matches!(
+                classification,
+                recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
+            )
+        {
+            return false;
+        }
         if self == terminal || self.validate_successor(terminal).is_ok() {
             return true;
         }
@@ -652,6 +726,32 @@ impl DurableIlmRecordCheckpoint {
                         .is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
                     && (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
             }
+            (
+                Self::RecoveryControl {
+                    identity_sha256: previous_identity,
+                    source_generation_sha256: previous_generation,
+                    first_seen_at_unix_nanos: previous_first_seen,
+                    revision: previous_revision,
+                    attempt_count: previous_attempts,
+                    ..
+                },
+                Self::RecoveryControl {
+                    identity_sha256: terminal_identity,
+                    source_generation_sha256: terminal_generation,
+                    first_seen_at_unix_nanos: terminal_first_seen,
+                    revision: terminal_revision,
+                    attempt_count: terminal_attempts,
+                    classification:
+                        recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
+                    ..
+                },
+            ) => {
+                previous_identity == terminal_identity
+                    && (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
+                    && previous_first_seen == terminal_first_seen
+                    && terminal_revision > previous_revision
+                    && terminal_attempts >= previous_attempts
+            }
             _ => false,
         }
     }
@@ -1219,6 +1319,35 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 },
             )
         }
+        DurableIlmRecordKind::RecoveryControl => {
+            let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
+                .map_err(|err| Error::other(err.to_string()))?;
+            let control =
+                recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
+            let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
+                .map_err(|err| Error::other(err.to_string()))?;
+            if canonical != path || control.identity.protocol != protocol {
+                return Err(Error::other("ILM recovery control path is not canonical"));
+            }
+            let identity_sha256 = checkpoint_hash(&control.identity)?;
+            let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
+            let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
+            (
+                "control_id",
+                control_id,
+                DurableIlmRecordCheckpoint::RecoveryControl {
+                    content_sha256,
+                    identity_sha256,
+                    source_generation_sha256,
+                    first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
+                    revision: control.revision,
+                    classification: control.classification,
+                    attempt_count: control.attempt_count,
+                    consecutive_failure_count: control.consecutive_failure_count,
+                    owner_fence_sha256,
+                },
+            )
+        }
         DurableIlmRecordKind::ManualTransitionJob => {
             let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
                 .map_err(|err| Error::other(err.to_string()))?;
@@ -1412,6 +1541,94 @@ mod tests {
             .checkpoint
     }
 
+    fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
+        let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
+        let generation = recovery_control::IlmRecoverySourceGeneration::new(
+            transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
+            "source-etag",
+            "a".repeat(64),
+            vec![recovery_control::IlmRecoverySourceCopy {
+                authority: "pool-0/set-0".to_string(),
+                canonical_path: source_path.to_string(),
+                etag: "source-etag".to_string(),
+                encoded_len: 128,
+                content_sha256: "a".repeat(64),
+            }],
+        )
+        .expect("source generation should build");
+        recovery_control::IlmRecoveryControl::new(
+            recovery_control::IlmRecoveryControlIdentity {
+                protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
+                canonical_source_path: source_path.to_string(),
+                stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
+                record_class: "transition_transaction_v1".to_string(),
+            },
+            generation,
+            recovery_control::IlmRecoveryClassification::Retrying,
+            1_000_000_000,
+            recovery_control::IlmRecoveryErrorCode::None,
+        )
+        .expect("recovery control should build")
+    }
+
+    fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
+        let control_id = control.identity.source_operation_digest().expect("control id should derive");
+        let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
+            .expect("control path should build");
+        let encoded = control.encode().expect("control should encode");
+        let namespace = classify_durable_ilm_record(&path)
+            .expect("recovery control namespace should classify")
+            .expect("recovery control should be durable");
+        assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
+        validate_durable_ilm_record(&path, &encoded)
+            .expect("recovery control should validate")
+            .checkpoint
+    }
+
+    #[test]
+    fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
+        let initial_control = recovery_control_fixture();
+        let initial = recovery_control_checkpoint(&initial_control);
+
+        let mut claimed_control = initial_control;
+        let mut advanced_generation = claimed_control.observed_source_generation.clone();
+        advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
+        claimed_control
+            .claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
+            .expect("control should claim");
+        let claimed = recovery_control_checkpoint(&claimed_control);
+        initial.validate_successor(&claimed).expect("claim should advance receipt");
+
+        let mut retry_control = claimed_control;
+        retry_control
+            .record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
+            .expect("retry should persist");
+        let retry = recovery_control_checkpoint(&retry_control);
+        claimed.validate_successor(&retry).expect("retry should advance receipt");
+
+        let ready_at = retry_control
+            .next_attempt_at_unix_nanos
+            .expect("retry deadline should persist");
+        let mut terminal_control = retry_control;
+        terminal_control
+            .claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
+            .expect("retry should claim");
+        let reclaimed = recovery_control_checkpoint(&terminal_control);
+        retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
+        terminal_control
+            .finish_attempt(
+                recovery_control::IlmRecoveryClassification::Terminal,
+                recovery_control::IlmRecoveryErrorCode::None,
+            )
+            .expect("control should terminate");
+        let terminal = recovery_control_checkpoint(&terminal_control);
+        reclaimed
+            .validate_successor(&terminal)
+            .expect("terminal state should advance receipt");
+        assert!(initial.is_predecessor_of_terminal(&terminal));
+        assert!(!initial.is_predecessor_of_terminal(&retry));
+    }
+
     #[test]
     fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
         let initial_intent = tier_probe_intent_fixture();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -24,6 +24,7 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
 mod object_handlers_common;
 mod object_lock_boundary;
 pub use self::core as lifecycle;
+pub mod recovery_control;
 mod replication_sink;
 pub mod rule;
 mod runtime_boundary;

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -35,6 +35,10 @@ use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::durable_namespace::{
     TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record,
 };
+use crate::bucket::lifecycle::recovery_control::{
+    IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
+    load_recovery_control, observe_recovery_source, save_recovery_control_if_absent,
+};
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
     Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -78,6 +82,13 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
 const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
 const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
 const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
+const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
+const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
+const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
+const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
+const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
+const TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS: &str = "tier_delete_journal_corrupt";
+const CORRUPT_TIER_DELETE_JOURNAL_IDENTITY: &str = "corrupt";
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
@@ -5509,6 +5520,125 @@ enum TierDeleteJournalEntryRecoveryOutcome {
     Failed,
 }
 
+fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&str> {
+    let identity = object_name
+        .strip_prefix(TIER_DELETE_JOURNAL_LEGACY_PREFIX)?
+        .strip_suffix(".json")?;
+    (rustfs_utils::crypto::is_sha256_checksum(identity)
+        && !identity
+            .bytes()
+            .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()))
+    .then_some(identity)
+}
+
+fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
+    match entry.persisted_version {
+        1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
+        TIER_DELETE_JOURNAL_VERSION => Some((TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS)),
+        _ => None,
+    }
+}
+
+fn legacy_tier_delete_control_matches(
+    control: &IlmRecoveryControl,
+    identity: &IlmRecoveryControlIdentity,
+    generation: &crate::bucket::lifecycle::recovery_control::IlmRecoverySourceGeneration,
+    classification: IlmRecoveryClassification,
+    error_code: IlmRecoveryErrorCode,
+) -> bool {
+    control.identity == *identity
+        && control.observed_source_generation == *generation
+        && control.classification == classification
+        && control.last_error_code == error_code
+        && control.owner.is_none()
+        && control.attempt_count == 0
+        && control.consecutive_failure_count == 0
+}
+
+fn legacy_tier_delete_control_is_scheduler_fence(control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity) -> bool {
+    control.identity == *identity && control.owner.is_none() && !control.classification.permits_automatic_attempt()
+}
+
+async fn persist_legacy_tier_delete_recovery_control(
+    api: Arc<ECStore>,
+    object_name: &str,
+    observed_data: &[u8],
+    stable_operation_identity: String,
+    (source_schema, record_class): (&'static str, &'static str),
+    intended_classification: IlmRecoveryClassification,
+    intended_error_code: IlmRecoveryErrorCode,
+) -> Result<()> {
+    let identity = IlmRecoveryControlIdentity {
+        protocol: IlmRecoveryProtocol::TierDeleteJournal,
+        canonical_source_path: object_name.to_string(),
+        stable_operation_identity,
+        record_class: record_class.to_string(),
+    };
+    let control_id = identity.source_operation_digest().map_err(Error::other)?;
+    match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
+        Ok(observed) if legacy_tier_delete_control_is_scheduler_fence(&observed.control, &identity) => return Ok(()),
+        Ok(_) => return Err(Error::PreconditionFailed),
+        Err(Error::ConfigNotFound) => {}
+        Err(err) => return Err(err),
+    }
+
+    let source = observe_recovery_source(api.clone(), object_name, source_schema).await?;
+    let exact_source = source.is_consistent() && source.canonical_data.as_deref() == Some(observed_data);
+    let (classification, error_code) = if exact_source {
+        (intended_classification, intended_error_code)
+    } else {
+        (IlmRecoveryClassification::Corrupt, IlmRecoveryErrorCode::SourceDivergent)
+    };
+    let candidate = IlmRecoveryControl::new(
+        identity.clone(),
+        source.generation.clone(),
+        classification,
+        i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
+            .map_err(|_| Error::other("tier delete journal recovery timestamp does not fit i64"))?,
+        error_code,
+    )
+    .map_err(Error::other)?;
+
+    match save_recovery_control_if_absent(api.clone(), &candidate).await {
+        Ok(()) | Err(Error::PreconditionFailed) => {}
+        Err(save_error) => match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
+            Ok(observed)
+                if legacy_tier_delete_control_matches(
+                    &observed.control,
+                    &identity,
+                    &source.generation,
+                    classification,
+                    error_code,
+                ) =>
+            {
+                return Ok(());
+            }
+            Ok(_) | Err(_) => return Err(save_error),
+        },
+    }
+
+    let observed = load_recovery_control(api, IlmRecoveryProtocol::TierDeleteJournal, &control_id).await?;
+    if !legacy_tier_delete_control_matches(&observed.control, &identity, &source.generation, classification, error_code) {
+        return Err(Error::PreconditionFailed);
+    }
+    Ok(())
+}
+
+async fn retain_corrupt_legacy_tier_delete_journal(api: Arc<ECStore>, object_name: &str, data: &[u8]) -> Result<()> {
+    canonical_legacy_tier_delete_journal_identity(object_name)
+        .ok_or_else(|| Error::other("tier delete journal path is not canonical"))?;
+    persist_legacy_tier_delete_recovery_control(
+        api,
+        object_name,
+        data,
+        CORRUPT_TIER_DELETE_JOURNAL_IDENTITY.to_string(),
+        (TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS),
+        IlmRecoveryClassification::Corrupt,
+        IlmRecoveryErrorCode::SourceCorrupt,
+    )
+    .await
+}
+
 async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome {
     let data = match config_boundary::read_config(api.clone(), &object_name).await {
         Ok(data) => data,
@@ -5529,6 +5659,22 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
     let je = match decode_tier_delete_journal_entry(&data) {
         Ok(je) => je,
         Err(err) => {
+            if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
+                return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
+                    Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+                    Err(control_error) => {
+                        warn!(
+                            event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                            journal_object = %object_name,
+                            error = ?control_error,
+                            "Failed to retain corrupt tier delete journal recovery control"
+                        );
+                        TierDeleteJournalEntryRecoveryOutcome::Failed
+                    }
+                };
+            }
             warn!(
                 event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
                 component = LOG_COMPONENT_ECSTORE,
@@ -5542,6 +5688,22 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
     };
 
     if tier_delete_journal_object_name(&je) != object_name {
+        if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
+            return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
+                Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+                Err(err) => {
+                    warn!(
+                        event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                        journal_object = %object_name,
+                        error = ?err,
+                        "Failed to retain mismatched tier delete journal recovery control"
+                    );
+                    TierDeleteJournalEntryRecoveryOutcome::Failed
+                }
+            };
+        }
         warn!(
             event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
             component = LOG_COMPONENT_ECSTORE,
@@ -5552,6 +5714,36 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
         return TierDeleteJournalEntryRecoveryOutcome::Failed;
     }
 
+    if let Some((source_schema, record_class)) = legacy_tier_delete_recovery_descriptor(&je) {
+        let stable_operation_identity = canonical_legacy_tier_delete_journal_identity(&object_name)
+            .expect("decoded legacy journal path was validated against its canonical object name")
+            .to_string();
+        return match persist_legacy_tier_delete_recovery_control(
+            api,
+            &object_name,
+            &data,
+            stable_operation_identity,
+            (source_schema, record_class),
+            IlmRecoveryClassification::RetainedAmbiguous,
+            IlmRecoveryErrorCode::RemoteVersionUnknown,
+        )
+        .await
+        {
+            Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+            Err(err) => {
+                warn!(
+                    event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                    journal_object = %object_name,
+                    error = ?err,
+                    "Failed to retain legacy tier delete journal recovery control"
+                );
+                TierDeleteJournalEntryRecoveryOutcome::Failed
+            }
+        };
+    }
+
     match api
         .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data)
         .await

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
 #[cfg(test)]
 static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
     obj_name: &str,
     rv_id: &str,
@@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
 
 #[cfg(test)]
 mod test {
+    #[cfg(feature = "test-util")]
+    use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
     use rustfs_s3_client::signer_error::invalid_utf8_header_error;
 
     use super::{
         CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
         RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
-        delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
+        delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
-        delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
+        is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
-        is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
+        should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
-        transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
     };
     use crate::storage_api_contracts::lifecycle::TransitionedObject;
     use rustfs_filemeta::TransitionVersionState;

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,6 +23,11 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
+use crate::bucket::lifecycle::recovery_control::{
+    IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
+    ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
+    save_recovery_control_if_absent, save_recovery_control_if_current,
+};
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
@@ -44,6 +49,7 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
 pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
 const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
+const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
 pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
@@ -737,9 +743,11 @@ pub enum TransitionTransactionRecoveryOutcome {
     RemoteCandidateDeleted,
     RecordDeleted,
     Retained,
+    RetainedAmbiguous(IlmRecoveryErrorCode),
+    OperatorRequired(IlmRecoveryErrorCode),
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 #[derive(Default)]
 struct TransitionRecoveryClaimBarrierState {
     transaction_id: Uuid,
@@ -747,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState {
     release: tokio::sync::Notify,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) struct TransitionRecoveryClaimBarrier {
     state: Arc<TransitionRecoveryClaimBarrierState>,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
     std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
 > = std::sync::OnceLock::new();
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl TransitionRecoveryClaimBarrier {
     pub(crate) fn install(transaction_id: Uuid) -> Self {
         let state = Arc::new(TransitionRecoveryClaimBarrierState {
@@ -788,7 +796,7 @@ impl TransitionRecoveryClaimBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl Drop for TransitionRecoveryClaimBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -802,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
     let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -817,6 +825,80 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
     }
 }
 
+#[cfg(all(test, feature = "test-util"))]
+#[derive(Default)]
+struct TransitionRecoveryTerminalBarrierState {
+    transaction_id: Uuid,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) struct TransitionRecoveryTerminalBarrier {
+    state: Arc<TransitionRecoveryTerminalBarrierState>,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(all(test, feature = "test-util"))]
+impl TransitionRecoveryTerminalBarrier {
+    pub(crate) fn install(transaction_id: Uuid) -> Self {
+        let state = Arc::new(TransitionRecoveryTerminalBarrierState {
+            transaction_id,
+            ..Default::default()
+        });
+        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("transition recovery terminal barrier mutex should not poison");
+        assert!(
+            slot.is_none(),
+            "transition recovery terminal barrier must be installed by one test at a time"
+        );
+        *slot = Some(Arc::clone(&state));
+        drop(slot);
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("transition recovery should persist terminal control before source cleanup");
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl Drop for TransitionRecoveryTerminalBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("transition recovery terminal barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
+    let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("transition recovery terminal barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.transaction_id == transaction_id)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum TransitionOperatorProbe {
@@ -1020,17 +1102,35 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
     let suffix = object
         .strip_prefix(&prefix)
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
-    let file_name = suffix
+    let mut parts = suffix.split('/');
-        .rsplit('/')
+    let shard_a = parts
         .next()
         .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
+    let shard_b = parts
+        .next()
+        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
+    let file_name = parts
+        .next()
+        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
+    if parts.next().is_some() {
+        return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
+    }
     let transaction_key = file_name
         .strip_suffix(".json")
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
-    if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
+    if transaction_key.len() != 32
+        || !transaction_key
+            .bytes()
+            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+        || shard_a != &transaction_key[..2]
+        || shard_b != &transaction_key[2..4]
+    {
         return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
     }
-    Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
+    Uuid::parse_str(transaction_key)
+        .ok()
+        .filter(|transaction_id| !transaction_id.is_nil())
+        .ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
 }
 
 pub async fn process_transition_transaction_record(
@@ -1055,6 +1155,27 @@ async fn process_transition_transaction_record_at(
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     let record_name =
         transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
+    let now_unix_nanos =
+        i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
+    let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
+    let recovery_control_id = recovery_control_identity
+        .source_operation_digest()
+        .map_err(|err| Error::other(err.to_string()))?;
+    let control_record_name =
+        recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
+            .map_err(|err| Error::other(err.to_string()))?;
+    let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
+        Some(
+            api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
+                .await?,
+        )
+    } else {
+        None
+    };
+    let _control_guard = match &control_lock {
+        Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
+        None => None,
+    };
     // The synthetic key avoids nesting the recovery lock with the config
     // object's own I/O lock. Holding it across the bounded source proof and
     // remote DELETE elects one destructive recovery worker across nodes.
@@ -1073,55 +1194,400 @@ async fn process_transition_transaction_record_at(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
 
-    match current.state {
+    let mut recovery_control = if transition_state_needs_recovery_control(&current, now_unix_nanos) {
+        if cleanup_terminal_transition_recovery_control(
+            api.clone(),
+            &current,
+            &record_name,
+            &recovery_control_identity,
+            &recovery_control_id,
+        )
+        .await?
+        {
+            return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
+        }
+        match claim_transition_recovery_control(
+            api.clone(),
+            &current,
+            &record_name,
+            recovery_control_identity,
+            &recovery_control_id,
+            now_unix_nanos,
+        )
+        .await?
+        {
+            Some(control) => Some(control),
+            None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
+        }
+    } else {
+        None
+    };
+
+    let recovery = match current.state {
         TransitionTransactionState::Uploaded => {
-            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
+            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
-                return Ok(TransitionTransactionRecoveryOutcome::Retained);
+                Ok(TransitionTransactionRecoveryOutcome::Retained)
-            }
+            } else {
-            let mut cleanup = current.clone();
+                let mut cleanup = current.clone();
-            cleanup
+                cleanup
-                .mark_cleanup_pending(
+                    .mark_cleanup_pending(
-                    current.fence(),
+                        current.fence(),
-                    TransitionCleanupProof {
+                        TransitionCleanupProof {
-                        transaction_id: current.transaction_id,
+                            transaction_id: current.transaction_id,
-                        write_id: current.write_id,
+                            write_id: current.write_id,
-                        remote_object: current.remote_object.clone(),
+                            remote_object: current.remote_object.clone(),
-                        remote_version: current.remote_version.clone(),
+                            remote_version: current.remote_version.clone(),
-                        backend_fingerprint: current.backend_fingerprint,
+                            backend_fingerprint: current.backend_fingerprint,
-                        decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
+                            decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
-                    },
+                        },
-                )
+                    )
-                .map_err(transition_transaction_store_error)?;
+                    .map_err(transition_transaction_store_error)?;
-            #[cfg(test)]
+                #[cfg(all(test, feature = "test-util"))]
-            pause_before_transition_recovery_claim(current.transaction_id).await;
+                pause_before_transition_recovery_claim(current.transaction_id).await;
-            match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
+                match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
-                Ok(()) => recover_cleanup_pending(api, &cleanup).await,
+                    Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
-                Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                    Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
-                Err(err) => Err(err),
+                        Ok(TransitionTransactionRecoveryOutcome::Retained)
+                    }
+                    Err(err) => Err(err),
+                }
             }
         }
-        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, &current).await,
+        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), &current).await,
         TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), &current).await {
-            Ok(true) => {
+            Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
-                delete_transition_transaction_record(api, &current).await?;
+            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-                Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+                IlmRecoveryErrorCode::LocalCommitAmbiguous,
-            }
+            )),
-            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                IlmRecoveryErrorCode::LocalCommitAmbiguous,
+            )),
             Err(err) => Err(err),
         },
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, &current).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => {
-            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
+            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
                 Ok(TransitionTransactionRecoveryOutcome::Retained)
             } else {
-                recover_unknown_upload_outcome(api, &current).await
+                recover_unknown_upload_outcome(api.clone(), &current).await
             }
         }
-        TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
+        TransitionTransactionState::UploadStarted => {
+            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
+                Ok(TransitionTransactionRecoveryOutcome::Retained)
+            } else {
+                Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
+                    IlmRecoveryErrorCode::RemoteVersionUnknown,
+                ))
+            }
+        }
+    };
+
+    if let Some(mut control) = recovery_control.take() {
+        let source_to_delete = if matches!(
+            recovery,
+            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
+                | TransitionTransactionRecoveryOutcome::RecordDeleted)
+        ) {
+            let refreshed =
+                refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
+            control = refreshed.0;
+            refreshed.1
+        } else {
+            None
+        };
+        persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
+        if let Some(source) = source_to_delete {
+            #[cfg(all(test, feature = "test-util"))]
+            pause_after_transition_recovery_terminal(source.transaction_id).await;
+            delete_transition_transaction_record(api, &source).await?;
+        }
+    } else if matches!(
+        recovery,
+        Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted)
+    ) {
+        delete_transition_transaction_record(api, &current).await?;
+    }
+    recovery
+}
+
+fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
+    IlmRecoveryControlIdentity {
+        protocol: IlmRecoveryProtocol::TransitionTransaction,
+        canonical_source_path: record_name.to_string(),
+        stable_operation_identity: transaction.transaction_id.to_string(),
+        record_class: "transition_transaction_v1".to_string(),
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
+    let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
+    transition_recovery_control_identity(transaction, &record_name)
+        .source_operation_digest()
+        .map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
+}
+
+fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
+    now_unix_nanos >= transaction.not_after_unix_nanos
+        && !matches!(
+            transaction.state,
+            TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
+        )
+}
+
+async fn cleanup_terminal_transition_recovery_control(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+    record_name: &str,
+    identity: &IlmRecoveryControlIdentity,
+    control_id: &str,
+) -> EcstoreResult<bool> {
+    let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
+        Ok(observed) => observed,
+        Err(Error::ConfigNotFound) => return Ok(false),
+        Err(err) => return Err(err),
+    };
+    if observed.control.classification != IlmRecoveryClassification::Terminal {
+        return Ok(false);
+    }
+    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
+    let exact_source = source.is_consistent()
+        && source.generation == observed.control.observed_source_generation
+        && source.canonical_data.as_deref().is_some_and(|data| {
+            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
+        });
+    if observed.control.identity != *identity || !exact_source {
+        return Ok(false);
+    }
+    delete_transition_transaction_record(api, transaction).await?;
+    Ok(true)
+}
+
+async fn claim_transition_recovery_control(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+    record_name: &str,
+    identity: IlmRecoveryControlIdentity,
+    control_id: &str,
+    now_unix_nanos: i64,
+) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
+    let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
+        Ok(control) => Some(control),
+        Err(Error::ConfigNotFound) => None,
+        Err(err) => return Err(err),
+    };
+    if let Some(observed) = existing.as_ref() {
+        if observed.control.identity != identity {
+            return Ok(None);
+        }
+        if observed
+            .control
+            .owner
+            .as_ref()
+            .is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos)
+        {
+            let mut expired = observed.control.clone();
+            expired
+                .record_expired_attempt(now_unix_nanos)
+                .map_err(|err| Error::other(err.to_string()))?;
+            save_recovery_control_if_current(api, observed, &expired).await?;
+            return Ok(None);
+        }
+        if !observed.control.should_attempt_at(now_unix_nanos) {
+            return Ok(None);
+        }
+    }
+
+    let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
+        Ok(source) => source,
+        Err(err) => {
+            if let Some(observed) = existing {
+                persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
+                return Ok(None);
+            }
+            return Err(err);
+        }
+    };
+    let source_matches = source.is_consistent()
+        && source.canonical_data.as_deref().is_some_and(|data| {
+            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
+        });
+    let source_error = if source_matches {
+        IlmRecoveryErrorCode::None
+    } else if source.canonical_data.is_some() {
+        IlmRecoveryErrorCode::SourceGenerationChanged
+    } else {
+        IlmRecoveryErrorCode::SourceDivergent
+    };
+
+    let mut observed = match existing {
+        Some(control) => control,
+        None => {
+            let candidate = IlmRecoveryControl::new(
+                identity.clone(),
+                source.generation.clone(),
+                if source_matches {
+                    IlmRecoveryClassification::Retrying
+                } else {
+                    IlmRecoveryClassification::Corrupt
+                },
+                now_unix_nanos,
+                source_error,
+            )
+            .map_err(|err| Error::other(err.to_string()))?;
+            match save_recovery_control_if_absent(api.clone(), &candidate).await {
+                Ok(()) | Err(Error::PreconditionFailed) => {}
+                Err(err) => return Err(err),
+            }
+            load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
+        }
+    };
+    if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
+        return Ok(None);
+    }
+
+    let mut claimed = observed.control.clone();
+    claimed
+        .claim_for_source_generation(
+            api.id.to_string(),
+            Uuid::new_v4(),
+            now_unix_nanos,
+            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
+            source.generation,
+        )
+        .map_err(|err| Error::other(err.to_string()))?;
+    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
+    observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
+    if observed.control != claimed {
+        return Err(Error::PreconditionFailed);
+    }
+    if !source_matches {
+        let mut corrupt = observed.control.clone();
+        corrupt
+            .finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
+            .map_err(|err| Error::other(err.to_string()))?;
+        save_recovery_control_if_current(api, &observed, &corrupt).await?;
+        return Ok(None);
+    }
+    Ok(Some(observed))
+}
+
+async fn persist_transition_recovery_source_failure(
+    api: Arc<ECStore>,
+    observed: ObservedIlmRecoveryControl,
+    now_unix_nanos: i64,
+) -> EcstoreResult<()> {
+    let mut claimed = observed.control.clone();
+    claimed
+        .claim(
+            api.id.to_string(),
+            Uuid::new_v4(),
+            now_unix_nanos,
+            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
+        )
+        .map_err(|err| Error::other(err.to_string()))?;
+    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
+    let claimed = load_recovery_control(
+        api.clone(),
+        IlmRecoveryProtocol::TransitionTransaction,
+        &claimed
+            .identity
+            .source_operation_digest()
+            .map_err(|err| Error::other(err.to_string()))?,
+    )
+    .await?;
+    let mut failed = claimed.control.clone();
+    failed
+        .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
+        .map_err(|err| Error::other(err.to_string()))?;
+    save_recovery_control_if_current(api, &claimed, &failed).await
+}
+
+async fn refresh_transition_recovery_control_source(
+    api: Arc<ECStore>,
+    mut observed: ObservedIlmRecoveryControl,
+    record_name: &str,
+    transaction_id: Uuid,
+) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
+    let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
+        Ok(transaction) => transaction,
+        Err(Error::ConfigNotFound) => return Ok((observed, None)),
+        Err(err) => return Err(err),
+    };
+    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
+    let exact_source = source.is_consistent()
+        && source
+            .canonical_data
+            .as_deref()
+            .is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
+    if !exact_source {
+        return Err(Error::PreconditionFailed);
+    }
+    if observed.control.observed_source_generation != source.generation {
+        let mut refreshed = observed.control.clone();
+        refreshed
+            .refresh_owned_source_generation(source.generation)
+            .map_err(|err| Error::other(err.to_string()))?;
+        save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
+        observed = load_recovery_control(
+            api,
+            IlmRecoveryProtocol::TransitionTransaction,
+            &refreshed
+                .identity
+                .source_operation_digest()
+                .map_err(|err| Error::other(err.to_string()))?,
+        )
+        .await?;
+        if observed.control != refreshed {
+            return Err(Error::PreconditionFailed);
+        }
+    }
+    Ok((observed, Some(transaction)))
+}
+
+async fn persist_transition_recovery_result(
+    api: Arc<ECStore>,
+    observed: ObservedIlmRecoveryControl,
+    recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
+    now_unix_nanos: i64,
+) -> EcstoreResult<()> {
+    let mut next = observed.control.clone();
+    match recovery {
+        Ok(
+            TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
+        ) => next
+            .finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
+            .map_err(|err| Error::other(err.to_string()))?,
+        Ok(TransitionTransactionRecoveryOutcome::Retained) => next
+            .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
+            .map_err(|err| Error::other(err.to_string()))?,
+        Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
+            .finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
+            .map_err(|err| Error::other(err.to_string()))?,
+        Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
+            .finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
+            .map_err(|err| Error::other(err.to_string()))?,
+        Err(err) => next
+            .record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
+            .map_err(|err| Error::other(err.to_string()))?,
+    }
+    save_recovery_control_if_current(api, &observed, &next).await
+}
+
+fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
+    match err {
+        Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
+        Error::ConfigNotFound
+        | Error::FileNotFound
+        | Error::FileVersionNotFound
+        | Error::ObjectNotFound(_, _)
+        | Error::VersionNotFound(_, _, _)
+        | Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
+        Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
+        _ => IlmRecoveryErrorCode::Unknown,
     }
 }
 
@@ -1134,10 +1600,7 @@ async fn recover_cleanup_pending(
     transaction: &TransitionTransaction,
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     match local_commit_matches_transaction(api.clone(), transaction).await {
-        Ok(true) => {
+        Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
-            delete_transition_transaction_record(api, transaction).await?;
-            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
-        }
         Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) => Err(err),
@@ -1157,7 +1620,6 @@ async fn delete_unreferenced_transition_candidate(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
     delete_transition_remote_candidate(api.clone(), &current).await?;
-    delete_transition_transaction_record(api, &current).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 
@@ -1178,24 +1640,26 @@ async fn recover_unknown_upload_outcome(
         .await
         .map_err(Error::other)?
     {
-        TransitionCandidateProbe::Missing => {
+        TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
-            delete_transition_transaction_record(api, transaction).await?;
-            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
-        }
         TransitionCandidateProbe::UnversionedPresent => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
         }
         TransitionCandidateProbe::VersionedPresent(version_id)
             if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
         {
-            Ok(TransitionTransactionRecoveryOutcome::Retained)
+            Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
+                IlmRecoveryErrorCode::RemoteVersionUnknown,
+            ))
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
         }
-        TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
+        TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-            Ok(TransitionTransactionRecoveryOutcome::Retained)
+            IlmRecoveryErrorCode::RemoteProbeAmbiguous,
-        }
+        )),
+        TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
+            IlmRecoveryErrorCode::RemoteProbeUnsupported,
+        )),
     }
 }
 
@@ -1289,7 +1753,7 @@ pub async fn recover_transition_transaction_records(
     recover_transition_transaction_records_with_now(api, limit, marker, None).await
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 pub async fn recover_transition_transaction_records_at(
     api: Arc<ECStore>,
     limit: usize,
@@ -1323,6 +1787,11 @@ async fn recover_transition_transaction_records_with_now(
             false,
         )
         .await?;
+    if list.is_truncated && list.next_continuation_token.is_none() {
+        return Err(Error::other(
+            "transition transaction recovery returned a truncated page without a continuation marker",
+        ));
+    }
 
     let mut stats = TransitionTransactionRecoveryStats {
         scanned: 0,
@@ -1381,7 +1850,11 @@ async fn recover_transition_transaction_records_with_now(
             ) => {
                 stats.recovered += 1;
             }
-            Ok(TransitionTransactionRecoveryOutcome::Retained) => {
+            Ok(
+                TransitionTransactionRecoveryOutcome::Retained
+                | TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
+                | TransitionTransactionRecoveryOutcome::OperatorRequired(_),
+            ) => {
                 stats.retained += 1;
                 debug!(
                     event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
@@ -1509,11 +1982,74 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
+    use std::sync::atomic::{AtomicBool, Ordering};
 
     use super::*;
 
     const BACKEND_FINGERPRINT: [u8; 32] = [7; 32];
 
+    struct RecoveryAttemptDropGuard(Arc<AtomicBool>);
+
+    impl Drop for RecoveryAttemptDropGuard {
+        fn drop(&mut self) {
+            self.0.store(true, Ordering::SeqCst);
+        }
+    }
+
+    async fn pending_recovery_attempt(started: Arc<tokio::sync::Notify>, dropped: Arc<AtomicBool>) -> EcstoreResult<()> {
+        let _drop_guard = RecoveryAttemptDropGuard(dropped);
+        started.notify_one();
+        std::future::pending().await
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() {
+        let timeout_started = Arc::new(tokio::sync::Notify::new());
+        let timeout_dropped = Arc::new(AtomicBool::new(false));
+        let timeout_task = tokio::spawn({
+            let started = Arc::clone(&timeout_started);
+            let dropped = Arc::clone(&timeout_dropped);
+            async move {
+                await_transition_transaction_recovery(
+                    &CancellationToken::new(),
+                    TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
+                    pending_recovery_attempt(started, dropped),
+                )
+                .await
+            }
+        });
+        timeout_started.notified().await;
+        tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await;
+        let timed_out = timeout_task.await.expect("timeout wrapper task should join");
+        assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass");
+        assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt");
+
+        let cancel_token = CancellationToken::new();
+        let cancel_started = Arc::new(tokio::sync::Notify::new());
+        let cancel_dropped = Arc::new(AtomicBool::new(false));
+        let cancel_task = tokio::spawn({
+            let cancel_token = cancel_token.clone();
+            let started = Arc::clone(&cancel_started);
+            let dropped = Arc::clone(&cancel_dropped);
+            async move {
+                await_transition_transaction_recovery(
+                    &cancel_token,
+                    TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
+                    pending_recovery_attempt(started, dropped),
+                )
+                .await
+            }
+        });
+        cancel_started.notified().await;
+        cancel_token.cancel();
+        let cancelled = cancel_task.await.expect("cancellation wrapper task should join");
+        assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop");
+        assert!(
+            cancel_dropped.load(Ordering::SeqCst),
+            "outer cancellation must drop its in-flight attempt"
+        );
+    }
+
     #[derive(Default)]
     struct MemoryTransactionStore {
         records: HashMap<Uuid, Vec<u8>>,
@@ -1968,5 +2504,19 @@ mod tests {
             transition_transaction_record_object_name(Uuid::nil()),
             Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
         ));
+        assert_eq!(
+            transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
+            transaction_id
+        );
+        for malformed in [
+            object.to_ascii_uppercase(),
+            object.replace("/aa/aa/", "/ff/aa/"),
+            object.replace("/aa/aa/", "/aa/aa/extra/"),
+        ] {
+            assert!(matches!(
+                transition_transaction_id_from_record_object_name(&malformed),
+                Err(TransitionTransactionError::Corrupt(_))
+            ));
+        }
     }
 }

crates/ecstore/src/bucket/metadata.rs

@@ -791,9 +791,22 @@ impl BucketMetadata {
         }
     }
 
+    /// Replace one config payload and stamp its `*_config_updated_at` with the
+    /// local clock. This is the entry for edits that originate here: the
+    /// local write time is the edit's source time.
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
-        let updated = OffsetDateTime::now_utc();
+        self.update_config_at(config_file, data, OffsetDateTime::now_utc())
+    }
 
+    /// [`Self::update_config`] with an explicit `updated_at` stamp.
+    ///
+    /// For a config replicated from another site the edit's source time is
+    /// the peer's `updated_at`, not the moment it lands here: staleness of
+    /// the next incoming item is judged against the stored stamp, so stamping
+    /// the local apply time would reject a newer source edit that was merely
+    /// delivered late (backlog#2292). Only replication receivers should pass
+    /// a foreign time; local edits keep [`Self::update_config`].
+    pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
         match config_file {
             BUCKET_POLICY_CONFIG => {
                 self.policy_config_json = data;
@@ -1525,6 +1538,39 @@ mod test {
         assert_eq!(metadata.bucket_incarnation_id, incarnation);
     }
 
+    /// backlog#2292: a replicated config is stamped with the source
+    /// `updated_at` it was given, not the local clock, while the plain
+    /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+        let mut metadata = BucketMetadata::new("source-stamped");
+
+        let stamped = metadata
+            .update_config_at(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), source_time)
+            .unwrap();
+        assert_eq!(stamped, source_time);
+        assert_eq!(metadata.policy_config_updated_at, source_time);
+
+        let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
+        let stamped = metadata
+            .update_config_at(BUCKET_TAGGING_CONFIG, tagging, source_time)
+            .unwrap();
+        assert_eq!(stamped, source_time);
+        assert_eq!(metadata.tagging_config_updated_at, source_time);
+
+        let before = OffsetDateTime::now_utc();
+        let stamped = metadata
+            .update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec())
+            .unwrap();
+        assert!(stamped >= before, "a local edit is stamped with the local clock");
+        assert_eq!(metadata.policy_config_updated_at, stamped);
+        assert_eq!(
+            metadata.tagging_config_updated_at, source_time,
+            "restamping one config must not move another config's stamp"
+        );
+    }
+
     #[test]
     fn object_locking_requires_lock_metadata_not_plain_versioning() {
         use s3s::dto::ObjectLockEnabled;
@@ -1611,6 +1657,34 @@ mod test {
         assert!(bm.bucket_target_config.is_none());
     }
 
+    /// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
+    /// already carries as a compatibility fixture stores
+    /// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
+    /// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
+    /// bytes below are the exact payload the fixture in
+    /// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
+    /// ever grows the array-shaped compatibility parse, this test is where the
+    /// upgrade break is pinned and where the decision has to be recorded.
+    #[test]
+    fn minio_array_shaped_bucket_targets_are_unreadable() {
+        let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
+        let mut bm = BucketMetadata::new("minio-array-targets");
+        bm.bucket_targets_config_json = minio_array.clone();
+
+        bm.parse_all_configs()
+            .expect("a MinIO-shaped targets blob must not fail the whole metadata load");
+
+        assert!(
+            bm.bucket_targets_unreadable(),
+            "an array-shaped MinIO targets blob is unreadable, not an empty target set"
+        );
+        assert!(bm.bucket_target_config.is_none());
+        assert_eq!(
+            bm.bucket_targets_config_json, minio_array,
+            "the raw MinIO bytes must survive so the configuration stays recoverable"
+        );
+    }
+
     /// The invariant every branch of `parse_all_configs` shares: a stored but
     /// undecodable payload keeps its raw bytes and leaves the typed field
     /// `None`, so no branch fabricates a value. What a reader may then do with

crates/ecstore/src/bucket/metadata_sys.rs

@@ -567,6 +567,32 @@ pub async fn update_if_incarnation(
         config_file,
         data,
         Some(expected_incarnation_id),
+        None,
+    ))
+    .await
+}
+
+/// [`update_if_incarnation`] stamping the config with `updated_at` instead of
+/// the local clock.
+///
+/// For a site-replication receiver the edit's source time is the peer's
+/// `updated_at`; persisting it keeps the stored `*_config_updated_at` on the
+/// source clock so the next item's staleness is judged source-time against
+/// source-time (backlog#2292). See [`BucketMetadata::update_config_at`].
+pub async fn update_if_incarnation_at(
+    bucket: &str,
+    config_file: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    Box::pin(update_with_sys_expected(
+        get_bucket_metadata_sys()?,
+        bucket,
+        config_file,
+        data,
+        Some(expected_incarnation_id),
+        Some(updated_at),
     ))
     .await
 }
@@ -577,6 +603,30 @@ pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_inc
         bucket,
         config_file,
         Some(expected_incarnation_id),
+        None,
+    ))
+    .await
+}
+
+/// [`delete_if_incarnation`] stamping the cleared config with `updated_at`
+/// (a replicated deletion's source time) instead of the local clock.
+///
+/// The stamp survives the deletion as the config's `*_config_updated_at`, and
+/// that is what the next incoming item is judged against: a local stamp on
+/// the delete would reject a newer source re-create that was merely delivered
+/// later (backlog#2292). See [`update_if_incarnation_at`].
+pub async fn delete_if_incarnation_at(
+    bucket: &str,
+    config_file: &str,
+    expected_incarnation_id: Uuid,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    Box::pin(delete_with_sys_expected(
+        get_bucket_metadata_sys()?,
+        bucket,
+        config_file,
+        Some(expected_incarnation_id),
+        Some(updated_at),
     ))
     .await
 }
@@ -598,34 +648,41 @@ async fn update_with_sys(
     config_file: &str,
     data: Vec<u8>,
 ) -> Result<OffsetDateTime> {
-    update_with_sys_expected(sys, bucket, config_file, data, None).await
+    update_with_sys_expected(sys, bucket, config_file, data, None, None).await
 }
 
+/// `updated_at` is the stamp persisted on the config; `None` uses the local
+/// clock (the edit originates here), `Some` carries a replicated edit's
+/// source time (backlog#2292).
 async fn update_with_sys_expected(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
     expected_incarnation_id: Option<Uuid>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
-    update_under_config_write_guard(sys, &guard, config_file, data).await
+    update_under_config_write_guard(sys, &guard, config_file, data, updated_at).await
 }
 
 /// [`delete`] against an explicitly supplied metadata system. See
 /// [`update_with_sys`].
 async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
-    delete_with_sys_expected(sys, bucket, config_file, None).await
+    delete_with_sys_expected(sys, bucket, config_file, None, None).await
 }
 
+/// `updated_at`: `None` stamps the local clock; `Some` persists a replicated
+/// deletion's source time (backlog#2292).
 async fn delete_with_sys_expected(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
     config_file: &str,
     expected_incarnation_id: Option<Uuid>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
-    delete_under_config_write_guard(sys, &guard, config_file).await
+    delete_under_config_write_guard(sys, &guard, config_file, updated_at).await
 }
 
 /// Owns the complete bucket-config mutation fence.
@@ -772,7 +829,21 @@ pub async fn update_under_transaction_lock(
     data: Vec<u8>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(bucket)?;
-    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
+    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, None).await
+}
+
+/// [`update_under_transaction_lock`] stamping the config with `updated_at`
+/// (a replicated edit's source time) instead of the local clock; see
+/// [`update_if_incarnation_at`] (backlog#2292).
+pub async fn update_under_transaction_lock_at(
+    guard: &BucketMetadataMutationGuard,
+    bucket: &str,
+    config_file: &str,
+    data: Vec<u8>,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    guard.ensure_valid(bucket)?;
+    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, Some(updated_at)).await
 }
 
 /// Clear one config file while the caller holds this bucket's transaction lock.
@@ -782,7 +853,7 @@ pub async fn delete_under_transaction_lock(
     config_file: &str,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(bucket)?;
-    delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
+    delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, None).await
 }
 
 pub async fn update_quota_if_incarnation(
@@ -790,6 +861,29 @@ pub async fn update_quota_if_incarnation(
     data: Vec<u8>,
     expected_incarnation_id: Uuid,
     proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+) -> Result<OffsetDateTime> {
+    update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, None).await
+}
+
+/// [`update_quota_if_incarnation`] stamping the quota config with
+/// `updated_at` (a replicated edit's source time) instead of the local
+/// clock; see [`update_if_incarnation_at`] (backlog#2292).
+pub async fn update_quota_if_incarnation_at(
+    bucket: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, Some(updated_at)).await
+}
+
+async fn update_quota_if_incarnation_stamped(
+    bucket: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let sys = get_bucket_metadata_sys()?;
     let guard = Box::pin(acquire_config_write_guard_for_incarnation(
@@ -807,7 +901,7 @@ pub async fn update_quota_if_incarnation(
             achieved: 0,
         });
     }
-    update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
+    update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data, updated_at).await
 }
 
 pub async fn update_bucket_targets_under_transaction_lock(
@@ -823,6 +917,7 @@ async fn update_under_config_write_guard(
     guard: &BucketMetadataMutationGuard,
     config_file: &str,
     data: Vec<u8>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(&guard.bucket)?;
     let metadata_sys = sys.read().await.clone();
@@ -834,7 +929,7 @@ async fn update_under_config_write_guard(
             Some(&guard.transaction_guard),
             &guard.bucket,
             "bucket config transaction",
-            metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id),
+            metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id, updated_at),
         ),
     )
     .await?;
@@ -846,6 +941,7 @@ async fn delete_under_config_write_guard(
     sys: Arc<RwLock<BucketMetadataSys>>,
     guard: &BucketMetadataMutationGuard,
     config_file: &str,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(&guard.bucket)?;
     let metadata_sys = sys.read().await.clone();
@@ -857,7 +953,7 @@ async fn delete_under_config_write_guard(
             Some(&guard.transaction_guard),
             &guard.bucket,
             "bucket config deletion transaction",
-            metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id),
+            metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id, updated_at),
         ),
     )
     .await?;
@@ -1762,15 +1858,17 @@ impl BucketMetadataSys {
     /// `update` and the config read alone). Keep these boxed.
     pub async fn update(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
         let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?;
-        Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await
+        Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id, None)).await
     }
 
     pub async fn delete(&self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
         let incarnation_id = self.get_bucket_incarnation_id(bucket).await?;
-        self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id)
+        self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id, None)
             .await
     }
 
+    /// `updated_at`: `None` stamps the local clock; `Some` persists a
+    /// replicated edit's source time (backlog#2292).
     async fn update_checked(
         &self,
         bucket: &str,
@@ -1778,6 +1876,7 @@ impl BucketMetadataSys {
         data: Vec<u8>,
         parse: bool,
         expected_incarnation_id: Uuid,
+        updated_at: Option<OffsetDateTime>,
     ) -> Result<OffsetDateTime> {
         // Load through this system's own store, the one `save` persists to
         // (backlog#1052 S7). Reading from the ambient handle instead made the
@@ -1788,7 +1887,10 @@ impl BucketMetadataSys {
             return Err(Error::BucketNotFound(bucket.to_string()));
         }
 
-        let updated = bm.update_config(config_file, data)?;
+        let updated = match updated_at {
+            Some(updated_at) => bm.update_config_at(config_file, data, updated_at)?,
+            None => bm.update_config(config_file, data)?,
+        };
 
         Box::pin(self.save(bm)).await?;
 
@@ -3755,6 +3857,106 @@ mod tests {
         );
     }
 
+    /// backlog#2292: the explicit-stamp write path persists the given source
+    /// time as the config's `*_config_updated_at` — through the incarnation
+    /// path and through an already-held transaction guard — and survives a
+    /// reload from disk, while the plain path keeps stamping the local clock.
+    #[tokio::test]
+    async fn explicit_updated_at_is_persisted_as_the_config_stamp() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let bucket = "source-stamped-config";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+        }
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
+        let source_time = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
+        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
+        let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
+
+        // Incarnation path (`update_if_incarnation_at` minus the ambient lookup).
+        let stamped =
+            update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(source_time))
+                .await
+                .expect("source-stamped policy write should persist");
+        assert_eq!(stamped, source_time);
+
+        // Held-guard path (`update_under_transaction_lock_at` minus the ambient lookup).
+        let guard = acquire_config_write_guard(sys.clone(), bucket).await.expect("write guard");
+        let stamped = update_under_config_write_guard(sys.clone(), &guard, BUCKET_TAGGING_CONFIG, tagging, Some(source_time))
+            .await
+            .expect("source-stamped tagging write should persist");
+        drop(guard);
+        assert_eq!(stamped, source_time);
+
+        let metadata_sys = sys.read().await.clone();
+        metadata_sys.metadata_map.write().await.clear();
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, source_time);
+        assert_eq!(reloaded.tagging_config_updated_at, source_time);
+
+        // The plain path is unchanged: a local edit is stamped with the local clock.
+        let before = OffsetDateTime::now_utc();
+        let stamped = update_with_sys(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy)
+            .await
+            .expect("locally stamped policy write should persist");
+        assert!(stamped >= before, "the plain write path must keep stamping the local clock");
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, stamped);
+        assert_eq!(
+            reloaded.tagging_config_updated_at, source_time,
+            "an unrelated config keeps its source stamp"
+        );
+    }
+
+    /// backlog#2292: a replicated delete persists the source time as the
+    /// cleared config's `*_config_updated_at`, so the receive-side gate
+    /// (source time against stored stamp) lets a newer source re-create land
+    /// even when the delete was applied later than the re-create's source
+    /// time; the plain delete keeps stamping the local clock.
+    #[tokio::test]
+    async fn explicit_updated_at_is_persisted_by_a_delete() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let bucket = "source-stamped-delete";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+        }
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
+        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
+        let created_at = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
+        let deleted_at = created_at + Duration::from_secs(60);
+        let recreated_at = deleted_at + Duration::from_secs(60);
+
+        update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(created_at))
+            .await
+            .expect("source-stamped policy write should persist");
+        let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, Some(deleted_at))
+            .await
+            .expect("source-stamped policy delete should persist");
+        assert_eq!(stamped, deleted_at);
+
+        let metadata_sys = sys.read().await.clone();
+        metadata_sys.metadata_map.write().await.clear();
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert!(reloaded.policy_config_json.is_empty(), "the delete cleared the payload");
+        assert_eq!(reloaded.policy_config_updated_at, deleted_at, "the delete kept the source stamp");
+        assert!(
+            recreated_at >= reloaded.policy_config_updated_at,
+            "a re-create newer than the delete's source time is not stale against the stored stamp"
+        );
+
+        // The plain delete path is unchanged: stamped with the local clock.
+        update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy, None, Some(recreated_at))
+            .await
+            .expect("re-create should persist");
+        let before = OffsetDateTime::now_utc();
+        let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, None)
+            .await
+            .expect("locally stamped delete should persist");
+        assert!(stamped >= before, "the plain delete path must keep stamping the local clock");
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, stamped);
+    }
+
     /// The load and the persisted write share one write guard, so concurrent
     /// rewrites of the same config compose instead of clobbering each other.
     /// Moving the load outside that guard loses all but the last tag.
@@ -3971,10 +4173,16 @@ mod tests {
         let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
         assert_ne!(old_incarnation, new_incarnation);
 
-        let err =
+        let err = update_with_sys_expected(
-            update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec(), Some(old_incarnation))
+            sys.clone(),
-                .await
+            bucket,
-                .expect_err("a request authorized for the deleted incarnation must fail closed");
+            BUCKET_TAGGING_CONFIG,
+            b"<Tagging/>".to_vec(),
+            Some(old_incarnation),
+            None,
+        )
+        .await
+        .expect_err("a request authorized for the deleted incarnation must fail closed");
         assert!(matches!(err, Error::BucketNotFound(name) if name == bucket));
 
         let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap();
@@ -4009,7 +4217,7 @@ mod tests {
             }],
         })
         .unwrap();
-        update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging)
+        update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging, None)
             .await
             .unwrap();
         assert!(!delete.is_finished());

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -20,9 +20,9 @@ pub use rustfs_replication::{
 pub(crate) use rustfs_replication::{
     ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
     delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
-    delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
+    delete_replication_object_opts, delete_replication_target_version_id, heal_uses_delete_replication_path,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
+    is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
-    resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
+    replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
-    single_part_replica_etag_mismatch, target_delete_version_id,
+    resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -882,6 +882,20 @@ fn reconstructed_heal_delete_info(
 ) -> DeletedObjectReplicationInfo {
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
+    // The caller hands us a blank ObjectInfo (the source marker may already be
+    // gone), so the state above carries no target-assigned marker version ids.
+    // Restore them from the journal: `delete_marker_purge_version_id` must hit
+    // the id the target reported, not fall back to the source marker id, which
+    // a target that mints its own ids answers with an idempotent 204 that would
+    // acknowledge the intent while the real marker stays behind (backlog#2290).
+    // The corrupt flag rides along so a refusal stays a refusal after restart.
+    for (arn, version_id) in &entry.target_delete_marker_version_ids {
+        rstate
+            .target_delete_marker_version_ids
+            .entry(arn.clone())
+            .or_insert_with(|| version_id.clone());
+    }
+    rstate.target_delete_marker_version_ids_corrupt |= entry.target_delete_marker_version_ids_corrupt;
 
     let delete_marker_mtime = entry
         .delete_marker_mtime
@@ -6601,4 +6615,87 @@ mod tests {
             replacement_data
         );
     }
+
+    /// backlog#2290: a delete-marker purge intent that survives a restart
+    /// through the MRF journal addresses the marker version the TARGET
+    /// assigned, exactly as the live watcher does (see the
+    /// `requires_delayed_purge` spawn). The journal carries the per-ARN ids
+    /// (`targetDeleteMarkerVersionIDs`) and replay restores them into the
+    /// reconstructed replication state; without that the replay would fall
+    /// back to the source marker id, which a target that mints its own ids
+    /// answers with an idempotent 204 — the entry would be acknowledged while
+    /// the real marker stayed behind.
+    #[test]
+    fn mrf_delete_marker_purge_replay_preserves_target_assigned_marker_version() {
+        use super::super::replication_object_decision_boundary::{delete_marker_purge_mrf_entry, delete_marker_purge_version_id};
+
+        let arn = "arn:minio:replication::generic-target:photos".to_string();
+        let source_marker = uuid::Uuid::new_v4();
+        let remote_marker = "remote-assigned-marker-version".to_string();
+
+        let live_oi = ObjectInfo {
+            bucket: "photos".to_string(),
+            name: "obj".to_string(),
+            version_id: Some(source_marker),
+            delete_marker: true,
+            ..Default::default()
+        };
+        let mut live_state = live_oi.replication_state();
+        live_state.replicate_decision_str = replicate_decision_for_admitted_targets(std::slice::from_ref(&arn)).to_string();
+        live_state
+            .target_delete_marker_version_ids
+            .insert(arn.clone(), remote_marker.clone());
+        let live = DeletedObjectReplicationInfo {
+            delete_object: ReplicationDeletedObject {
+                object_name: "obj".to_string(),
+                delete_marker: true,
+                delete_marker_version_id: Some(source_marker),
+                replication_state: Some(live_state),
+                ..Default::default()
+            },
+            bucket: "photos".to_string(),
+            ..Default::default()
+        };
+        assert_eq!(
+            delete_marker_purge_version_id(live.delete_object.replication_state.as_ref(), &arn, source_marker),
+            Some(Some(remote_marker.clone())),
+            "the live purge addresses the recorded target version"
+        );
+
+        // Watch window exhausted: persist the intent, restart, replay it.
+        let entry = delete_marker_purge_mrf_entry(&live, vec![arn.clone()]);
+        let replay_oi = ObjectInfo {
+            bucket: entry.bucket.clone(),
+            name: entry.object.clone(),
+            version_id: entry.version_id,
+            delete_marker: entry.delete_marker,
+            ..Default::default()
+        };
+        let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
+        let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
+
+        assert_eq!(
+            delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
+            Some(Some(remote_marker)),
+            "the MRF replay must address the target-assigned marker version, not source marker {source_marker}"
+        );
+
+        // A refusal (inconsistent recorded ids) must stay a refusal across the
+        // journal round trip instead of degrading into the source-id fallback.
+        let mut refused = live;
+        refused
+            .delete_object
+            .replication_state
+            .as_mut()
+            .expect("state was set above")
+            .target_delete_marker_version_ids_corrupt = true;
+        let entry = delete_marker_purge_mrf_entry(&refused, vec![arn.clone()]);
+        assert!(entry.target_delete_marker_version_ids_corrupt);
+        let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
+        assert_eq!(
+            delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
+            None,
+            "the MRF replay must keep refusing to guess when the recorded ids were inconsistent"
+        );
+    }
 }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -32,11 +32,11 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec;
 use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
 use super::replication_object_decision_boundary::{
     MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-    delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
+    delete_replication_creates_marker, delete_replication_target_version_id, heal_uses_delete_replication_path,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
+    is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
-    resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
+    replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
-    target_delete_version_id,
+    should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
 };
 use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
 use super::replication_resync_boundary::ResyncStatusType;
@@ -2051,7 +2051,11 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
 
     let is_version_purge = is_version_delete_replication(&dobj.delete_object);
 
-    let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
+    // The watcher exists to purge a replicated marker once the SOURCE marker
+    // vanishes. A version purge is that purge already (its failures reach the
+    // journal as a purge entry), so it must not spawn a second watcher that
+    // journals a duplicate intent (backlog#2290).
+    let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object) && !is_version_purge;
 
     let (replication_status, prev_status) = if !is_version_purge {
         (
@@ -2761,12 +2765,6 @@ fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str
 }
 
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
-    let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
-        version_id.to_owned()
-    } else {
-        dobj.delete_object.version_id.unwrap_or_default()
-    };
-
     let mut rinfo = dobj
         .delete_object
         .replication_state
@@ -2799,7 +2797,25 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         return rinfo;
     }
 
-    let version_id = target_delete_version_id(version_id, is_version_purge);
+    // Purging a replicated delete marker addresses the version the target
+    // assigned (recorded when the marker was created there); see
+    // `delete_replication_target_version_id`. A corrupt record is a failure,
+    // not a guess: the entry stays visible until the metadata is repaired.
+    let Some(version_id) = delete_replication_target_version_id(&dobj.delete_object, &tgt_client.arn) else {
+        warn!(
+            event = EVENT_DELETE_MARKER_PURGE_FAILED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            bucket = tgt_client.bucket,
+            object = dobj.delete_object.object_name,
+            arn = %tgt_client.arn,
+            reason = "recorded_target_version_inconsistent",
+            "Replicated version purge refused: recorded target delete-marker version metadata is inconsistent"
+        );
+        rinfo.version_purge_status = VersionPurgeStatusType::Failed;
+        rinfo.error = Some("recorded target delete-marker version metadata is inconsistent".to_string());
+        return rinfo;
+    };
 
     if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
         match head_object_for_worker(

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -68,7 +68,10 @@ use std::{
     },
     time::SystemTime,
 };
-use tokio::{net::TcpStream, time::Duration};
+use tokio::{
+    net::TcpStream,
+    time::{Duration, timeout},
+};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tracing::{debug, info, warn};
@@ -874,6 +877,16 @@ impl PeerRestClient {
         self.offline.store(false, Ordering::Release);
     }
 
+    /// Prepare a retry without allowing connection-cache cleanup to extend the
+    /// caller's probe deadline. The offline gate is cleared even when eviction
+    /// times out so a cancelled cleanup cannot strand the peer in fast-fail
+    /// mode; a later request can perform a fresh eviction if needed.
+    pub async fn prepare_retry_with_timeout(&self, timeout_duration: Duration) -> bool {
+        let evicted = timeout(timeout_duration, self.evict_connection()).await.is_ok();
+        self.offline.store(false, Ordering::Release);
+        evicted
+    }
+
     /// Whether this failure means the peer is unreachable, so it should be
     /// gated offline and its connection evicted.
     ///

crates/ecstore/src/core/pools.rs

@@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
         .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
     let id_kind = parts
         .next()
-        .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id"))
+        .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
         .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
     let source_path = parts
         .next()
@@ -3023,8 +3023,9 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
         return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
     }
     match id_kind {
-        "operation_id" if !is_sha256_checksum(id) => {
+        "operation_id" | "control_id" if !is_sha256_checksum(id) => {
-            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id")));
+            let id_label = id_kind.trim_end_matches("_id");
+            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
         }
         "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
             return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
@@ -19300,8 +19301,8 @@ mod pools_tests {
         load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
         merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
         missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
-        pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica,
+        parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
-        reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
+        read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
         record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
         renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
         reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
@@ -20169,6 +20170,26 @@ mod pools_tests {
         assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
     }
 
+    #[test]
+    fn decommission_recovery_control_receipt_path_round_trips() {
+        let run_token = "b".repeat(64);
+        let control_id = "a".repeat(64);
+        let source_path = format!(
+            "ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
+            &control_id[..2],
+            &control_id[2..4],
+            control_id
+        );
+        let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
+
+        let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
+
+        assert_eq!(locator.run_token, run_token);
+        assert_eq!(locator.source_path, source_path);
+        assert_eq!(locator.id_kind, "control_id");
+        assert_eq!(locator.id, control_id);
+    }
+
     #[test]
     fn decommission_receipt_merge_preserves_terminal_proof() {
         let operation_id = "a".repeat(64);

crates/ecstore/src/diagnostics/get.rs

@@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
 pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
 pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
 pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
+#[cfg(unix)]
 pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
 pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
 pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";

crates/ecstore/src/disk/disk_store.rs

@@ -324,6 +324,70 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
 }
 
 impl LocalDiskWrapper {
+    pub(in crate::disk) async fn delete_version_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        force_del_marker: bool,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        self.track_disk_health_mutation(
+            "delete_version",
+            DiskMetricMutation::Delete,
+            || async {
+                Box::pin(
+                    self.disk
+                        .delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner),
+                )
+                .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    pub(in crate::disk) async fn delete_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        self.track_disk_health_mutation(
+            "delete",
+            DiskMetricMutation::Delete,
+            || async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    pub(in crate::disk) async fn undo_write_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        self.track_disk_health_mutation(
+            "delete_version",
+            DiskMetricMutation::Delete,
+            || async {
+                // Preserve the old DiskAPI future's boxing boundary.
+                Box::pin(
+                    self.disk
+                        .undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner),
+                )
+                .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     pub(in crate::disk) async fn rename_data_observed(
         &self,
         src_volume: &str,
@@ -333,6 +397,34 @@ impl LocalDiskWrapper {
         dst_path: &str,
         external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> super::RenameDataObservation {
+        self.rename_data_observed_with_guards(
+            src_volume,
+            src_path,
+            fi,
+            dst_volume,
+            dst_path,
+            super::RenameDataGuards {
+                external_guard,
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    pub(in crate::disk) async fn rename_data_observed_with_guards(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        guards: super::RenameDataGuards,
+    ) -> super::RenameDataObservation {
+        let super::RenameDataGuards {
+            external_guard,
+            namespace_owner,
+            ..
+        } = guards;
         let operation = self.clone();
         let src_volume = src_volume.to_owned();
         let src_path = src_path.to_owned();
@@ -357,13 +449,15 @@ impl LocalDiskWrapper {
                     DiskMetricMutation::Write,
                     || async {
                         // Preserve the former DiskAPI future's single boxing boundary.
-                        let observed =
+                        let observed = Box::pin(operation.disk.rename_data_observed(
-                            Box::pin(
+                            &src_volume,
-                                operation
+                            &src_path,
-                                    .disk
+                            &fi,
-                                    .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
+                            &dst_volume,
-                            )
+                            &dst_path,
-                            .await;
+                            namespace_owner,
+                        ))
+                        .await;
                         preflight_rejection = observed.preflight_rejection;
                         observed.result
                     },
@@ -1301,6 +1395,7 @@ impl LocalDiskWrapper {
         self.disk.get_object_path(volume, path)
     }
 
+    #[cfg(unix)]
     pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
         self.disk.get_object_path_for_io(volume, path)
     }

crates/ecstore/src/disk/fs.rs

@@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
     fs::rename(from, to).await
 }
 
+#[cfg(any(not(windows), test))]
 pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
     std::fs::rename(from, to)
 }
 
+#[cfg(any(not(windows), test))]
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
     fs::read(path.as_ref()).await

crates/ecstore/src/disk/local/commit.rs

@@ -17,13 +17,14 @@
 
 #[cfg(all(test, windows))]
 use super::run_destination_commit_directory_preparation;
+#[cfg(any(not(windows), test))]
+use super::should_fail_local_inline_rollback_hardlink;
 use super::{
     EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
     LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
     remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
     run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
-    should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
+    should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
-    skip_access_checks,
 };
 #[cfg(test)]
 use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
@@ -33,7 +34,7 @@ use crate::disk::{
     error::{DiskError, Result},
     error_conv::{to_access_error, to_file_error},
     os,
-    os::{check_path_length, rename_all},
+    os::check_path_length,
 };
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileMeta};
@@ -73,6 +74,8 @@ fn rollback_inline_metadata_commit_std(
     rollback_data_dir: Option<Uuid>,
     local_rollback_path: Option<&Path>,
 ) -> std::io::Result<()> {
+    #[cfg(all(test, not(windows)))]
+    os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
     if let Some(backup_path) = local_rollback_path {
         // The commit immediately before this rollback renamed the staged
         // xl.meta from the same directory as `backup_path` onto
@@ -86,6 +89,7 @@ fn rollback_inline_metadata_commit_std(
     Ok(())
 }
 
+#[cfg(any(not(windows), test))]
 pub(super) fn create_local_inline_rollback_backup(
     dst_file_path: &Path,
     staging_file_path: &Path,
@@ -231,6 +235,12 @@ async fn restore_published_data_source(
 #[derive(Debug)]
 pub(in crate::disk) struct LocalRenamePreflightRejection(());
 
+#[derive(Default)]
+pub(super) struct RenameDataState {
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+    preflight_rejection: Option<LocalRenamePreflightRejection>,
+}
+
 impl LocalDisk {
     #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
     pub(super) async fn rename_data_inner(
@@ -240,7 +250,7 @@ impl LocalDisk {
         fi: FileInfo,
         dst_volume: &str,
         dst_path: &str,
-        preflight_rejection: &mut Option<LocalRenamePreflightRejection>,
+        state: &mut RenameDataState,
     ) -> Result<RenameDataResp> {
         crate::hp_guard!("LocalDisk::rename_data");
         let mut fi = fi;
@@ -269,7 +279,13 @@ impl LocalDisk {
             Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
             None => None,
         };
-        let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
+        let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
+            &self.root,
+            dst_volume,
+            &destination_object_path,
+            state.namespace_owner.take(),
+        )
+        .await;
         if let Some(claim) = quota_fence_claim {
             mutation_lease.attach_external_guard(claim);
         }
@@ -302,7 +318,7 @@ impl LocalDisk {
                 error = %e,
                 "Disk local access check failed"
             );
-            *preflight_rejection = Some(LocalRenamePreflightRejection(()));
+            state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
             return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
         }
 
@@ -320,7 +336,7 @@ impl LocalDisk {
                 error = %e,
                 "Disk local access check failed"
             );
-            *preflight_rejection = Some(LocalRenamePreflightRejection(()));
+            state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
             return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
         }
 
@@ -528,7 +544,9 @@ impl LocalDisk {
             // rename below.
             if fi_healing
                 && let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
-                && let Err(err) = self.move_to_trash(dst_data_path, true, false).await
+                && let Err(err) = self
+                    .move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
+                    .await
             {
                 warn!(
                     target: "rustfs_ecstore::disk::local",
@@ -755,7 +773,7 @@ impl LocalDisk {
                 && let Some(parent) = dst_file_path.parent()
             {
                 let fsync_started = rustfs_io_metrics::put_stage_timer();
-                if let Err(err) = os::fsync_dst_dir_group_commit(parent).await {
+                if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
                     rustfs_io_metrics::record_put_object_stage_duration_from(
                         rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
                         fsync_started,
@@ -793,7 +811,7 @@ impl LocalDisk {
                         break;
                     }
                     let fsync_started = rustfs_io_metrics::put_stage_timer();
-                    if let Err(err) = os::fsync_dir(dir).await {
+                    if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
                         rustfs_io_metrics::record_put_object_stage_duration_from(
                             rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
                             fsync_started,
@@ -1024,7 +1042,15 @@ impl LocalDisk {
                 // rename_all acquires the backup path's namespace lease. Do not
                 // hold a disk admission while acquiring another namespace lock.
                 drop(file_sync_admission.take());
-                if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
+                if let Err(err) = os::rename_all_with_owner(
+                    staged_backup,
+                    &backup_path,
+                    &dst_volume_dir,
+                    &self.publication_root,
+                    Some(mutation_lease.clone()),
+                )
+                .await
+                {
                     let _ = remove_file_if_exists(staged_backup);
                     return Err(err);
                 }
@@ -1220,14 +1246,18 @@ impl LocalDisk {
         fi: &FileInfo,
         dst_volume: &str,
         dst_path: &str,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> super::super::RenameDataObservation {
-        let mut preflight_rejection = None;
+        let mut state = RenameDataState {
+            namespace_owner,
+            ..Default::default()
+        };
         let result = self
-            .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection)
+            .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state)
             .await;
         super::super::RenameDataObservation {
             result,
-            preflight_rejection,
+            preflight_rejection: state.preflight_rejection,
         }
     }
 }

crates/ecstore/src/disk/mod.rs

@@ -75,6 +75,14 @@ use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
 
+/// Independent admission and physical ownership for one disk rename.
+#[derive(Default)]
+pub(crate) struct RenameDataGuards {
+    pub(crate) scanner_publication_lease_token: Option<Uuid>,
+    pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
+    pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
+}
+
 /// Local preflight evidence stays outside DiskAPI and the RPC response format.
 pub(crate) struct RenameDataObservation {
     pub(crate) result: Result<RenameDataResp>,
@@ -190,11 +198,17 @@ pub struct MmapCopyStageMetrics {
     pub(crate) path_resolve_stage: &'static str,
     pub(crate) metadata_lookup_stage: &'static str,
     pub(crate) metadata_validate_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) blocking_wait_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) blocking_task_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) file_open_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) mmap_map_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) mmap_copy_stage: &'static str,
+    #[cfg(unix)]
     pub(crate) direct_read_copy_stage: &'static str,
 }
 
@@ -718,6 +732,65 @@ impl Disk {
         }
     }
 
+    pub(crate) async fn delete_version_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        force_del_marker: bool,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        match self {
+            Self::Local(disk) => {
+                disk.delete_version_with_namespace_owner(volume, path, fi, force_del_marker, opts, namespace_owner)
+                    .await
+            }
+            Self::Remote(disk) => {
+                let result = disk.delete_version(volume, path, fi, force_del_marker, opts).await;
+                // This is sender lifetime only, not proof of a remote physical drain.
+                drop(namespace_owner);
+                result
+            }
+        }
+    }
+
+    pub(crate) async fn delete_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        match self {
+            Self::Local(disk) => disk.delete_with_namespace_owner(volume, path, opts, namespace_owner).await,
+            Self::Remote(disk) => {
+                let result = disk.delete(volume, path, opts).await;
+                drop(namespace_owner);
+                result
+            }
+        }
+    }
+
+    /// Keep local undo publication owned independently of the wrapper deadline.
+    /// Remote undo retains its existing RPC contract; this is not a remote drain proof.
+    pub(crate) async fn undo_write_with_namespace_owner(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        opts: DeleteOptions,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        match self {
+            Self::Local(disk) => {
+                disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner)
+                    .await
+            }
+            Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await,
+        }
+    }
+
     pub(crate) async fn rename_data_borrowed(
         &self,
         src_volume: &str,
@@ -737,12 +810,12 @@ impl Disk {
         fi: &FileInfo,
         dst_volume: &str,
         dst_path: &str,
-        scanner_publication_lease_token: Option<Uuid>,
+        guards: RenameDataGuards,
     ) -> RenameDataObservation {
         match self {
             Disk::Local(local_disk) => {
                 local_disk
-                    .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
+                    .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
                     .await
             }
             Disk::Remote(remote_disk) => RenameDataObservation::unknown(
@@ -753,7 +826,7 @@ impl Disk {
                         fi,
                         dst_volume,
                         dst_path,
-                        scanner_publication_lease_token,
+                        guards.scanner_publication_lease_token,
                     )
                     .await,
             ),
@@ -922,6 +995,7 @@ impl Disk {
         }
     }
 
+    #[cfg(unix)]
     pub(crate) fn get_object_path_for_io_if_local(
         &self,
         volume: &str,

crates/ecstore/src/disk/os.rs

@@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder {
     static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
     static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
     static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
+    #[cfg(unix)]
     static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
         std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
     static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
@@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder {
         contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
     }
 
+    #[cfg(unix)]
     pub(crate) fn record_limited(dir: &Path) {
         record_path(&LIMITED, dir, "limited fsync dir recorder");
         let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
@@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder {
         contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
     }
 
+    #[cfg(unix)]
     pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
         BEFORE_LIMITED
             .lock()
@@ -237,11 +240,96 @@ pub(crate) mod fsync_dir_recorder {
             .insert(dir.to_path_buf(), kind);
     }
 
+    #[cfg(unix)]
     pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
         remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
     }
 }
 
+/// Pause a real namespace mutation inside its physical executor.
+#[cfg(all(test, not(windows)))]
+pub(crate) mod prepared_publication_test_hooks {
+    use super::*;
+
+    #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+    pub(crate) enum Stage {
+        PreparedRename,
+        Rename,
+        Remove,
+        Rollback,
+        DirFsync,
+    }
+
+    type Hook = Box<dyn FnOnce() + Send>;
+    type Key = (Stage, PathBuf);
+    static BEFORE_PUBLICATION: LazyLock<Mutex<HashMap<Key, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+    pub(crate) struct Guard(Key);
+
+    impl Drop for Guard {
+        fn drop(&mut self) {
+            BEFORE_PUBLICATION.lock().remove(&self.0);
+        }
+    }
+
+    pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
+        install_at(Stage::PreparedRename, path, hook)
+    }
+
+    pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
+        let key = (stage, path.to_path_buf());
+        assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none());
+        Guard(key)
+    }
+
+    pub(crate) fn run(stage: Stage, path: &Path) {
+        let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf()));
+        if let Some(hook) = hook {
+            hook();
+        }
+    }
+
+    #[cfg(test)]
+    type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
+    #[cfg(test)]
+    static RENAME_DESTINATIONS: LazyLock<Mutex<HashMap<PathBuf, RenameDestinationHook>>> =
+        LazyLock::new(|| Mutex::new(HashMap::new()));
+
+    #[cfg(test)]
+    pub(crate) struct RenameDestinationGuard(PathBuf);
+
+    #[cfg(test)]
+    impl Drop for RenameDestinationGuard {
+        fn drop(&mut self) {
+            RENAME_DESTINATIONS.lock().remove(&self.0);
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn observe_rename_destination(source: &Path, hook: impl FnOnce(&Path) + Send + 'static) -> RenameDestinationGuard {
+        assert!(
+            RENAME_DESTINATIONS
+                .lock()
+                .insert(source.to_path_buf(), Box::new(hook))
+                .is_none()
+        );
+        RenameDestinationGuard(source.to_path_buf())
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn drain_namespace_key(path: &Path) {
+        drop(super::acquire_namespace_mutation_lease(path).await);
+    }
+
+    #[cfg(test)]
+    pub(super) fn run_rename_destination(source: &Path, destination: &Path) {
+        let hook = RENAME_DESTINATIONS.lock().remove(source);
+        if let Some(hook) = hook {
+            hook(destination);
+        }
+    }
+}
+
 #[cfg(all(test, windows))]
 pub(crate) mod windows_rename_test_hooks {
     use super::*;
@@ -576,6 +664,7 @@ impl OpenedDstDirFsyncGroup {
 }
 
 struct DstDirFsyncWaiter {
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
     result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
 }
 
@@ -634,6 +723,7 @@ impl DstDirFsyncGroupCommit {
     fn enqueue_opened(
         &self,
         opened: OpenedDstDirFsyncGroup,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
     ) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
         let (result_tx, result_rx) = oneshot::channel();
         let mut registry = self.inner.lock();
@@ -664,7 +754,10 @@ impl DstDirFsyncGroupCommit {
             group
         };
         let mut group_state = group.inner.lock();
-        group_state.pending.push_back(DstDirFsyncWaiter { result_tx });
+        group_state.pending.push_back(DstDirFsyncWaiter {
+            result_tx,
+            namespace_owner,
+        });
         let start_worker = !group_state.worker_running;
         if start_worker {
             group_state.worker_running = true;
@@ -686,7 +779,13 @@ impl DstDirFsyncGroupCommit {
     fn remove_idle_group(&self, group: &Arc<DstDirFsyncGroup>) {
         let mut registry = self.inner.lock();
         let group_state = group.inner.lock();
-        if !group_state.worker_running && group_state.pending.is_empty() {
+        if !group_state.worker_running
+            && group_state.pending.is_empty()
+            && registry
+                .groups
+                .get(&group.key)
+                .is_some_and(|registered| Arc::ptr_eq(registered, group))
+        {
             registry.groups.remove(&group.key);
         }
     }
@@ -709,16 +808,20 @@ impl DstDirFsyncGroupCommit {
         &self,
         dir: &Path,
     ) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
-        self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?)
+        self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None)
     }
 }
 
 #[cfg(unix)]
-async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
+async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
     fsync_spawn_blocking(move || {
+        // The batch worker may be cancelled while this syscall is still running.
+        let _namespace_owners = namespace_owners;
+        #[cfg(all(test, not(windows)))]
+        prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -733,66 +836,118 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
 }
 
 #[cfg(not(unix))]
-async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
+async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
+    let _namespace_owners = namespace_owners;
     fsync_dir(&group.dir).await
 }
 
-async fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) {
+struct DstDirFsyncWorkerGuard {
-    loop {
+    group: Arc<DstDirFsyncGroup>,
-        #[cfg(test)]
+    in_flight: usize,
-        fsync_dir_recorder::run_before_group_batch(&group.dir);
+    armed: bool,
-        tokio::task::yield_now().await;
+}
-        let batch: Vec<DstDirFsyncWaiter> = {
+
-            let mut group_state = group.inner.lock();
+impl Drop for DstDirFsyncWorkerGuard {
-            group_state.pending.drain(..).collect()
+    fn drop(&mut self) {
-        };
+        if !self.armed {
-        if batch.is_empty() {
+            return;
-            let mut group_state = group.inner.lock();
+        }
+        // Cancellation must release queued owners, but the physical batch keeps
+        // its own owners until its blocking syscall returns.
+        let pending = {
+            let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock();
+            let mut group_state = self.group.inner.lock();
+            let pending = std::mem::take(&mut group_state.pending);
             group_state.worker_running = false;
-            drop(group_state);
+            if registry
-            DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
+                .groups
-            return;
+                .get(&self.group.key)
-        }
+                .is_some_and(|group| Arc::ptr_eq(group, &self.group))
-
+            {
-        #[cfg(test)]
+                registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight);
-        fsync_dir_recorder::record_grouped(&group.dir, batch.len());
+                registry.groups.remove(&self.group.key);
-        let result = fsync_open_dst_dir_group(&group)
-            .await
-            .map_err(SharedDstDirFsyncError::from_error);
-        let batch_len = batch.len();
-        DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
-
-        let should_stop = {
-            let mut group_state = group.inner.lock();
-            if group_state.pending.is_empty() {
-                group_state.worker_running = false;
-                true
-            } else {
-                false
             }
+            pending
         };
-        if should_stop {
+        // Lease and channel destructors must run outside the registry locks.
-            DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
+        drop(pending);
-        }
+    }
-        for waiter in batch {
+}
-            let _ = waiter.result_tx.send(result.clone());
+
-        }
+fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) -> impl std::future::Future<Output = ()> {
-        if should_stop {
+    // Capture before spawning: shutdown may drop the future without polling it.
-            return;
+    let worker_guard = DstDirFsyncWorkerGuard {
+        group: group.clone(),
+        in_flight: 0,
+        armed: true,
+    };
+    async move {
+        let mut worker_guard = worker_guard;
+        loop {
+            #[cfg(test)]
+            fsync_dir_recorder::run_before_group_batch(&group.dir);
+            tokio::task::yield_now().await;
+            let mut batch: Vec<DstDirFsyncWaiter> = {
+                let mut group_state = group.inner.lock();
+                group_state.pending.drain(..).collect()
+            };
+            if batch.is_empty() {
+                let mut group_state = group.inner.lock();
+                worker_guard.armed = false;
+                group_state.worker_running = false;
+                drop(group_state);
+                DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
+                return;
+            }
+            worker_guard.in_flight = batch.len();
+
+            #[cfg(test)]
+            fsync_dir_recorder::record_grouped(&group.dir, batch.len());
+            let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect();
+            let result = fsync_open_dst_dir_group(&group, namespace_owners)
+                .await
+                .map_err(SharedDstDirFsyncError::from_error);
+            let batch_len = batch.len();
+            DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
+            worker_guard.in_flight = 0;
+
+            let should_stop = {
+                let mut group_state = group.inner.lock();
+                if group_state.pending.is_empty() {
+                    worker_guard.armed = false;
+                    group_state.worker_running = false;
+                    true
+                } else {
+                    false
+                }
+            };
+            if should_stop {
+                DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
+            }
+            for waiter in batch {
+                let _ = waiter.result_tx.send(result.clone());
+            }
+            if should_stop {
+                return;
+            }
         }
     }
 }
 
-async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
+async fn fsync_dst_dir_group_commit_with_enabled(
+    dir: impl AsRef<Path>,
+    enabled: bool,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
     if !enabled {
-        return fsync_dir(dir).await;
+        return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await;
     }
 
     let dir = dir.as_ref().to_path_buf();
     let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir))
         .await
         .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??;
-    let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?;
+    let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?;
     if let Some(group) = worker {
         tokio::spawn(run_dst_dir_fsync_group_worker(group));
     }
@@ -804,8 +959,11 @@ async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled:
     }
 }
 
-pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef<Path>) -> io::Result<()> {
+pub(crate) async fn fsync_dst_dir_group_commit(
-    fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await
+    dir: impl AsRef<Path>,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
+    fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await
 }
 
 pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
@@ -814,7 +972,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
     admission: &FileSyncAdmission,
 ) -> io::Result<()> {
     if dst_dir_fsync_group_commit_enabled() {
-        fsync_dst_dir_group_commit_with_enabled(dir, true).await
+        fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await
     } else {
         fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await
     }
@@ -822,7 +980,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
 
 #[cfg(test)]
 pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
-    fsync_dst_dir_group_commit_with_enabled(dir, enabled).await
+    fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await
 }
 
 #[cfg(test)]
@@ -1229,6 +1387,8 @@ pub(crate) struct NamespaceMutationLease {
     _namespace_guard: OwnedMutexGuard<()>,
     _volume_guard: Option<OwnedRwLockReadGuard<()>>,
     external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
+    // Independent of the quota claim; both survive cancellation of the waiter.
+    _namespace_owner: Option<Arc<dyn Send + Sync>>,
 }
 
 impl NamespaceMutationLease {
@@ -1238,10 +1398,18 @@ impl NamespaceMutationLease {
 }
 
 async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
+    acquire_namespace_mutation_lease_with_owner(path, None).await
+}
+
+pub(in crate::disk) async fn acquire_namespace_mutation_lease_with_owner(
+    path: &Path,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> Arc<NamespaceMutationLease> {
     Arc::new(NamespaceMutationLease {
         _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
         _volume_guard: None,
         external_guard: Mutex::new(None),
+        _namespace_owner: namespace_owner,
     })
 }
 
@@ -1251,6 +1419,15 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
     root: &Path,
     volume: &str,
     destination_object: &Path,
+) -> Arc<NamespaceMutationLease> {
+    acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await
+}
+
+pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
+    root: &Path,
+    volume: &str,
+    destination_object: &Path,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
 ) -> Arc<NamespaceMutationLease> {
     let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await;
     let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
@@ -1258,6 +1435,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
         _namespace_guard: namespace_guard,
         _volume_guard: Some(volume_guard),
         external_guard: Mutex::new(None),
+        _namespace_owner: namespace_owner,
     })
 }
 
@@ -1747,6 +1925,105 @@ pub async fn rename_all(
     Ok(())
 }
 
+pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
+    #[cfg(unix)]
+    {
+        if namespace_owner.is_none() {
+            return fsync_dir(path).await;
+        }
+        let path = path.to_path_buf();
+        fsync_spawn_blocking(move || {
+            let _namespace_owner = namespace_owner;
+            fsync_dir_std(path)
+        })
+        .await?
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = namespace_owner;
+        fsync_dir(path).await
+    }
+}
+
+/// Retain namespace ownership in the actual filesystem executor after timeout.
+pub(crate) async fn remove_file_with_owner(
+    path: impl AsRef<Path>,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
+    if namespace_owner.is_none() {
+        return tokio::fs::remove_file(path).await;
+    }
+    let path = path.as_ref().to_path_buf();
+    let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
+    run_blocking_namespace_operation(lease, move || {
+        #[cfg(all(test, not(windows)))]
+        prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
+        std::fs::remove_file(path)
+    })
+    .await
+}
+
+/// Retain namespace ownership in the actual filesystem executor after timeout.
+pub(crate) async fn remove_dir_with_owner(
+    path: impl AsRef<Path>,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
+    if namespace_owner.is_none() {
+        return tokio::fs::remove_dir(path).await;
+    }
+    let path = path.as_ref().to_path_buf();
+    let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
+    run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
+}
+
+/// Preserve raw rename semantics while retaining a counted owner in the syscall.
+/// Unlike reliable rename, this never creates parents or retries a missing source.
+pub(in crate::disk) async fn rename_with_namespace_owner(
+    src: &Path,
+    dst: &Path,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
+    if namespace_owner.is_none() {
+        return tokio::fs::rename(src, dst).await;
+    }
+    let src = src.to_path_buf();
+    let dst = dst.to_path_buf();
+    let lease = acquire_namespace_mutation_lease_with_owner(&dst, namespace_owner).await;
+    run_blocking_namespace_operation(lease, move || {
+        #[cfg(all(test, not(windows)))]
+        {
+            prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src);
+            prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst);
+        }
+        std::fs::rename(src, dst)
+    })
+    .await
+}
+
+pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
+    path: &Path,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> io::Result<()> {
+    if namespace_owner.is_none() {
+        return tokio::fs::create_dir_all(path).await;
+    }
+    let path = path.to_path_buf();
+    let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
+    run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
+}
+
+#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
+pub(crate) async fn rename_all_with_owner(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+    publication_root: &PublicationRoot,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> Result<()> {
+    let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
+    rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
+}
+
 pub(crate) async fn rename_all_with_lease(
     src_file_path: impl AsRef<Path>,
     dst_file_path: impl AsRef<Path>,
@@ -1939,6 +2216,8 @@ pub(crate) async fn rename_all_with_prepared_source(
         move || {
             validate_prepared_rename_source(&prepared_source, &src_file_path)?;
             let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            #[cfg(test)]
+            prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
             rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
@@ -1977,6 +2256,32 @@ pub async fn rename_all_ignore_missing_source(
     }
 }
 
+#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)]
+pub(crate) async fn rename_all_ignore_missing_source_with_owner(
+    src_file_path: impl AsRef<Path>,
+    dst_file_path: impl AsRef<Path>,
+    base_dir: impl AsRef<Path>,
+    publication_root: &PublicationRoot,
+    namespace_owner: Option<Arc<dyn Send + Sync>>,
+) -> Result<()> {
+    let src_file_path = src_file_path.as_ref();
+    let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
+    match reliable_rename_inner_with_lease(
+        src_file_path.to_path_buf(),
+        dst_file_path.as_ref().to_path_buf(),
+        base_dir.as_ref().to_path_buf(),
+        publication_root.clone(),
+        false,
+        lease,
+    )
+    .await
+    {
+        Ok(()) => Ok(()),
+        Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()),
+        Err(err) => Err(to_file_error(err).into()),
+    }
+}
+
 #[cfg(windows)]
 pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool {
     let Some(source_parent) = src_file_path.parent() else {
@@ -2042,6 +2347,13 @@ async fn reliable_rename_inner_with_lease(
         let base_dir = base_dir.clone();
         move || {
             let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            #[cfg(all(test, not(windows)))]
+            prepared_publication_test_hooks::run_rename_destination(&src_file_path, &dst_file_path);
+            #[cfg(all(test, not(windows)))]
+            {
+                prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
+                prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
+            }
             rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
@@ -6136,6 +6448,245 @@ mod tests {
         wait_for_dst_dir_fsync_group_commit_idle().await;
     }
 
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial(dst_dir_fsync_group_commit)]
+    async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() {
+        let temp_dir = tempdir().expect("fixture directory");
+        let dir = temp_dir.path().canonicalize().expect("canonical fsync path");
+        let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let first_owner = first_ctx.begin_namespace_commit();
+        let second_owner = second_ctx.begin_namespace_commit();
+        let first_probe = Arc::downgrade(&first_owner);
+        let second_probe = Arc::downgrade(&second_owner);
+        let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_opened(
+                OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"),
+                Some(first_owner),
+            )
+            .expect("queue first real waiter");
+        let group = group.expect("first waiter starts the group");
+        let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_opened(
+                OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"),
+                Some(second_owner),
+            )
+            .expect("queue second real waiter");
+        assert!(second_worker.is_none(), "same directory must join the same batch");
+        assert_eq!(group.inner.lock().pending.len(), 2);
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let _hook =
+            prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || {
+                let _ = entered_tx.send(());
+                let _ = release_rx.recv();
+            });
+        let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone()));
+        tokio::time::timeout(Duration::from_secs(5), entered_rx)
+            .await
+            .expect("batch must reach its physical fsync")
+            .expect("physical fsync entry");
+        assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]);
+        assert!(
+            group.inner.lock().pending.is_empty(),
+            "both waiters were transferred into the physical batch"
+        );
+        let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let queued_owner = queued_ctx.begin_namespace_commit();
+        let queued_probe = Arc::downgrade(&queued_owner);
+        let queued_generation = queued_ctx.namespace_commit_generation();
+        let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_opened(
+                OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"),
+                Some(queued_owner),
+            )
+            .expect("queue a waiter after the physical batch was frozen");
+        assert!(queued_worker.is_none());
+        assert_eq!(group.inner.lock().pending.len(), 1);
+        drop((first_rx, second_rx));
+        worker.abort();
+        assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled());
+        assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation");
+        assert!(queued_probe.upgrade().is_none());
+        assert!(!queued_ctx.namespace_commits_pending());
+        assert!(queued_ctx.namespace_commit_generation() > queued_generation);
+        assert!(group.inner.lock().pending.is_empty());
+        assert!(!group.inner.lock().worker_running);
+        assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
+        let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some();
+        let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some();
+        let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation());
+        drop(release_tx);
+        tokio::time::timeout(Duration::from_secs(5), async {
+            while Arc::strong_count(&group.dir_file) != 1
+                || first_probe.upgrade().is_some()
+                || second_probe.upgrade().is_some()
+                || first_ctx.namespace_commits_pending()
+                || second_ctx.namespace_commits_pending()
+            {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("physical fsync must release every batch owner");
+        assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute");
+        assert!(
+            first_pending && second_pending,
+            "one physical batch must preserve both independent namespace owners"
+        );
+        assert!(!first_ctx.namespace_commits_pending());
+        assert!(!second_ctx.namespace_commits_pending());
+        assert!(first_ctx.namespace_commit_generation() > generations.0);
+        assert!(second_ctx.namespace_commit_generation() > generations.1);
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial(dst_dir_fsync_group_commit)]
+    async fn grouped_fsync_unpolled_worker_releases_queued_owner() {
+        let temp_dir = tempdir().expect("fixture directory");
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let owner = ctx.begin_namespace_commit();
+        let probe = Arc::downgrade(&owner);
+        let generation = ctx.namespace_commit_generation();
+        let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_opened(
+                OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"),
+                Some(owner),
+            )
+            .expect("queue a real waiter");
+        let group = group.expect("first waiter starts the group");
+        let worker = run_dst_dir_fsync_group_worker(group.clone());
+        assert!(ctx.namespace_commits_pending());
+        drop(worker);
+        assert!(rx.await.is_err(), "shutdown before first poll must release the waiter");
+        assert!(probe.upgrade().is_none());
+        assert!(!ctx.namespace_commits_pending());
+        assert!(ctx.namespace_commit_generation() > generation);
+        assert!(group.inner.lock().pending.is_empty());
+        assert!(!group.inner.lock().worker_running);
+        assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
+        assert!(
+            fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(),
+            "the dropped future must not dispatch a physical batch"
+        );
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn stale_idle_group_cleanup_preserves_successor_registration() {
+        let temp_dir = tempdir().expect("fixture directory");
+        let registry = DstDirFsyncGroupCommit::default();
+        let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker");
+        let old_group = first_worker.expect("first waiter starts a worker");
+        // W1 has completed its batch and marked G idle, but has not cleaned G up.
+        let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter");
+        registry.complete_batch(1);
+        old_group.inner.lock().worker_running = false;
+
+        let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker");
+        let reused_group = second_worker.expect("idle G starts another worker");
+        assert!(Arc::ptr_eq(&old_group, &reused_group));
+        let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter");
+        registry.complete_batch(1);
+        reused_group.inner.lock().worker_running = false;
+        registry.remove_idle_group(&reused_group);
+        assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G");
+        assert!(second_waiter.result_tx.send(Ok(())).is_ok());
+        assert!(second_rx.try_recv().expect("second worker reports completion").is_ok());
+
+        let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor");
+        let successor = successor_worker.expect("successor starts a new group");
+        assert!(!Arc::ptr_eq(&old_group, &successor));
+        assert_eq!(registry.counts_for_test(), (1, 1));
+        // W1 resumes with its old Arc after W2 removed G and W3 installed G2.
+        registry.remove_idle_group(&old_group);
+        assert!(first_waiter.result_tx.send(Ok(())).is_ok());
+        assert!(first_rx.try_recv().expect("first worker reports completion").is_ok());
+        assert!(
+            registry
+                .inner
+                .lock()
+                .groups
+                .get(&successor.key)
+                .is_some_and(|registered| Arc::ptr_eq(registered, &successor)),
+            "stale cleanup must retain the exact successor Arc"
+        );
+        assert_eq!(registry.counts_for_test(), (1, 1));
+        assert!(successor.inner.lock().worker_running);
+        assert_eq!(successor.inner.lock().pending.len(), 1);
+        assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)));
+
+        let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor");
+        assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3");
+        assert_eq!(successor.inner.lock().pending.len(), 2);
+        assert_eq!(registry.counts_for_test(), (1, 2));
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial(dst_dir_fsync_group_commit)]
+    async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() {
+        wait_for_dst_dir_fsync_group_commit_idle().await;
+        let temp_dir = tempdir().expect("fixture directory");
+        let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_for_test(temp_dir.path())
+            .expect("enqueue old group");
+        let old_group = old_worker.expect("old group starts a worker");
+        tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone()))
+            .await
+            .expect("old worker must finish its actual fsync");
+        assert!(old_rx.await.expect("old worker reports completion").is_ok());
+        assert!(fsync_dir_recorder::was_fsynced(temp_dir.path()));
+        assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]);
+        assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
+
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let owner = ctx.begin_namespace_commit();
+        let probe = Arc::downgrade(&owner);
+        let generation = ctx.namespace_commit_generation();
+        let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT
+            .enqueue_opened(
+                OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"),
+                Some(owner),
+            )
+            .expect("enqueue successor owner");
+        let successor = successor_worker.expect("successor starts a new group");
+        assert!(!Arc::ptr_eq(&old_group, &successor));
+        let worker = run_dst_dir_fsync_group_worker(successor.clone());
+        // The stale Arc represents W1 resuming after another worker removed G.
+        DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group);
+        assert!(ctx.namespace_commits_pending());
+        assert!(probe.upgrade().is_some());
+        drop(worker);
+        let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx)
+            .await
+            .expect("dropping the unpolled worker must release its channel")
+            .is_err();
+        let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test();
+        let owner_released = probe.upgrade().is_none();
+        let namespace_pending = ctx.namespace_commits_pending();
+        let generation_after_drop = ctx.namespace_commit_generation();
+        let successor_pending = successor.inner.lock().pending.len();
+        let worker_running = successor.inner.lock().worker_running;
+        // Preserve the observed result before cleanup, so a RED run cannot leak
+        // its phantom count into unrelated tests in the same process.
+        clear_dst_dir_fsync_group_commit_for_test();
+        assert!(channel_closed);
+        assert!(owner_released);
+        assert!(!namespace_pending);
+        assert!(generation_after_drop > generation);
+        assert_eq!(successor_pending, 0);
+        assert!(!worker_running);
+        assert_eq!(
+            fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()),
+            vec![1],
+            "dropping the successor before its first poll must not dispatch another fsync"
+        );
+        assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter");
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial_test::serial(dst_dir_fsync_group_commit)]
     async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() {

crates/ecstore/src/io_support/bitrot.rs

@@ -13,12 +13,15 @@
 // limitations under the License.
 
 use crate::diagnostics::get::{
-    GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
+    GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
-    GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
-    GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
     GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
     GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
 };
+#[cfg(unix)]
+use crate::diagnostics::get::{
+    GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
+    GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
+};
 #[cfg(feature = "hotpath")]
 use crate::disk::FileWriter;
 use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
@@ -406,11 +409,17 @@ async fn open_disk_reader(
             path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
             metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
             metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
+            #[cfg(unix)]
             blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
+            #[cfg(unix)]
             blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
+            #[cfg(unix)]
             file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
+            #[cfg(unix)]
             mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
+            #[cfg(unix)]
             mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
+            #[cfg(unix)]
             direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
         });
         let mmap_result = {

crates/ecstore/src/services/notification_sys.rs

@@ -72,6 +72,29 @@ const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
 /// service must not advertise this version until the conditional writer from
 /// rustfs/backlog#684 is available.
 const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
+
+fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
+    configured
+        .filter(|seconds| *seconds > 0)
+        .unwrap_or(rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS)
+        .min(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS)
+}
+
+fn admin_peer_probe_timeout() -> Duration {
+    let configured = rustfs_utils::get_env_opt_u64_with_aliases(rustfs_config::ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, &[]);
+    let seconds = resolve_admin_peer_probe_timeout_secs(configured);
+    Duration::from_secs(seconds)
+}
+
+fn remaining_admin_peer_probe_timeout(deadline: Instant) -> Option<Duration> {
+    remaining_admin_peer_probe_timeout_at(deadline, Instant::now())
+}
+
+fn remaining_admin_peer_probe_timeout_at(deadline: Instant, now: Instant) -> Option<Duration> {
+    let remaining = deadline.saturating_duration_since(now);
+    (!remaining.is_zero()).then_some(remaining)
+}
+
 type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
 
 fn cross_pool_fence_policy_results(
@@ -1538,7 +1561,7 @@ impl NotificationSys {
     {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
-        let peer_timeout = Duration::from_secs(5);
+        let peer_timeout = admin_peer_probe_timeout();
 
         for (idx, client) in self.peer_clients.iter().enumerate() {
             let endpoints = endpoints.clone();
@@ -1546,7 +1569,9 @@ impl NotificationSys {
             futures.push(async move {
                 if let Some(client) = client {
                     let host = client.host.to_string();
-                    match timeout(peer_timeout, client.local_storage_info()).await {
+                    let deadline = Instant::now() + peer_timeout;
+                    let probe_timeout = remaining_admin_peer_probe_timeout(deadline).unwrap_or_default();
+                    match timeout(probe_timeout, client.local_storage_info()).await {
                         Ok(Ok(mut info)) => {
                             normalize_and_cache_peer_storage_info(cache, &host, &mut info);
                             Some(info)
@@ -1557,7 +1582,6 @@ impl NotificationSys {
                         }
                         Err(_) => {
                             warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
-                            client.evict_connection().await;
                             handle_peer_failure(cache, &host, &endpoints)
                         }
                     }
@@ -1583,7 +1607,7 @@ impl NotificationSys {
     pub async fn server_info(&self) -> Vec<ServerProperties> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
-        let peer_timeout = Duration::from_secs(5);
+        let peer_timeout = admin_peer_probe_timeout();
 
         for (idx, client) in self.peer_clients.iter().enumerate() {
             let host = self
@@ -1600,12 +1624,23 @@ impl NotificationSys {
                     };
                 };
 
+                let deadline = Instant::now() + peer_timeout;
+                let Some(first_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+
                 // First attempt. A single evicted or half-open internode channel
                 // is enough to fail one probe and, before retrying, would drop
-                // the member to unknown/offline for this whole snapshot. So on any
+                // the member to unknown/offline for this whole snapshot. On a
-                // first-attempt failure we evict the channel and re-dial once
+                // quick failure we evict the channel and re-dial once before
-                // before falling back (rustfs/backlog#1049, P1-B).
+                // falling back (rustfs/backlog#1049, P1-B). A slow attempt
-                match timeout(peer_timeout, client.server_info()).await {
+                // consumes the round budget and therefore does not trigger a
+                // second full wait or an asynchronous eviction side effect.
+                match timeout(first_timeout, client.server_info()).await {
                     Ok(Ok(info)) => {
                         return PeerServerInfoProbe { host, result: Ok(info) };
                     }
@@ -1619,14 +1654,37 @@ impl NotificationSys {
                 // `evict_connection` would leave that gate up and the retry would
                 // fast-fail with "temporarily offline" instead of reconnecting
                 // (rustfs/backlog#1049 P1-B).
-                client.prepare_retry().await;
+                let Some(retry_budget) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+                // Bound connection-cache cleanup too. The helper clears the offline gate even
+                // when eviction itself times out, so cancellation cannot strand this peer in
+                // fast-fail mode.
+                if !client.prepare_retry_with_timeout(retry_budget).await {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                }
 
                 // Second and final attempt on the fresh channel.
-                match timeout(peer_timeout, client.server_info()).await {
+                let Some(retry_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+                match timeout(retry_timeout, client.server_info()).await {
                     Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
                     Ok(Err(err)) => {
                         warn!("peer {host} server_info failed after retry: {err}");
-                        let health = peer_disk_health(&host).await;
+                        let health = peer_disk_health_with_deadline(&host, deadline).await;
                         PeerServerInfoProbe {
                             host,
                             result: Err(PeerServerInfoProbeFailure::Rpc { health }),
@@ -1634,8 +1692,7 @@ impl NotificationSys {
                     }
                     Err(_) => {
                         warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
-                        client.evict_connection().await;
+                        let health = peer_disk_health_with_deadline(&host, deadline).await;
-                        let health = peer_disk_health(&host).await;
                         PeerServerInfoProbe {
                             host,
                             result: Err(PeerServerInfoProbeFailure::Rpc { health }),
@@ -3023,6 +3080,11 @@ async fn peer_disk_health(host: &str) -> Option<PeerDiskHealth> {
     }
 }
 
+async fn peer_disk_health_with_deadline(host: &str, deadline: Instant) -> Option<PeerDiskHealth> {
+    let remaining = remaining_admin_peer_probe_timeout(deadline)?;
+    timeout(remaining, peer_disk_health(host)).await.ok().flatten()
+}
+
 /// Handle a peer failure for server_info: return cached data if available, or
 /// classify the member as `unknown` / `degraded` / `offline` depending on how
 /// many consecutive probes have failed and whether the peer's drives are still
@@ -4017,6 +4079,37 @@ mod tests {
         }
     }
 
+    #[test]
+    fn admin_peer_probe_timeout_rejects_zero_and_caps_large_values() {
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(None),
+            rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(Some(0)),
+            rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(Some(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + 1)),
+            rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(resolve_admin_peer_probe_timeout_secs(Some(7)), 7);
+    }
+
+    #[tokio::test]
+    async fn admin_peer_probe_health_fallback_respects_expired_deadline() {
+        let deadline = Instant::now();
+        assert!(peer_disk_health_with_deadline("peer-1", deadline).await.is_none());
+    }
+
+    #[test]
+    fn admin_peer_probe_deadline_is_shared_across_attempts() {
+        let start = Instant::now();
+        let deadline = start + Duration::from_secs(10);
+        assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(6)).is_some());
+        assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(10)).is_none());
+    }
+
     #[tokio::test]
     async fn call_peer_with_timeout_returns_value_when_fast() {
         let result = call_peer_with_timeout(

crates/ecstore/src/services/tier/test_util.rs

@@ -56,6 +56,7 @@
 
 use std::collections::HashMap;
 use std::io::Cursor;
+#[cfg(feature = "test-util")]
 use std::path::Path;
 use std::sync::{
     Arc,
@@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt;
 use tokio::sync::{Mutex, Notify, RwLock};
 use uuid::Uuid;
 
+#[cfg(feature = "test-util")]
 use crate::disk::endpoint::Endpoint;
+#[cfg(feature = "test-util")]
 use crate::disk::format::FormatV3;
+#[cfg(feature = "test-util")]
 use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
 use crate::services::tier::tier::TierConfigMgr;
 use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
 use crate::services::tier::warm_backend::{
     TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
 };
+#[cfg(feature = "test-util")]
 use rustfs_filemeta::FileMeta;
 use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
+#[cfg(feature = "test-util")]
 use rustfs_utils::path::path_join_buf;
 
 /// One-shot barrier before rejected transition cleanup resolves its ECStore.
+#[cfg(feature = "test-util")]
 pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
 
+#[cfg(feature = "test-util")]
 impl TransitionCleanupStoreBarrier {
     /// Install the barrier for the next rejected transition cleanup.
     pub fn install() -> Self {
@@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier {
 }
 
 /// Default polling cadence used by the `wait_for_*` helpers.
+#[cfg(feature = "test-util")]
 const POLL_INTERVAL: Duration = Duration::from_millis(50);
 
 /// A fault to inject into [`MockWarmBackend`] operations.
@@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard {
 }
 
 /// One-shot barrier that pauses a mock tier PUT after storing its remote body.
+#[cfg(feature = "test-util")]
 pub struct MockPutBarrier {
     state: Arc<MockPutBarrierState>,
 }
 
+#[cfg(feature = "test-util")]
 impl MockPutBarrier {
     /// Wait until the remote body is stored and the PUT is paused before returning.
     pub async fn wait_until_paused(&self) {
@@ -226,6 +237,7 @@ impl MockPutBarrier {
     }
 }
 
+#[cfg(feature = "test-util")]
 impl Drop for MockPutBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -258,10 +270,12 @@ impl Drop for MockGetBarrier {
 }
 
 /// One-shot barrier that pauses and then fails a mock tier DELETE.
+#[cfg(feature = "test-util")]
 pub struct MockRemoveBarrier {
     state: Arc<MockRemoveBarrierState>,
 }
 
+#[cfg(feature = "test-util")]
 impl MockRemoveBarrier {
     /// Wait until DELETE reaches the deterministic failure point.
     pub async fn wait_until_paused(&self) {
@@ -283,6 +297,7 @@ impl MockRemoveBarrier {
     }
 }
 
+#[cfg(feature = "test-util")]
 impl Drop for MockRemoveBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -306,6 +321,7 @@ impl MockWarmBackend {
     }
 
     /// Arm a one-shot pause after the next tier PUT stores its remote body.
+    #[cfg(feature = "test-util")]
     pub async fn arm_put_barrier(&self) -> MockPutBarrier {
         let state = Arc::new(MockPutBarrierState::default());
         *self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
@@ -313,6 +329,7 @@ impl MockWarmBackend {
     }
 
     /// Pause and then fail the next DELETE after it reaches the backend.
+    #[cfg(feature = "test-util")]
     pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
         let state = Arc::new(MockRemoveBarrierState::default());
         let mut barrier = self.inner.remove_barrier.lock().await;
@@ -323,6 +340,7 @@ impl MockWarmBackend {
 
     /// Arm a one-shot pause before the next tier GET, then return an error
     /// after the test releases it.
+    #[cfg(feature = "test-util")]
     pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
         let state = Arc::new(MockGetBarrierState {
             fail_after_release: true,
@@ -343,6 +361,7 @@ impl MockWarmBackend {
     // ---- fault injection -------------------------------------------------
 
     /// Replace the entire fault configuration.
+    #[cfg(feature = "test-util")]
     pub async fn set_faults(&self, faults: FaultConfig) {
         *self.inner.faults.lock().await = faults;
     }
@@ -353,6 +372,7 @@ impl MockWarmBackend {
     }
 
     /// Toggle "HTTP 5xx" server errors on every operation.
+    #[cfg(feature = "test-util")]
     pub async fn set_server_error(&self, server_error: bool) {
         self.inner.faults.lock().await.server_error = server_error;
     }
@@ -363,11 +383,13 @@ impl MockWarmBackend {
     }
 
     /// Set (or clear, with `None`) injected latency applied before each op.
+    #[cfg(feature = "test-util")]
     pub async fn set_latency(&self, latency: Option<Duration>) {
         self.inner.faults.lock().await.latency = latency;
     }
 
     /// Clear all injected faults, restoring healthy behaviour.
+    #[cfg(feature = "test-util")]
     pub async fn clear_faults(&self) {
         *self.inner.faults.lock().await = FaultConfig::default();
     }
@@ -375,6 +397,7 @@ impl MockWarmBackend {
     /// Limit how many body bytes a successful mock PUT consumes. `None` drains
     /// the complete body. This models a backend that incorrectly accepts a
     /// truncated stream while still returning success.
+    #[cfg(feature = "test-util")]
     pub async fn set_put_read_limit(&self, limit: Option<usize>) {
         *self.inner.put_read_limit.lock().await = limit;
     }
@@ -395,12 +418,14 @@ impl MockWarmBackend {
     }
 
     /// Reject non-empty remote versions before transition metadata is committed.
+    #[cfg(feature = "test-util")]
     pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
         self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
     }
 
     /// Reject the next non-empty remote version validation without changing
     /// subsequent exact-version backend cleanup behavior.
+    #[cfg(feature = "test-util")]
     pub fn reject_next_non_empty_remote_version_validation(&self) {
         self.inner
             .reject_non_empty_remote_version_validations
@@ -438,6 +463,7 @@ impl MockWarmBackend {
     }
 
     /// Clear the operation log without touching stored objects or faults.
+    #[cfg(feature = "test-util")]
     pub async fn clear_op_log(&self) {
         self.inner.op_log.lock().await.clear();
     }
@@ -459,11 +485,13 @@ impl MockWarmBackend {
     }
 
     /// Return the exact object/version pairs produced by successful tier PUTs.
+    #[cfg(feature = "test-util")]
     pub async fn put_versions(&self) -> Vec<(String, String)> {
         self.inner.put_versions.lock().await.clone()
     }
 
     /// Return the exact object/version pairs passed to successful tier removes.
+    #[cfg(feature = "test-util")]
     pub async fn remove_versions(&self) -> Vec<(String, String)> {
         self.inner.remove_versions.lock().await.clone()
     }
@@ -475,6 +503,7 @@ impl MockWarmBackend {
 
     /// Number of `get` calls recorded — useful to assert restore reads hit the
     /// local copy rather than the remote tier.
+    #[cfg(feature = "test-util")]
     pub async fn get_count(&self) -> usize {
         self.inner
             .op_log
@@ -486,6 +515,7 @@ impl MockWarmBackend {
     }
 
     /// Number of `put` calls recorded.
+    #[cfg(feature = "test-util")]
     pub async fn put_count(&self) -> usize {
         self.inner
             .op_log
@@ -499,6 +529,7 @@ impl MockWarmBackend {
     // ---- storage inspection ---------------------------------------------
 
     /// Whether the backend currently stores `object`.
+    #[cfg(feature = "test-util")]
     pub async fn contains(&self, object: &str) -> bool {
         self.inner.objects.lock().await.contains_key(object)
     }
@@ -509,11 +540,13 @@ impl MockWarmBackend {
     }
 
     /// A clone of the stored object, if present.
+    #[cfg(feature = "test-util")]
     pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
         self.inner.objects.lock().await.get(object).cloned()
     }
 
     /// A clone of the raw bytes stored for `object`, if present.
+    #[cfg(feature = "test-util")]
     pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
         self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
     }
@@ -538,6 +571,7 @@ impl MockWarmBackend {
 
     /// Poll until `object` is absent from the backend, or `timeout` elapses.
     /// Returns `true` if the object disappeared within the budget.
+    #[cfg(feature = "test-util")]
     pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
         let deadline = tokio::time::Instant::now() + timeout;
         loop {
@@ -553,6 +587,7 @@ impl MockWarmBackend {
 
     /// Poll until the backend holds exactly `expected` objects, or `timeout`
     /// elapses. Returns `true` if the count was reached within the budget.
+    #[cfg(feature = "test-util")]
     pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
         let deadline = tokio::time::Instant::now() + timeout;
         loop {
@@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
 /// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
 /// free-version count.
 #[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg(feature = "test-util")]
 pub struct TransitionMeta {
     /// `transition_status` (e.g. `"complete"`), empty when not transitioned.
     pub status: String,
@@ -860,6 +896,7 @@ pub struct TransitionMeta {
     pub free_version_count: usize,
 }
 
+#[cfg(feature = "test-util")]
 async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
     // `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
     // with the position recorded in the disk's own format.json, so derive the
@@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
 /// The free-version metadata removal lands asynchronously after the remote
 /// object disappears, so callers typically poll via
 /// [`wait_for_free_version_absence`] instead of asserting a single read.
+#[cfg(feature = "test-util")]
 pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
     let Some(disk) = open_disk(disk_path).await else {
         return 0;
@@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
 /// fields are taken from the newest version that carries a transition record;
 /// if no version is transitioned, they are taken from the current version (and
 /// will be empty).
+#[cfg(feature = "test-util")]
 pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
     let disk = open_disk(disk_path).await?;
     let data = disk
@@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
 /// disk is missing the object or disagrees — this is the shard-consistency
 /// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
 /// four-tuple plus free-version count must match across all erasure shards).
+#[cfg(feature = "test-util")]
 pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
     assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
 
@@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
 
 /// Poll until `object` retains no free versions on `disk_path`, or `timeout`
 /// elapses. Returns `true` if the free versions drained within the budget.
+#[cfg(feature = "test-util")]
 pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
     let deadline = tokio::time::Instant::now() + timeout;
     loop {
@@ -1040,6 +1081,44 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
+        let backend = MockWarmBackend::new();
+        let metadata = HashMap::from([
+            ("content-type".to_string(), "text/plain".to_string()),
+            ("project".to_string(), "archive".to_string()),
+        ]);
+        let version = backend
+            .put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
+            .await
+            .expect("mock PUT should preserve remote metadata");
+        assert_eq!(backend.metadata("object").await, Some(metadata));
+        assert_eq!(
+            backend
+                .probe_transition_candidate_state("object")
+                .await
+                .expect("probe stored object"),
+            TransitionCandidateProbe::VersionedPresent(version)
+        );
+
+        backend.external_remove("object").await;
+        assert_eq!(backend.metadata("object").await, None);
+        assert_eq!(
+            backend
+                .probe_transition_candidate_state("object")
+                .await
+                .expect("probe removed object"),
+            TransitionCandidateProbe::Missing
+        );
+        let operations = backend.op_log().await;
+        assert!(
+            operations
+                .iter()
+                .any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
+        );
+        assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
+    }
+
     #[tokio::test]
     async fn mock_probe_preserves_fault_fail_closed_behavior() {
         let backend = MockWarmBackend::new();

crates/ecstore/src/services/tier/tier.rs

@@ -143,11 +143,12 @@ struct TierDriverBuildBarrier {
 static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
 
 #[cfg(test)]
-type TierDriverTestFactory = Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
+pub(crate) type TierDriverTestFactory =
+    Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
 
 #[cfg(test)]
 tokio::task_local! {
-    static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
+    pub(crate) static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
 }
 
 #[cfg(test)]
@@ -1377,27 +1378,42 @@ async fn ensure_no_authoritative_persisted_references<S>(
 where
     S: TierReferenceProofStore,
 {
-    ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| {
+    ensure_no_authoritative_persisted_references_with(
-        let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
+        api.clone(),
-        Ok((
+        TIER_DELETE_JOURNAL_PREFIX,
-            journal.tier_name.clone(),
+        "tier-delete journal",
-            tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
+        |_object, data| {
-        ))
+            let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
-    })
+            Ok((
+                journal.tier_name.clone(),
+                tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
+            ))
+        },
+    )
     .await?;
-    ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| {
+    ensure_no_authoritative_persisted_references_with(
-        let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
+        api,
-        Ok((
+        TRANSITION_TRANSACTION_RECORD_PREFIX,
-            transaction.tier_name.clone(),
+        "transition transaction",
-            tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets),
+        |object, data| {
-        ))
+            let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
-    })
+            Ok((
+                transaction.tier_name.clone(),
+                tier_persisted_reference_blocks_any_target(
+                    &transaction.tier_name,
+                    Some(transaction.backend_fingerprint),
+                    targets,
+                ),
+            ))
+        },
+    )
     .await
 }
 
 async fn ensure_no_authoritative_persisted_references_with<S, F>(
     api: Arc<S>,
     prefix: &str,
+    reference_kind: &str,
     blocks_target: F,
 ) -> std::result::Result<(), AdminError>
 where
@@ -1426,7 +1442,7 @@ where
                 .map_err(tier_reference_proof_admin_error)?;
             let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?;
             if blocks {
-                return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name));
+                return Err(tier_reference_proof_persisted_in_use_error(&tier_name, reference_kind, &object.name));
             }
         }
         if !page.is_truncated {
@@ -1488,16 +1504,21 @@ fn tier_persisted_reference_blocks_target(
 
 fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
     let mut err = ERR_TIER_BACKEND_IN_USE.clone();
+    let reference_kind = if object.transitioned_object.free_version {
+        "free-version ownership"
+    } else {
+        "transitioned-object"
+    };
     err.message = format!(
-        "Remote tier {tier_name} still has object references, for example {}/{}",
+        "Remote tier {tier_name} still has a {reference_kind} reference, for example {}/{}",
         object.bucket, object.name
     );
     err
 }
 
-fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError {
+fn tier_reference_proof_persisted_in_use_error(tier_name: &str, reference_kind: &str, object: &str) -> AdminError {
     let mut err = ERR_TIER_BACKEND_IN_USE.clone();
-    err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}");
+    err.message = format!("Remote tier {tier_name} still has a {reference_kind} reference, for example {object}");
     err
 }
 
@@ -3685,13 +3706,17 @@ impl TierConfigMgr {
         let manager = handle.read().await;
         let runtime = tier_driver_runtime(handle, &manager);
         let runtime = lock_unpoisoned(&runtime);
-        if !runtime
+        let prepared = runtime
+            .prepared_mutation_blocks
+            .values()
+            .any(|blocked_mutation_id| *blocked_mutation_id == mutation_id);
+        let committed = runtime
             .committed_mutation_blocks
             .values()
-            .any(|mutation_ids| mutation_ids.contains(&mutation_id))
+            .any(|mutation_ids| mutation_ids.contains(&mutation_id));
-        {
+        if !prepared && !committed {
             let mut err = ERR_TIER_INVALID_CONFIG.clone();
-            err.message = "Remote tier committed mutation fence was not installed".to_string();
+            err.message = "Remote tier mutation fence was not installed".to_string();
             return Err(err);
         }
         Ok(MutationBlockAllowance {
@@ -3898,14 +3923,6 @@ impl TierConfigMgr {
         Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, mutation_block_allowance)
     }
 
-    fn begin_tier_transition(
-        handle: &Arc<RwLock<Self>>,
-        manager: &mut Self,
-        changed: HashSet<String>,
-    ) -> std::result::Result<TierPublishTransition, AdminError> {
-        Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None)
-    }
-
     fn begin_tier_transition_with_destinations(
         handle: &Arc<RwLock<Self>>,
         manager: &mut Self,
@@ -4176,7 +4193,7 @@ impl TierConfigMgr {
                     let mut config_lock = config_lock;
                     let coordinated_config_update = config_lock.is_some();
                     let mut update = Some(update);
-                    let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) =
+                    let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, target_tiers) =
                         match mutation {
                             TierCandidateMutation::Prevalidated(prepared) => {
                                 if version != prepared.version {
@@ -4192,9 +4209,8 @@ impl TierConfigMgr {
                                     )));
                                 }
                                 candidate = prepared.candidate;
-                                let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
+                                let target_tiers = {
-                                let mut transition = {
+                                    let manager = handle.read().await;
-                                    let mut manager = handle.write().await;
                                     let mut target_tiers = changed_tier_names(&manager, &candidate);
                                     if let Some(tier_name) = prepared.explicit_tier_name.as_ref()
                                         && (manager.tiers.contains_key(tier_name)
@@ -4203,8 +4219,7 @@ impl TierConfigMgr {
                                     {
                                         target_tiers.insert(tier_name.clone());
                                     }
-                                    Self::begin_tier_transition(&handle, &mut manager, target_tiers)
+                                    target_tiers
-                                        .map_err(TierConfigUpdateError::Publish)?
                                 };
                                 (
                                     prepared.kind,
@@ -4212,7 +4227,7 @@ impl TierConfigMgr {
                                     prepared.force,
                                     prepared.current,
                                     prepared.driver_tier,
-                                    transition,
+                                    target_tiers,
                                 )
                             }
                             mutation => {
@@ -4237,11 +4252,9 @@ impl TierConfigMgr {
                                     last_refreshed_at: candidate.last_refreshed_at,
                                 };
                                 let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
-                                let mut transition = {
+                                let target_tiers = {
-                                    let mut manager = handle.write().await;
+                                    let manager = handle.read().await;
-                                    let target_tiers = mutation.target_tiers(&manager, &candidate);
+                                    mutation.target_tiers(&manager, &candidate)
-                                    Self::begin_tier_transition(&handle, &mut manager, target_tiers)
-                                        .map_err(TierConfigUpdateError::Publish)?
                                 };
                                 let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline)
                                     .await
@@ -4252,7 +4265,7 @@ impl TierConfigMgr {
                                     mutation_force,
                                     current_for_targets,
                                     driver_tier,
-                                    transition,
+                                    target_tiers,
                                 )
                             }
                         };
@@ -4275,28 +4288,83 @@ impl TierConfigMgr {
                     save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref())
                         .await
                         .map_err(TierConfigUpdateError::Save)?;
+                    let mut blocked_target_tiers = target_tiers.clone();
                     if let Some(intent) = coordinator_intent.as_ref() {
-                        TierConfigMgr::apply_prepared_mutation_intent_block(&handle, intent)
+                        blocked_target_tiers.extend(intent.affected_targets.iter().map(|target| target.tier_name.clone()));
+                    }
+                    if let Some(intent) = coordinator_intent.as_ref() {
+                        // `target_tiers` may include a stale local-only manager
+                        // entry that is absent from the persisted proof
+                        // snapshot. Fence that local transition under the same
+                        // mutation ID as well; recovery may discard this
+                        // process-local superset, which advances the revision
+                        // and makes the deferred transition fail closed.
+                        TierConfigMgr::apply_prepared_mutation_intent_block_for_tiers(&handle, intent, &blocked_target_tiers)
                             .await
                             .map_err(TierConfigUpdateError::Publish)?;
                     }
+                    let prepared_mutation_block_allowance = match coordinator_intent.as_ref() {
+                        Some(intent) => Some(
+                            TierConfigMgr::mutation_block_allowance_for(&handle, intent.mutation_id)
+                                .await
+                                .map_err(TierConfigUpdateError::Publish)?,
+                        ),
+                        None => None,
+                    };
+                    // A durable coordinator intent supplies the admission
+                    // fence that lets us defer generation revocation. Keep the
+                    // original early transition for no-intent paths (for
+                    // example, reconciling a stale local manager to an
+                    // idempotently removed persisted tier), where there is no
+                    // Prepared record capable of blocking a new lease.
+                    let (mut transition, deferred_target_tiers) = if coordinator_intent.is_some() {
+                        (None, Some(target_tiers))
+                    } else {
+                        let transition = {
+                            let mut manager = handle.write().await;
+                            Self::begin_tier_transition_with_destinations(
+                                &handle,
+                                &mut manager,
+                                target_tiers,
+                                HashMap::new(),
+                                None,
+                            )
+                            .map_err(TierConfigUpdateError::Publish)?
+                        };
+                        (Some(transition), None)
+                    };
                     if coordinated_config_update {
                         drop(update.take());
                         drop(config_lock.take());
                     }
-                    let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
+                    // The durable Prepared block closes admission before the
-                    if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
+                    // zero-reference proof, but deliberately leaves already
-                        if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await {
+                    // issued generations current. In particular, an exact
+                    // free-version cleanup that has completed remote DELETE
+                    // must still be able to remove its local ownership marker;
+                    // revoking its generation here would strand that marker
+                    // and make this mutation reject its own interrupted work.
+                    if let Some(intent) = coordinator_intent.as_ref()
+                        && let Err(drain_error) =
+                            TierConfigMgr::wait_for_blocked_tier_operation_leases_for_tiers(&handle, &blocked_target_tiers).await
+                    {
+                        if !abort_prepared_tier_mutation(&handle, api.clone(), Some(intent), Vec::new()).await {
                             warn!(
                                 event = "tier_mutation_abort",
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_TIER,
                                 result = "prepared_intent_retained",
-                                coordinator_intent = coordinator_intent.is_some(),
+                                mutation_id = %intent.mutation_id,
                                 "tier mutation lease drain failed and abort was incomplete"
                             );
                         }
                         return Err(TierConfigUpdateError::Publish(drain_error));
+                    } else if let Some(transition) = transition.as_ref() {
+                        let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
+                        transition
+                            .wait_for_active_leases_until(drain_deadline)
+                            .await
+                            .map_err(TierConfigUpdateError::Publish)?;
                     }
                     let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() {
                         let peers = match remote_tier_mutation_peers().await {
@@ -4363,6 +4431,71 @@ impl TierConfigMgr {
                         }
                         return Err(TierConfigUpdateError::Publish(proof_error));
                     }
+                    // No affected-tier lease can start after Prepared, and the
+                    // existing set was drained above. It is now safe to revoke
+                    // the generation for publication without invalidating a
+                    // cleanup between its remote and local commit boundaries.
+                    if transition.is_none() {
+                        let target_tiers = deferred_target_tiers.ok_or_else(|| {
+                            let mut err = ERR_TIER_INVALID_CONFIG.clone();
+                            err.message = "Remote tier mutation lost its deferred transition targets".to_string();
+                            TierConfigUpdateError::Publish(err)
+                        })?;
+                        let mut manager = handle.write().await;
+                        transition = Some(
+                            match Self::begin_tier_transition_with_destinations(
+                                &handle,
+                                &mut manager,
+                                target_tiers,
+                                HashMap::new(),
+                                prepared_mutation_block_allowance.as_ref(),
+                            ) {
+                                Ok(transition) => transition,
+                                Err(transition_error) => {
+                                    drop(manager);
+                                    if !abort_prepared_tier_mutation(
+                                        &handle,
+                                        api.clone(),
+                                        coordinator_intent.as_ref(),
+                                        prepared_peers,
+                                    )
+                                    .await
+                                    {
+                                        warn!(
+                                            event = "tier_mutation_abort",
+                                            component = LOG_COMPONENT_ECSTORE,
+                                            subsystem = LOG_SUBSYSTEM_TIER,
+                                            result = "prepared_intent_retained",
+                                            coordinator_intent = coordinator_intent.is_some(),
+                                            "tier mutation publish transition failed and abort was incomplete"
+                                        );
+                                    }
+                                    return Err(TierConfigUpdateError::Publish(transition_error));
+                                }
+                            },
+                        );
+                    }
+                    let mut transition = transition.ok_or_else(|| {
+                        let mut err = ERR_TIER_INVALID_CONFIG.clone();
+                        err.message = "Remote tier mutation lost its publish transition".to_string();
+                        TierConfigUpdateError::Publish(err)
+                    })?;
+                    let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
+                    if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
+                        drop(transition);
+                        if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await
+                        {
+                            warn!(
+                                event = "tier_mutation_abort",
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_TIER,
+                                result = "prepared_intent_retained",
+                                coordinator_intent = coordinator_intent.is_some(),
+                                "tier mutation publish drain failed and abort was incomplete"
+                            );
+                        }
+                        return Err(TierConfigUpdateError::Publish(drain_error));
+                    }
                     let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?;
                     if coordinated_config_update {
                         config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await {
@@ -5325,7 +5458,7 @@ impl TierConfigMgr {
         let manager = handle.read().await;
         let published_digest = if intents
             .iter()
-            .any(|recovered| recovered.is_peer_only_terminal() && recovered.intent.state == TierMutationIntentState::Committed)
+            .any(|recovered| recovered.intent.state == TierMutationIntentState::Committed)
         {
             Some(tier_config_candidate_digest(&manager).map_err(|err| {
                 let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
@@ -5335,18 +5468,29 @@ impl TierConfigMgr {
         } else {
             None
         };
+        let locally_published_committed_mutations = intents
+            .iter()
+            .filter(|recovered| {
+                recovered.intent.state == TierMutationIntentState::Committed
+                    && published_digest == Some(recovered.intent.candidate_digest)
+            })
+            .map(|recovered| recovered.intent.mutation_id)
+            .collect::<HashSet<_>>();
         let mut prepared_mutation_blocks = HashMap::new();
         let mut committed_mutation_blocks: HashMap<String, HashSet<uuid::Uuid>> = HashMap::new();
         for recovered in intents {
-            let settled_tombstone = recovered.is_peer_only_terminal()
+            // A matching in-memory manager has already crossed the local
-                && match recovered.intent.state {
+            // publication boundary. Keep replaying and durably cleaning the
-                    TierMutationIntentState::Aborted => true,
+            // record, but do not re-fence object operations while that
-                    TierMutationIntentState::Committed => {
+            // terminal work finishes.
-                        !retain_missing_mutation_blocks || published_digest == Some(recovered.intent.candidate_digest)
+            let skip_runtime_fence = locally_published_committed_mutations.contains(&recovered.intent.mutation_id)
-                    }
+                || (recovered.is_peer_only_terminal()
-                    TierMutationIntentState::Prepared => false,
+                    && match recovered.intent.state {
-                };
+                        TierMutationIntentState::Aborted => true,
-            if settled_tombstone {
+                        TierMutationIntentState::Committed => !retain_missing_mutation_blocks,
+                        TierMutationIntentState::Prepared => false,
+                    });
+            if skip_runtime_fence {
                 continue;
             }
             Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, &recovered.intent)?;
@@ -5359,6 +5503,9 @@ impl TierConfigMgr {
         }
         if retain_missing_mutation_blocks {
             for (tier_name, mutation_id) in &runtime.prepared_mutation_blocks {
+                if locally_published_committed_mutations.contains(mutation_id) {
+                    continue;
+                }
                 match prepared_mutation_blocks.entry(tier_name.clone()) {
                     Entry::Vacant(entry) => {
                         entry.insert(*mutation_id);
@@ -5372,10 +5519,15 @@ impl TierConfigMgr {
                 }
             }
             for (tier_name, mutation_ids) in &runtime.committed_mutation_blocks {
-                committed_mutation_blocks
+                for mutation_id in mutation_ids {
-                    .entry(tier_name.clone())
+                    if locally_published_committed_mutations.contains(mutation_id) {
-                    .or_default()
+                        continue;
-                    .extend(mutation_ids);
+                    }
+                    committed_mutation_blocks
+                        .entry(tier_name.clone())
+                        .or_default()
+                        .insert(*mutation_id);
+                }
             }
         }
         let changed = runtime.prepared_mutation_blocks != prepared_mutation_blocks
@@ -5414,11 +5566,40 @@ impl TierConfigMgr {
         handle: &Arc<RwLock<Self>>,
         intent: &TierMutationIntent,
     ) -> std::result::Result<(), AdminError> {
+        let target_tiers = intent
+            .affected_targets
+            .iter()
+            .map(|target| target.tier_name.clone())
+            .collect();
+        Self::apply_prepared_mutation_intent_block_for_tiers(handle, intent, &target_tiers).await
+    }
+
+    async fn apply_prepared_mutation_intent_block_for_tiers(
+        handle: &Arc<RwLock<Self>>,
+        intent: &TierMutationIntent,
+        target_tiers: &HashSet<String>,
+    ) -> std::result::Result<(), AdminError> {
+        if intent.state != TierMutationIntentState::Prepared {
+            return Ok(());
+        }
         let manager = handle.read().await;
         let runtime = tier_driver_runtime(handle, &manager);
         let mut runtime = lock_unpoisoned(&runtime);
         let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
         Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
+        for tier_name in target_tiers {
+            match prepared_mutation_blocks.entry(tier_name.clone()) {
+                Entry::Vacant(entry) => {
+                    entry.insert(intent.mutation_id);
+                }
+                Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
+                Entry::Occupied(_) => {
+                    let mut err = ERR_TIER_BACKEND_IN_USE.clone();
+                    err.message = format!("Remote tier {tier_name} already has another prepared mutation");
+                    return Err(err);
+                }
+            }
+        }
         if prepared_mutation_blocks == runtime.prepared_mutation_blocks {
             return Ok(());
         }
@@ -5434,6 +5615,18 @@ impl TierConfigMgr {
     pub(crate) async fn wait_for_blocked_tier_operation_leases(
         handle: &Arc<RwLock<Self>>,
         intent: &TierMutationIntent,
+    ) -> std::result::Result<(), AdminError> {
+        let target_tiers = intent
+            .affected_targets
+            .iter()
+            .map(|target| target.tier_name.clone())
+            .collect();
+        Self::wait_for_blocked_tier_operation_leases_for_tiers(handle, &target_tiers).await
+    }
+
+    async fn wait_for_blocked_tier_operation_leases_for_tiers(
+        handle: &Arc<RwLock<Self>>,
+        target_tiers: &HashSet<String>,
     ) -> std::result::Result<(), AdminError> {
         let generations = {
             let manager = handle.read().await;
@@ -5441,10 +5634,9 @@ impl TierConfigMgr {
                 return Ok(());
             };
             let runtime = lock_unpoisoned(&runtime);
-            intent
+            target_tiers
-                .affected_targets
                 .iter()
-                .filter_map(|target| runtime.generations.get(&target.tier_name).cloned())
+                .filter_map(|tier_name| runtime.generations.get(tier_name).cloned())
                 .collect::<Vec<_>>()
         };
         let drain = async {
@@ -11125,6 +11317,115 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn published_dual_terminal_intent_does_not_restore_local_runtime_fence_during_replay() {
+        use crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record;
+
+        let store = Arc::new(CasConfigStore::default());
+        let mut persisted = empty_mgr();
+        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
+        persisted
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("published tier config fixture should persist");
+        let (_, current_etag) = load_tier_config_for_update(store.clone())
+            .await
+            .expect("published tier config fixture should load with metadata");
+        let current_etag = current_etag.expect("published tier config fixture should have an ETag");
+        let affected_targets = build_tier_mutation_affected_targets(
+            TierMutationIntentKind::Add,
+            HashSet::from(["COLD-A".to_string()]),
+            &empty_mgr(),
+            &persisted,
+        )
+        .expect("published AddTier targets should build");
+        let mut intent = build_coordinator_tier_mutation_intent(TierMutationIntentKind::Add, None, &persisted, affected_targets)
+            .expect("published AddTier intent should build")
+            .expect("published AddTier should require a durable intent");
+        intent
+            .advance(TierMutationIntentState::Committed, Some(current_etag))
+            .expect("published AddTier intent should commit");
+        save_tier_coordinator_mutation_intent_record_if_absent(store.clone(), &intent)
+            .await
+            .expect("published coordinator intent should persist");
+        save_tier_mutation_intent_record(store.clone(), &intent)
+            .await
+            .expect("published peer intent should persist");
+
+        let manager = TierConfigMgr::new();
+        {
+            let mut guard = manager.write().await;
+            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("published"));
+        }
+        {
+            let guard = manager.read().await;
+            assert_eq!(
+                tier_config_candidate_digest(&guard).expect("published manager digest should build"),
+                intent.candidate_digest
+            );
+        }
+        TierConfigMgr::apply_committed_mutation_intent_block(&manager, &intent)
+            .await
+            .expect("pre-existing committed runtime fence should install");
+        assert!(
+            TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await.is_err(),
+            "fixture must begin with the committed runtime fence installed"
+        );
+
+        let started = Arc::new(Notify::new());
+        let release = Arc::new(tokio::sync::Semaphore::new(0));
+        TIER_MUTATION_TEST_PEERS
+            .scope(
+                vec![Arc::new(BlockingCommitTierMutationPeer {
+                    started: started.clone(),
+                    release: release.clone(),
+                })],
+                async {
+                    let reload = TierConfigMgr::reload_handle_with(&manager, store.clone());
+                    tokio::pin!(reload);
+                    tokio::time::timeout(Duration::from_secs(5), async {
+                        tokio::select! {
+                            result = &mut reload => panic!("reload finished before terminal replay was released: {result:?}"),
+                            _ = started.notified() => {}
+                        }
+                    })
+                    .await
+                    .expect("terminal replay should reach the blocking peer");
+
+                    let lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
+                        .await
+                        .expect("terminal cleanup must not re-fence an already-published tier");
+                    drop(lease);
+                    release.add_permits(1);
+                    tokio::time::timeout(Duration::from_secs(5), &mut reload)
+                        .await
+                        .expect("terminal replay should finish after the peer responds")
+                        .expect("terminal replay should succeed after the peer responds");
+                },
+            )
+            .await;
+
+        assert!(manager.read().await.tiers.contains_key("COLD-A"));
+        assert!(
+            TierConfigMgr::load_coordinator_mutation_intents(store.clone())
+                .await
+                .expect("coordinator cleanup should be readable")
+                .is_empty()
+        );
+        assert_eq!(
+            TierConfigMgr::load_tier_mutation_intents(store)
+                .await
+                .expect("retained peer tombstone should be readable"),
+            vec![intent]
+        );
+        let guard = manager.read().await;
+        let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
+        assert!(
+            lock_unpoisoned(&runtime).committed_mutation_blocks.is_empty(),
+            "retained terminal evidence must not restore the published runtime fence"
+        );
+    }
+
     #[tokio::test]
     async fn peer_terminal_tombstone_gc_uses_etag_and_retains_racing_replacement() {
         use crate::services::tier::tier_mutation_intent::{
@@ -14414,6 +14715,13 @@ mod tests {
                 .push(object);
         }
 
+        fn remove_listed_version(&self, bucket: &str, object: &str) {
+            self.listed_versions
+                .lock()
+                .expect("tier reference fixture should not poison")
+                .retain(|version| version.bucket != bucket || version.name != object);
+        }
+
         fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
             self.lifecycle_configs
                 .lock()
@@ -15682,6 +15990,152 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn tier_remove_prepared_fence_allows_inflight_free_version_cleanup_to_finish() {
+        let store = Arc::new(CasConfigStore::default());
+        let tier = build_rustfs_tier("COLD-A");
+        let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
+        let mut persisted = empty_mgr();
+        persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
+        persisted
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("free-version drain fixture should persist");
+
+        let manager = TierConfigMgr::new();
+        {
+            let mut guard = manager.write().await;
+            guard.tiers.insert("COLD-A".to_string(), tier);
+            guard.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
+            guard
+                .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("cleanup")))
+                .expect("cleanup driver generation should install");
+            guard
+                .replace_driver("COLD-B", Box::new(LeaseTestBackend::ready("stale-local")))
+                .expect("stale local driver generation should install");
+        }
+        let cleanup_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
+            .await
+            .expect("in-flight cleanup lease should be available");
+        let stale_local_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
+            .await
+            .expect("stale local tier lease should be available");
+        let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(identity));
+        free_version.transitioned_object.status = "pending".to_string();
+        free_version.transitioned_object.free_version = true;
+        store.add_listed_version(free_version);
+
+        let remove_manager = manager.clone();
+        let remove_store = store.clone();
+        let remove = tokio::spawn(async move {
+            TIER_MUTATION_TEST_PEERS
+                .scope(
+                    Vec::new(),
+                    TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true),
+                )
+                .await
+        });
+
+        tokio::time::timeout(Duration::from_secs(1), async {
+            loop {
+                let guard = manager.read().await;
+                let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
+                let prepared = {
+                    let runtime = lock_unpoisoned(&runtime);
+                    runtime.prepared_mutation_blocks.contains_key("COLD-A")
+                        && runtime.prepared_mutation_blocks.contains_key("COLD-B")
+                };
+                if prepared {
+                    break;
+                }
+                drop(guard);
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("tier remove should install its durable prepared fence");
+
+        assert!(
+            cleanup_lease.is_current(&manager).await,
+            "the prepared fence must let the already leased cleanup finish its exact local marker deletion"
+        );
+        assert!(
+            stale_local_lease.is_current(&manager).await,
+            "the local superset fence must also let an already leased stale-manager operation finish"
+        );
+        let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
+            Ok(_) => panic!("the prepared fence must reject new tier operations"),
+            Err(err) => err,
+        };
+        assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
+        let stale_blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-B").await {
+            Ok(_) => panic!("the local superset fence must reject new stale-manager operations"),
+            Err(err) => err,
+        };
+        assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&stale_blocked));
+
+        store.remove_listed_version("photos", "2026/free-version.jpg");
+        drop(cleanup_lease);
+        drop(stale_local_lease);
+
+        tokio::time::timeout(Duration::from_secs(5), remove)
+            .await
+            .expect("tier remove should finish after both in-flight operations release their leases")
+            .expect("tier remove task should join")
+            .expect("tier remove should pass once the in-flight cleanup removes its marker");
+        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
+        assert!(!manager.read().await.tiers.contains_key("COLD-B"));
+        assert!(
+            !load_tier_config_for_update(store)
+                .await
+                .expect("removed tier config should reload")
+                .0
+                .tiers
+                .contains_key("COLD-A")
+        );
+    }
+
+    #[tokio::test]
+    async fn no_intent_stale_manager_removal_keeps_early_generation_drain() {
+        let store = Arc::new(CasConfigStore::default());
+        empty_mgr()
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("empty persisted tier config should exist");
+        let manager = TierConfigMgr::new();
+        {
+            let mut guard = manager.write().await;
+            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("stale"));
+        }
+        let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
+            .await
+            .expect("stale manager lease should be available");
+
+        let remove_manager = manager.clone();
+        let remove_store = store.clone();
+        let remove =
+            tokio::spawn(async move { TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true).await });
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while old.is_current(&manager).await {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("no-intent stale-manager reconciliation should revoke before its proof");
+        let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
+            Ok(_) => panic!("stale-manager reconciliation must not admit a new operation"),
+            Err(err) => err,
+        };
+        assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
+
+        drop(old);
+        remove
+            .await
+            .expect("stale-manager removal task should join")
+            .expect("stale-manager removal should converge to the persisted empty config");
+        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
+    }
+
     async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
         let store = Arc::new(CasConfigStore::default());
         let tier = build_rustfs_tier("COLD-A");
@@ -16585,12 +17039,23 @@ mod tests {
             .await
         });
         tokio::time::timeout(Duration::from_secs(1), async {
-            while old.is_current(&manager).await {
+            loop {
+                let guard = manager.read().await;
+                let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
+                let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
+                if prepared {
+                    break;
+                }
+                drop(guard);
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("owned update should revoke before caller cancellation");
+        .expect("owned update should install its prepared fence before caller cancellation");
+        assert!(
+            old.is_current(&manager).await,
+            "an already leased operation must remain current until it can finish"
+        );
         caller.abort();
         drop(old);
 
@@ -16635,12 +17100,23 @@ mod tests {
             .await
         });
         tokio::time::timeout(Duration::from_secs(1), async {
-            while old.is_current(&manager).await {
+            loop {
+                let guard = manager.read().await;
+                let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
+                let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
+                if prepared {
+                    break;
+                }
+                drop(guard);
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("owned update should revoke before caller cancellation");
+        .expect("owned update should install its prepared fence before caller cancellation");
+        assert!(
+            old.is_current(&manager).await,
+            "the prepared fence must not invalidate an already leased operation"
+        );
         caller.abort();
 
         let config_file = tier_config_lock_path();
@@ -17026,6 +17502,71 @@ mod tests {
         assert!(current.tiers.contains_key("COLD-B"));
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
+        let manager = TierConfigMgr::new();
+        let store = Arc::new(CasConfigStore::default());
+        let mut persisted = empty_mgr();
+        persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
+        persisted
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("prepared-fence revision fixture should persist");
+        {
+            let mut guard = manager.write().await;
+            install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
+        }
+
+        let barrier = tier_reference_proof_test_barrier();
+        let scoped_barrier = barrier.clone();
+        let update_manager = manager.clone();
+        let update_store = store.clone();
+        let update = tokio::spawn(async move {
+            TIER_REFERENCE_PROOF_TEST_BARRIER
+                .scope(
+                    scoped_barrier,
+                    TIER_MUTATION_TEST_PEERS.scope(
+                        Vec::new(),
+                        TierConfigMgr::update_candidate_with_config_lock(
+                            &update_manager,
+                            update_store,
+                            TierCandidateMutation::Remove("COLD-A".to_string(), true),
+                        ),
+                    ),
+                )
+                .await
+        });
+        barrier.arrived.notified().await;
+
+        let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
+        TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
+            .await
+            .expect("an unrelated prepared fence should advance the runtime revision");
+        barrier.release.add_permits(1);
+
+        let err = update
+            .await
+            .expect("tier update task should join")
+            .expect_err("a reference proof cannot authorize publication across a fence revision change");
+        let TierConfigUpdateError::Publish(err) = err else {
+            panic!("the stale prepared-fence allowance should fail publication: {err:?}");
+        };
+        assert!(err.message.contains("changed before replacement"), "{err}");
+        assert!(manager.read().await.tiers.contains_key("COLD-A"));
+        assert!(
+            load_tier_config_for_update(store)
+                .await
+                .expect("rejected tier config should remain readable")
+                .0
+                .tiers
+                .contains_key("COLD-A")
+        );
+        TierConfigMgr::clear_prepared_mutation_intent_block(&manager, unrelated.mutation_id)
+            .await
+            .expect("unrelated test fence should clear");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -111,10 +111,11 @@ use crate::disk::{
 use crate::erasure::coding::BitrotReader;
 use crate::io_support::bitrot::ShardReader;
 use crate::io_support::bitrot::{
-    BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
+    BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics,
-    create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
+    create_deferred_bitrot_reader_with_stripe_handle,
-    object_mmap_read_max_length,
 };
+#[cfg(unix)]
+use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length};
 use crate::set_disk::runtime_sources;
 use crate::set_disk::shard_source::ShardReadCost;
 use crate::storage_api_contracts::object::ObjectOperations;
@@ -3662,22 +3663,22 @@ async fn rollback_failed_rename(
             let object = object.to_string();
             let disk_namespace_commit_guard = namespace_commit_guard.clone();
             let task = tokio::spawn(async move {
-                let _namespace_commit_guard = disk_namespace_commit_guard;
+                let _namespace_commit_guard = disk_namespace_commit_guard.clone();
                 #[allow(clippy::let_unit_value)]
                 let _task_guard = SetDisks::rename_fanout_task_guard(&object);
                 SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await;
                 #[cfg(test)]
                 rollback_fault_injection::before_undo(&object, disk_index)?;
-                disk.delete_version(
+                disk.undo_write_with_namespace_owner(
                     &bucket,
                     &object,
                     fi,
-                    false,
                     DeleteOptions {
                         undo_write: true,
                         old_data_dir: rollback_dir,
                         ..Default::default()
                     },
+                    disk_namespace_commit_guard.map(|owner| owner as Arc<dyn Send + Sync>),
                 )
                 .await
             });
@@ -4237,7 +4238,7 @@ impl SetDisks {
                     let successful_rename_completion_rank = successful_rename_completion_rank.clone();
                     let namespace_commit_guard = namespace_commit_guard.clone();
                     tasks.spawn(async move {
-                        let _namespace_commit_guard = namespace_commit_guard;
+                        let _namespace_commit_guard = namespace_commit_guard.clone();
                         let mut dispatch_state = RenameDispatchState::NotDispatched;
                         let result = std::panic::AssertUnwindSafe(async {
                             #[allow(clippy::let_unit_value)]
@@ -4272,7 +4273,13 @@ impl SetDisks {
                                     &file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    scanner_publication_lease_token,
+                                    crate::disk::RenameDataGuards {
+                                        scanner_publication_lease_token,
+                                        namespace_owner: namespace_commit_guard
+                                            .clone()
+                                            .map(|owner| owner as Arc<dyn Send + Sync>),
+                                        ..Default::default()
+                                    },
                                 )
                                 .await;
                             let rejected_before_publication = observed.rejected_before_publication();
@@ -4601,7 +4608,7 @@ impl SetDisks {
             // Keep the storage-owned movement permit attached to the actual
             // fan-out owner, even if the caller future is cancelled.
             let _fanout_publication_scope = fanout_publication_scope;
-            let _namespace_commit_guard = fanout_namespace_commit_guard;
+            let _namespace_commit_guard = fanout_namespace_commit_guard.clone();
             let successful_rename_completion_rank =
                 rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
             let futures = fanout_disks
@@ -4616,6 +4623,7 @@ impl SetDisks {
                     let dst_bucket = fanout_dst_bucket.clone();
                     let successful_rename_completion_rank = successful_rename_completion_rank.clone();
                     let publication_scope = scanner_publication_commit_scope.clone();
+                    let namespace_commit_guard = fanout_namespace_commit_guard.clone();
 
                     async move {
                         let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4668,7 +4676,13 @@ impl SetDisks {
                                     file_info,
                                     &dst_bucket,
                                     &dst_object,
-                                    scanner_publication_lease_token,
+                                    crate::disk::RenameDataGuards {
+                                        scanner_publication_lease_token,
+                                        namespace_owner: namespace_commit_guard
+                                            .clone()
+                                            .map(|owner| owner as Arc<dyn Send + Sync>),
+                                        ..Default::default()
+                                    },
                                 )
                                 .await;
                             let rejected_before_publication = observed.rejected_before_publication();
@@ -10859,6 +10873,358 @@ mod tests {
         .await;
     }
 
+    #[cfg(not(windows))]
+    async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) {
+        use crate::disk::os;
+        use futures::FutureExt;
+
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
+                (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
+            ],
+            async {
+                const DISKS: usize = 4;
+                let bucket = "namespace-physical-tail";
+                let object = "inline-overwrite";
+                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+                let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
+                prepare_rename_source_dirs(&dirs, &disks, "source").await;
+                let mut old = metadata_test_fileinfo(object);
+                old.mod_time = Some(OffsetDateTime::now_utc());
+                old.size = 15;
+                old.parts.clear();
+                old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None);
+                old.data = Some(Bytes::from_static(b"old-inline-body"));
+                old.set_inline_data();
+                old.metadata.insert("etag".to_string(), "old-etag".to_string());
+                let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag");
+                let mut hooks = Vec::new();
+                let mut entered = Vec::new();
+                let mut releases = Vec::new();
+                let mut publication_paths = Vec::new();
+                for (disk, info) in disks.iter().flatten().zip(&mut infos) {
+                    disk.write_metadata(bucket, bucket, object, old.clone())
+                        .await
+                        .expect("the old inline version must be readable before overwrite");
+                    info.size = 11;
+                    info.parts.clear();
+                    info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
+                    let crate::disk::Disk::Local(local) = disk.as_ref() else {
+                        panic!("physical publication fixture requires local disks");
+                    };
+                    // Linux IO paths use a mount FD, which is also the namespace lock key.
+                    let destination = local
+                        .get_disk()
+                        .get_object_path_for_io(bucket, object)
+                        .expect("the publication path must resolve through the disk's mount lease");
+                    let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+                    let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+                    hooks.push(os::prepared_publication_test_hooks::install(
+                        &destination.join(STORAGE_FORMAT_FILE),
+                        move || {
+                            let _ = entered_tx.send(());
+                            // Sender drop also releases the syscall when an earlier assertion fails.
+                            let _ = release_rx.recv();
+                        },
+                    ));
+                    entered.push(entered_rx);
+                    releases.push(release_tx);
+                    publication_paths.push(destination);
+                }
+                let namespace_owner = ctx.begin_namespace_commit();
+                let namespace_probe = Arc::downgrade(&namespace_owner);
+                let receipt = RenameRollbackReceipt::default();
+                let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence(
+                    &disks,
+                    (RUSTFS_META_TMP_BUCKET, "source"),
+                    infos,
+                    (bucket, object),
+                    allow_early_ack,
+                    RenameDataFenceOptions::new(3, None)
+                        .with_rollback_receipt(receipt.clone())
+                        .with_namespace_commit_guard(Some(namespace_owner)),
+                ));
+                tokio::time::timeout(Duration::from_secs(10), async {
+                    tokio::select! {
+                        signals = join_all(entered) => {
+                            assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter");
+                        }
+                        _ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"),
+                    }
+                })
+                .await
+                .expect("all four prepared metadata renames must reach their blocking syscall");
+                assert!(ctx.namespace_commits_pending());
+                assert_eq!(ctx.namespace_commit_generation(), 1);
+
+                // Every wrapper timer exists before advancing; the physical closures stay blocked.
+                tokio::time::pause();
+                tokio::time::advance(Duration::from_secs(61)).await;
+                tokio::time::resume();
+                let result = tokio::time::timeout(Duration::from_secs(5), rename)
+                    .await
+                    .expect("ordinary disk timeout must not wait for the physical rename");
+                assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum");
+                let report = receipt.0.get().expect("failed fanout must finish rollback accounting");
+                assert_eq!(report.disks.len(), DISKS);
+                assert!(
+                    report
+                        .disks
+                        .iter()
+                        .all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout)))
+                );
+                let pending_before_release = ctx.namespace_commits_pending();
+                let owner_alive_before_release = namespace_probe.upgrade().is_some();
+                let old_snapshot_generation = ctx.namespace_commit_generation();
+                for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
+                    let root = disk.path();
+                    assert!(
+                        os::acquire_rename_data_mutation_lease(&root, bucket, destination)
+                            .now_or_never()
+                            .is_none(),
+                        "the physical publication must still own object serialization after the async timeout"
+                    );
+                    assert!(
+                        root.join(RUSTFS_META_TMP_BUCKET)
+                            .join("source")
+                            .join(STORAGE_FORMAT_FILE)
+                            .exists()
+                    );
+                    let stored = disk
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("a scanner can still read the complete old metadata while publication is paused");
+                    assert_eq!(stored.size, 15);
+                    assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice()));
+                }
+                assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation);
+
+                // Drain real syscalls before checking the regression, including on the RED run.
+                drop(releases);
+                for (disk, destination) in disks.iter().flatten().zip(&publication_paths) {
+                    let root = disk.path();
+                    let lease = tokio::time::timeout(
+                        Duration::from_secs(5),
+                        os::acquire_rename_data_mutation_lease(&root, bucket, destination),
+                    )
+                    .await
+                    .expect("released physical publishers must drain");
+                    drop(lease);
+                }
+                for dir in &dirs {
+                    let reopened = reopen_local_disk(dir).await;
+                    let stored = reopened
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("the detached prepared rename must actually publish after timeout");
+                    assert_eq!(stored.size, 11);
+                    assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
+                }
+                // The lease releases its locks before dropping the namespace owner, and the
+                // owner's `Drop` runs after its `Weak` probe stops upgrading, so wait for the
+                // pending counter itself instead of asserting it right after the drain.
+                tokio::time::timeout(Duration::from_secs(5), async {
+                    while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() {
+                        tokio::task::yield_now().await;
+                    }
+                })
+                .await
+                .expect("released physical publishers must release namespace ownership");
+                let generation_after_publication = ctx.namespace_commit_generation();
+                assert!(!ctx.namespace_commits_pending());
+                assert!(namespace_probe.upgrade().is_none());
+                assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence");
+                assert!(
+                    pending_before_release && owner_alive_before_release,
+                    "physical publication outlived namespace accounting: early_ack={allow_early_ack}, \
+                     pending={pending_before_release}, owner_alive={owner_alive_before_release}, \
+                     old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}"
+                );
+                assert!(
+                    generation_after_publication > old_snapshot_generation,
+                    "physical completion must invalidate the scanner's old metadata snapshot"
+                );
+            },
+        )
+        .await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() {
+        assert_namespace_owner_survives_physical_publication_timeout(false).await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() {
+        assert_namespace_owner_survives_physical_publication_timeout(true).await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() {
+        use crate::disk::os;
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")),
+                (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
+            ],
+            async {
+                let bucket = "physical-ack-tail";
+                let object = "ack-object";
+                let (dirs, disks) = call_counter_local_disks(bucket, 4).await;
+                prepare_rename_source_dirs(&dirs, &disks, "source").await;
+                let mut infos = rename_commit_fileinfos(object, 4, "new-etag");
+                for info in &mut infos {
+                    info.size = 11;
+                    info.parts.clear();
+                    info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
+                }
+                let disk = disks[3].as_ref().expect("tail disk");
+                let crate::disk::Disk::Local(local) = disk.as_ref() else {
+                    panic!("local fixture");
+                };
+                let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
+                let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+                let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+                let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
+                    let _ = entered_tx.send(());
+                    let _ = release_rx.recv();
+                });
+                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+                let owner = ctx.begin_namespace_commit();
+                let owner_probe = Arc::downgrade(&owner);
+                let receipt = RenameRollbackReceipt::default();
+                let caller_receipt = receipt.clone();
+                let caller_disks = disks.clone();
+                let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
+                let caller = tokio::spawn(async move {
+                    let commit = SetDisks::rename_data_owned_with_fence(
+                        &caller_disks,
+                        (RUSTFS_META_TMP_BUCKET, "source"),
+                        infos,
+                        (bucket, object),
+                        true,
+                        RenameDataFenceOptions::new(3, None)
+                            .with_namespace_commit_guard(Some(owner))
+                            .with_rollback_receipt(caller_receipt),
+                    )
+                    .await
+                    .expect("three real disk publications must produce a successful ACK");
+                    assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK");
+                    std::future::pending::<()>().await;
+                });
+                let mut commit = tokio::time::timeout(Duration::from_secs(10), async {
+                    entered_rx.await.expect("physical tail entry");
+                    ack_rx
+                        .await
+                        .expect("ACK must arrive while the fourth disk is physically paused")
+                })
+                .await
+                .expect("successful quorum ACK must not wait for its physical tail");
+                assert_eq!(commit.online_disks.iter().flatten().count(), 3);
+                assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK");
+                let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle");
+                drop(commit);
+                caller.abort();
+                assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled());
+                tokio::time::pause();
+                tokio::time::advance(Duration::from_secs(61)).await;
+                tokio::time::resume();
+                let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain)
+                    .await
+                    .expect("ordinary tail timeout stays bounded after ACK")
+                    .expect("tail owner must not panic")
+                    .expect("successful ACK keeps its convergence result");
+                assert_eq!(tail.convergence, RenameConvergence::PartialCommit);
+                assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback");
+                let pending = ctx.namespace_commits_pending();
+                let alive = owner_probe.upgrade().is_some();
+                let generation = ctx.namespace_commit_generation();
+                for disk in disks.iter().flatten().take(3) {
+                    let stored = disk
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("all ACK voters keep the new object after caller cancellation");
+                    assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
+                }
+                drop(release_tx);
+                let lease = tokio::time::timeout(
+                    Duration::from_secs(5),
+                    os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination),
+                )
+                .await
+                .expect("late physical tail drains");
+                drop(lease);
+                tokio::time::timeout(Duration::from_secs(5), async {
+                    while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() {
+                        tokio::task::yield_now().await;
+                    }
+                })
+                .await
+                .expect("late physical tail must release namespace ownership");
+                for dir in &dirs {
+                    let stored = reopen_local_disk(dir)
+                        .await
+                        .read_version(
+                            "",
+                            bucket,
+                            object,
+                            "",
+                            &ReadOptions {
+                                read_data: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect("successful ACK remains committed on every disk after late publication");
+                    assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice()));
+                }
+                assert!(
+                    pending && alive,
+                    "physical ACK tail must retain namespace ownership after the coordinator exits"
+                );
+                assert!(!ctx.namespace_commits_pending());
+                assert!(owner_probe.upgrade().is_none());
+                assert!(ctx.namespace_commit_generation() > generation);
+                assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback");
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(capacity_dirty_scope)]
     async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() {

crates/ecstore/src/set_disk/mod.rs

@@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
 #[cfg(test)]
 pub(crate) use ops::object::DeleteObjectCommitBarrier;
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 #[cfg(all(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
@@ -4498,6 +4498,12 @@ impl SetDisks {
         &self.ctx
     }
 
+    /// Read the persisted bucket identity through this set's metadata owner.
+    /// Missing or non-authoritative legacy identities remain errors.
+    pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
+        metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await
+    }
+
     /// Admit one short scanner cache publication under this set's instance
     /// movement fence. The caller must hold the returned guard through its
     /// final conditional cache write; no scan-round work belongs under it.

crates/ecstore/src/set_disk/ops/object.rs

@@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
 }
 
 async fn transition_cleanup_store(ctx: &Arc<crate::runtime::instance::InstanceContext>) -> Option<Arc<ECStore>> {
-    #[cfg(any(test, feature = "test-util"))]
+    #[cfg(feature = "test-util")]
     pause_transition_cleanup_store().await;
 
     transition_object_store(ctx).await
@@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup(
     }
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 #[derive(Default)]
 struct TransitionCleanupStoreBarrierState {
     arrived: tokio::sync::Notify,
     release: tokio::sync::Notify,
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 /// One-shot test barrier placed before transition cleanup resolves its ECStore.
 pub(crate) struct TransitionCleanupStoreBarrier {
     state: Arc<TransitionCleanupStoreBarrierState>,
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCleanupStoreBarrierState>>>> =
     std::sync::OnceLock::new();
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 impl TransitionCleanupStoreBarrier {
     /// Install the process-local barrier for the next cleanup-store resolution.
     pub(crate) fn install() -> Self {
@@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier {
     }
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 impl Drop for TransitionCleanupStoreBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier {
     }
 }
 
-#[cfg(any(test, feature = "test-util"))]
+#[cfg(feature = "test-util")]
 async fn pause_transition_cleanup_store() {
     let barrier = TRANSITION_CLEANUP_STORE_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 struct TransitionUploadedCommitBarrierState {
     bucket: String,
     object: String,
@@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState {
     release: tokio::sync::Notify,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) struct TransitionUploadedCommitBarrier {
     state: Arc<TransitionUploadedCommitBarrierState>,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock<
     std::sync::Mutex<Option<Arc<TransitionUploadedCommitBarrierState>>>,
 > = std::sync::OnceLock::new();
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl TransitionUploadedCommitBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(TransitionUploadedCommitBarrierState {
@@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl Drop for TransitionUploadedCommitBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) {
     let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -7497,6 +7497,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let transported = delete_file_info_with_replication_transport_metadata(fi);
         let fi = &transported;
         let disks = self.disk_inventory().await;
+        let namespace_owner = (!is_meta_bucketname(bucket)).then(|| self.ctx.begin_namespace_commit());
         let write_quorum = disks.len() / 2 + 1;
         let rollback_dir = Uuid::new_v4();
 
@@ -7504,10 +7505,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let mut errs = Vec::with_capacity(disks.len());
 
         for disk in disks.iter() {
+            let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
             futures.push(async move {
                 if let Some(disk) = disk {
                     match disk
-                        .delete_version(
+                        .delete_version_with_namespace_owner(
                             bucket,
                             object,
                             fi.clone(),
@@ -7516,6 +7518,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 old_data_dir: Some(rollback_dir),
                                 ..Default::default()
                             },
+                            disk_namespace_owner,
                         )
                         .await
                     {
@@ -7563,10 +7566,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             let bucket = bucket.to_string();
             let object = object.to_string();
             let fi = fi.clone();
+            let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
             rollback_futures.push(async move {
                 if should_rollback {
                     if let Err(err) = disk
-                        .delete_version(
+                        .delete_version_with_namespace_owner(
                             &bucket,
                             &object,
                             fi,
@@ -7577,6 +7581,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 old_data_dir: Some(rollback_dir),
                                 ..Default::default()
                             },
+                            disk_namespace_owner,
                         )
                         .await
                     {
@@ -7591,7 +7596,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 } else {
                     let rollback_path = format!("{object}/{rollback_dir}");
                     if let Err(err) = disk
-                        .delete(
+                        .delete_with_namespace_owner(
                             &bucket,
                             &rollback_path,
                             DeleteOptions {
@@ -7599,6 +7604,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 immediate: true,
                                 ..Default::default()
                             },
+                            disk_namespace_owner,
                         )
                         .await
                         && err != DiskError::FileNotFound
@@ -7617,6 +7623,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         join_all(rollback_futures).await;
+        drop(namespace_owner);
         quorum_result
     }
 
@@ -9154,7 +9161,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
         upload_cleanup.update_cleanup_transaction(&transaction);
 
-        #[cfg(test)]
+        #[cfg(all(test, feature = "test-util"))]
         pause_after_transition_uploaded_persisted(bucket, object).await;
 
         let commit_opts = opts.as_commit_opts();
@@ -12672,6 +12679,65 @@ mod metadata_mutation_generation_tests {
         set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial(metadata_cache_invalidation_probe)]
+    async fn segment_observation_equal_size_mutations_retire_metadata_generation() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "segment-observation-bucket";
+        let object = "hot/object";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create segment fixture bucket");
+        }
+        let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await;
+        let probe = MetadataCacheInvalidationProbe::install(bucket, object);
+        let mut replacement = PutObjReader::from_vec(b"after!".to_vec());
+        set_disks
+            .put_object(bucket, object, &mut replacement, &ObjectOptions::default())
+            .await
+            .expect("commit same-length replacement with normal owner locking");
+        assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation");
+        assert_retired(&set_disks, &old_key).await;
+        drop(probe);
+        let after = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read replacement metadata");
+        assert_eq!(before.size, after.size);
+        assert_ne!(before.etag, after.etag, "equal size is not equal content");
+        let mut reader = set_disks
+            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read replacement body through the owner");
+        let mut body = Vec::new();
+        reader.stream.read_to_end(&mut body).await.expect("drain replacement body");
+        assert_eq!(body, b"after!");
+        drop(reader);
+
+        let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await;
+        let probe = MetadataCacheInvalidationProbe::install(bucket, object);
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("commit metadata-only mutation with normal owner locking");
+        assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences");
+        assert_retired(&set_disks, &old_key).await;
+        let after = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read committed metadata-only mutation");
+        assert_eq!(before.size, after.size);
+        assert_eq!(before.etag, after.etag);
+        assert!(!before.user_defined.contains_key("x-amz-meta-segment"));
+        assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed"));
+    }
+
     #[tokio::test]
     #[serial_test::serial(metadata_cache_invalidation_probe)]
     async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
@@ -16225,7 +16291,23 @@ mod transition_upload_integrity_tests {
             let bucket = format!("transition-real-bitrot-{}", position.label());
             let object = format!("{}-corrupt.bin", position.label());
             let payload = vec![0x41; 2 * 1024 * 1024];
-            let original = write_source(&set_disks, &disk_stores, &bucket, &object, &payload).await;
+            // Corruption edits physical shards, so every healthy rename must finish first.
+            for disk in &disk_stores {
+                disk.make_volume(&bucket).await.expect("bucket volume should be created");
+            }
+            let mut reader = PutObjReader::from_vec(payload.to_vec());
+            let original = set_disks
+                .put_object(
+                    &bucket,
+                    &object,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("source object should be written");
             let source = set_disks
                 .get_object_fileinfo(
                     &bucket,
@@ -20985,3 +21067,417 @@ mod body_cache_hook_e2e_tests {
         );
     }
 }
+
+#[cfg(test)]
+mod single_delete_namespace_owner_tests {
+    use super::hermetic_set_disks_support::hermetic_set_disks_isolated;
+    use super::*;
+    use crate::disk::ReadOptions;
+    #[cfg(not(windows))]
+    use crate::disk::STORAGE_FORMAT_FILE;
+    use crate::object_api::WriteCompletion;
+    #[cfg(not(windows))]
+    use tokio::io::AsyncReadExt;
+
+    async fn seed_version(set: &Arc<SetDisks>, bucket: &str, object: &str, version: Uuid, body: &[u8]) {
+        set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(body.to_vec()),
+            &ObjectOptions {
+                versioned: true,
+                version_id: Some(version.to_string()),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("seed a complete real object version");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_advances_namespace_generation_through_cleanup() {
+        let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "single-delete-namespace";
+        let object = "last-version";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("fixture bucket");
+        }
+        let version = Uuid::new_v4();
+        seed_version(&set, bucket, object, version, &vec![0x41; 256 * 1024]).await;
+        let before = set.ctx.namespace_commit_generation();
+        assert!(!set.ctx.namespace_commits_pending());
+        let request = FileInfo {
+            name: object.to_string(),
+            version_id: Some(version),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        let result =
+            tokio::time::timeout(Duration::from_secs(10), set.delete_object_version(bucket, object, &request, false)).await;
+        if !matches!(result, Ok(Ok(()))) {
+            let retained = dirs.into_iter().map(tempfile::TempDir::keep).collect::<Vec<_>>();
+            panic!("single delete and cleanup did not finish: {result:?}; retained roots: {retained:?}");
+        }
+        for (disk, dir) in disks.iter().zip(&dirs) {
+            let result = disk
+                .read_version("", bucket, object, &version.to_string(), &ReadOptions::default())
+                .await;
+            assert!(matches!(result, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)));
+            assert!(
+                !dir.path().join(bucket).join(object).exists(),
+                "immediate cleanup must remove the rollback object tree"
+            );
+        }
+        assert!(!set.ctx.namespace_commits_pending());
+        assert_eq!(
+            set.ctx.namespace_commit_generation(),
+            before + 2,
+            "single delete must count one complete root lifetime"
+        );
+    }
+
+    #[cfg(not(windows))]
+    async fn assert_single_delete_physical_owner(case: &'static str) {
+        use crate::disk::os::prepared_publication_test_hooks as hooks;
+        use futures::FutureExt;
+
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
+            let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+            let bucket = "single-delete-physical-owner";
+            let first = Uuid::new_v4();
+            let second = Uuid::new_v4();
+            let first_body = vec![0x51; 256 * 1024];
+            let second_body = vec![0x62; 4096];
+            let missing = case == "missing-marker";
+            let rollback = case == "rollback";
+            let last = case == "last-version";
+            let cleanup = case == "immediate-cleanup";
+            for disk in &disks {
+                disk.make_volume(bucket).await.expect("fixture bucket");
+            }
+            if !missing {
+                seed_version(&set, bucket, case, first, &first_body).await;
+                if !last && !cleanup {
+                    seed_version(&set, bucket, case, second, &second_body).await;
+                }
+            }
+            let request = FileInfo {
+                name: case.to_string(),
+                version_id: Some(first),
+                deleted: missing,
+                mark_deleted: missing,
+                mod_time: Some(OffsetDateTime::now_utc()),
+                ..Default::default()
+            };
+            let before = set.ctx.namespace_commit_generation();
+            assert!(!set.ctx.namespace_commits_pending());
+            let mut metadata_paths = Vec::new();
+            let mut originals = Vec::new();
+            let mut cleanup_sources = Vec::new();
+            let mut cleanup_parts = Vec::new();
+            for disk in &disks {
+                let crate::disk::Disk::Local(local) = disk.as_ref() else {
+                    panic!("local fixture required");
+                };
+                let path = local
+                    .get_disk()
+                    .get_object_path_for_io(bucket, case)
+                    .expect("leased IO path")
+                    .join(STORAGE_FORMAT_FILE);
+                originals.push(if missing {
+                    None
+                } else {
+                    Some(std::fs::read(&path).expect("seeded raw metadata"))
+                });
+                if cleanup {
+                    let fi = disk.read_version("", bucket, case, &first.to_string(), &ReadOptions::default())
+                        .await.expect("real non-inline data directory");
+                    assert!(!fi.inline_data(), "cleanup fixture must have real shard files");
+                    let data = path.parent().expect("object parent").join(fi.data_dir.expect("data directory").to_string());
+                    cleanup_parts.push(std::fs::read(data.join("part.1")).expect("real pre-delete shard"));
+                    cleanup_sources.push(data);
+                }
+                metadata_paths.push(path);
+            }
+            if rollback {
+                // Two disks apply the real deletion and then error. Undo must
+                // restore all four disks, including these post-apply failures.
+                for disk in disks.iter().take(2) {
+                    crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), case);
+                }
+            }
+            let (entered_tx, mut entered_rx) = tokio::sync::mpsc::unbounded_channel();
+            let mut guards = Vec::new();
+            let mut destination_guards = Vec::new();
+            let later_guards = Arc::new(std::sync::Mutex::new(Vec::new()));
+            let mut releases = Vec::new();
+            for (index, path) in metadata_paths.iter().enumerate() {
+                let tx = entered_tx.clone();
+                let (release, rx) = std::sync::mpsc::channel::<()>();
+                if last || cleanup {
+                    let source = if cleanup { &cleanup_sources[index] } else { path };
+                    destination_guards.push(hooks::observe_rename_destination(source, move |destination| {
+                        let _ = tx.send((index, destination.to_path_buf()));
+                        let _ = rx.recv();
+                    }));
+                } else {
+                    let hook_path = path.clone();
+                    let path = path.clone();
+                    let pause_path = path.clone();
+                    let later_guards = Arc::clone(&later_guards);
+                    guards.push(hooks::install_at(hooks::Stage::Rename, &hook_path, move || {
+                        let pause = move || {
+                            let _ = tx.send((index, pause_path));
+                            let _ = rx.recv();
+                        };
+                        if rollback {
+                            // This first callback precedes forward metadata publication.
+                            // Arm only the subsequent real backup-restore rename.
+                            let next = hooks::install_at(hooks::Stage::Rename, &path, pause);
+                            later_guards.lock().expect("fixture hook guards").push(next);
+                        } else {
+                            pause();
+                        }
+                    }));
+                }
+                releases.push(release);
+            }
+            drop(entered_tx);
+            let deleting_set = Arc::clone(&set);
+            let mut delete =
+                tokio::spawn(async move { deleting_set.delete_object_version(bucket, case, &request, missing).await });
+            let mut joined = false;
+            let mut counts = None;
+            let mut physical_keys = std::collections::BTreeMap::new();
+            let observations = std::panic::AssertUnwindSafe(async {
+                tokio::time::timeout(Duration::from_secs(10), async {
+                    while physical_keys.len() < 4 {
+                        tokio::select! {
+                            entry = entered_rx.recv() => {
+                                let (index, key) = entry.expect("actual physical delete entry");
+                                assert!(physical_keys.insert(index, key).is_none());
+                            }
+                            result = &mut delete => {
+                                joined = true;
+                                panic!("delete returned before physical entry: {result:?}");
+                            }
+                        }
+                    }
+                })
+                .await
+                .expect("all four physical mutations must enter");
+                let pending_at_entry = set.ctx.namespace_commits_pending();
+                let generation_at_entry = set.ctx.namespace_commit_generation();
+                for (path, original) in metadata_paths.iter().zip(&originals) {
+                    if missing {
+                        assert!(!path.exists(), "missing marker must still be unpublished at entry");
+                    } else if cleanup {
+                        assert!(!path.exists(), "cleanup must follow the actual last-version deletion");
+                    } else {
+                        let bytes = std::fs::read(path).expect("paused metadata is readable");
+                        let metadata = rustfs_filemeta::FileMeta::load(&bytes).expect("real metadata must parse");
+                        if !last && !cleanup {
+                            assert!(metadata.find_version(Some(second)).is_ok());
+                        }
+                        assert_eq!(
+                            metadata.find_version(Some(first)).is_err(),
+                            rollback,
+                            "undo entry must follow actual deletion"
+                        );
+                        if !rollback {
+                            assert_eq!(Some(&bytes), original.as_ref());
+                        }
+                    }
+                }
+                if rollback {
+                    tokio::time::pause();
+                    tokio::time::advance(Duration::from_secs(61)).await;
+                    tokio::time::resume();
+                    let result = tokio::time::timeout(Duration::from_secs(5), &mut delete).await;
+                    joined = result.is_ok();
+                    let result = result
+                        .expect("ordinary undo deadlines must return")
+                        .expect("delete coordinator must not panic");
+                    assert!(
+                        matches!(&result, Err(StorageError::InsufficientWriteQuorum(error_bucket, error_object)) if error_bucket == bucket && error_object == case),
+                        "keep the original failed delete quorum: {result:?}"
+                    );
+                } else {
+                    delete.abort();
+                    let result = tokio::time::timeout(Duration::from_secs(5), &mut delete).await;
+                    joined = result.is_ok();
+                    assert!(
+                        result
+                            .expect("cancelled caller must join")
+                            .expect_err("the caller must be cancelled")
+                            .is_cancelled()
+                    );
+                }
+                counts = Some((
+                    pending_at_entry,
+                    generation_at_entry,
+                    set.ctx.namespace_commits_pending(),
+                    set.ctx.namespace_commit_generation(),
+                ));
+                for path in physical_keys.values() {
+                    assert!(
+                        hooks::drain_namespace_key(path)
+                            .now_or_never()
+                            .is_none(),
+                        "the physical metadata executor must still own its exact key"
+                    );
+                }
+            })
+            .catch_unwind()
+            .await;
+
+            drop(releases);
+            drop(guards);
+            drop(destination_guards);
+            let coordinator_drained = joined || tokio::time::timeout(Duration::from_secs(10), &mut delete).await.is_ok();
+            if !coordinator_drained {
+                delete.abort();
+                let _ = tokio::time::timeout(Duration::from_secs(5), &mut delete).await;
+            }
+            later_guards.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clear();
+            let drains = futures::future::join_all(physical_keys.values().map(|path| {
+                tokio::time::timeout(Duration::from_secs(5), hooks::drain_namespace_key(path))
+            })).await;
+            let owner_drained = tokio::time::timeout(Duration::from_secs(5), async {
+                while set.ctx.namespace_commits_pending() {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .is_ok();
+            if physical_keys.len() != 4 || !coordinator_drained || !owner_drained || drains.iter().any(|result| result.is_err()) {
+                let retained = dirs.into_iter().map(tempfile::TempDir::keep).collect::<Vec<_>>();
+                eprintln!("single delete cleanup incomplete; retained roots: {retained:?}");
+                if let Err(panic) = observations {
+                    std::panic::resume_unwind(panic);
+                }
+                panic!("single delete physical cleanup did not drain");
+            }
+            if let Err(panic) = observations {
+                std::panic::resume_unwind(panic);
+            }
+            for (index, (disk, (path, original))) in disks.iter().zip(metadata_paths.iter().zip(&originals)).enumerate() {
+                if cleanup {
+                    assert!(!path.exists(), "metadata must remain deleted after cleanup cancellation");
+                    assert!(!cleanup_sources[index].exists(), "physical cleanup must remove the shard directory");
+                    assert_eq!(
+                        std::fs::read(physical_keys[&index].join("part.1")).expect("actual trashed shard"),
+                        cleanup_parts[index],
+                        "trash must contain the exact original shard"
+                    );
+                    continue;
+                }
+                if last {
+                    assert!(!path.exists(), "late trash rename must remove the last metadata");
+                    assert_eq!(
+                        Some(std::fs::read(&physical_keys[&index]).expect("actual trash destination")),
+                        *original,
+                        "last-version trash must contain the exact old metadata"
+                    );
+                    continue;
+                }
+                let bytes = std::fs::read(path).expect("late metadata publication must finish");
+                let metadata = rustfs_filemeta::FileMeta::load(&bytes).expect("final metadata must parse");
+                if missing {
+                    assert!(
+                        metadata
+                            .find_version(Some(first))
+                            .expect("the marker must be published")
+                            .1
+                            .delete_marker
+                            .is_some()
+                    );
+                } else {
+                    assert!(metadata.find_version(Some(second)).is_ok());
+                    assert_eq!(metadata.find_version(Some(first)).is_ok(), rollback);
+                    if rollback {
+                        assert_eq!(Some(&bytes), original.as_ref(), "physical undo must restore exact old metadata");
+                    }
+                    let fi = disk
+                        .read_version("", bucket, case, &second.to_string(), &ReadOptions { read_data: true, ..Default::default() })
+                        .await
+                        .expect("remaining version");
+                    if fi.inline_data() {
+                        assert!(fi.data.as_ref().is_some_and(|data| !data.is_empty()), "remaining inline shard must survive");
+                    } else {
+                        let parts = disk.check_parts(bucket, case, &fi).await.expect("remaining shard check");
+                        assert_eq!(parts.results, vec![crate::disk::CHECK_PART_SUCCESS; fi.parts.len()]);
+                    }
+                }
+            }
+            if !missing && !last && !cleanup {
+                let mut actual = Vec::new();
+                let read_opts = ObjectOptions {
+                    version_id: Some(if rollback { first } else { second }.to_string()),
+                    versioned: true,
+                    ..Default::default()
+                };
+                let mut reader = tokio::time::timeout(
+                    Duration::from_secs(5),
+                    set.get_object_reader(bucket, case, None, HeaderMap::new(), &read_opts),
+                )
+                .await
+                .expect("final GET must finish")
+                .expect("the surviving version must be readable");
+                tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut actual))
+                    .await
+                    .expect("body must drain")
+                    .expect("read surviving body");
+                assert_eq!(actual, if rollback { first_body } else { second_body });
+            }
+            let (pending_at_entry, generation_at_entry, pending_after_return, generation_after_return) =
+                counts.expect("complete observations");
+            assert!(
+                pending_at_entry && pending_after_return,
+                "physical single delete outlived namespace accounting: {case}"
+            );
+            assert_eq!(generation_at_entry, before + 1);
+            assert_eq!(generation_after_return, generation_at_entry);
+            assert_eq!(set.ctx.namespace_commit_generation(), before + 2);
+            assert!(!set.ctx.namespace_commits_pending());
+        })
+        .await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_cancel_keeps_owner_until_immediate_data_cleanup() {
+        assert_single_delete_physical_owner("immediate-cleanup").await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_cancel_keeps_owner_until_last_version_trash() {
+        assert_single_delete_physical_owner("last-version").await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_cancel_keeps_owner_until_metadata_rewrite() {
+        assert_single_delete_physical_owner("remaining-version").await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_cancel_keeps_owner_until_missing_marker_publication() {
+        assert_single_delete_physical_owner("missing-marker").await;
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
+    async fn single_delete_failed_quorum_keeps_owner_until_physical_undo() {
+        assert_single_delete_physical_owner("rollback").await;
+    }
+}

crates/ecstore/src/store/init.rs

@@ -825,6 +825,11 @@ mod tests {
                 manual_transition_scope_record_object_name, manual_transition_task_object_name,
                 manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
             },
+            recovery_control::{
+                IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
+                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
+                observe_recovery_source, save_recovery_control_if_absent,
+            },
             tier_delete_journal::{
                 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
                 TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
@@ -844,12 +849,13 @@ mod tests {
             },
             transition_transaction::{
                 TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
-                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity,
+                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
-                TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
-                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
-                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
-                recover_transition_transaction_records, recover_transition_transaction_records_at,
+                load_transition_transaction_record, recover_transition_transaction_records,
-                save_transition_transaction_record, save_transition_transaction_record_if_current,
+                recover_transition_transaction_records_at, save_transition_transaction_record,
+                save_transition_transaction_record_if_current, transition_recovery_control_id,
                 transition_transaction_record_object_name,
             },
             validate_durable_ilm_record,
@@ -16754,6 +16760,141 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
+        let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "LEGACY-RECOVERY";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("legacy recovery tier lease should resolve")
+            .backend_identity();
+        let fixtures = [
+            serde_json::json!({
+                "version": 1,
+                "obj_name": "legacy/remote-v1",
+                "version_id": "opaque-v1",
+                "tier_name": tier_name,
+            }),
+            serde_json::json!({
+                "version": 2,
+                "obj_name": "legacy/remote-v2",
+                "version_id": "opaque-v2",
+                "tier_name": tier_name,
+                "backend_identity": backend_identity,
+            }),
+        ];
+        let mut journal_paths = Vec::new();
+        for fixture in &fixtures {
+            let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
+            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
+                .expect("legacy journal fixture should decode");
+            let path = tier_delete_journal_object_name(&entry);
+            com::save_config(store.clone(), &path, data)
+                .await
+                .expect("legacy journal fixture should persist");
+            journal_paths.push(path);
+        }
+        let corrupt_path = format!(
+            "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
+            rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
+        );
+        com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
+            .await
+            .expect("corrupt legacy journal fixture should persist");
+
+        let (first, concurrent) = tokio::join!(
+            recover_tier_delete_journal_entries(store.clone(), 100, None),
+            recover_tier_delete_journal_entries(store.clone(), 100, None),
+        );
+        for stats in [first, concurrent] {
+            let stats = stats.expect("concurrent legacy journal recovery scan should finish");
+            assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
+        }
+        assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
+        assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
+        assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
+        assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
+
+        let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
+            .await
+            .expect("legacy recovery controls should be listable")
+            .records;
+        first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
+        assert_eq!(first_controls.len(), 3);
+        assert_eq!(
+            first_controls
+                .iter()
+                .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
+                .count(),
+            2
+        );
+        assert_eq!(
+            first_controls
+                .iter()
+                .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
+                .count(),
+            1
+        );
+        for view in &first_controls {
+            assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
+            assert_eq!(view.revision, 1);
+            assert_eq!(view.attempt_count, 0);
+            let encoded = serde_json::to_string(view).expect("recovery control view should encode");
+            for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
+                assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
+            }
+            let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
+                .await
+                .expect("legacy recovery control should load");
+            match view.source_schema.as_str() {
+                "rustfs-tier-delete-journal-v1" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
+                }
+                "rustfs-tier-delete-journal-v2" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
+                }
+                "rustfs-tier-delete-journal-unknown" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
+                }
+                schema => panic!("unexpected legacy recovery source schema: {schema}"),
+            }
+        }
+
+        com::save_config(
+            store.clone(),
+            &journal_paths[0],
+            serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
+        )
+        .await
+        .expect("equivalent legacy journal rewrite should persist");
+        let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
+            .await
+            .expect("repeated legacy journal recovery scan should finish");
+        assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
+        let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
+            .await
+            .expect("repeated legacy recovery controls should remain listable")
+            .records;
+        second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
+        assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
+        assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
+        assert_eq!(backend.exact_remove_count(), 0);
+        assert!(
+            backend.op_log().await.is_empty(),
+            "repeated recovery must not invoke any backend operation"
+        );
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19239,6 +19380,105 @@ mod tests {
         assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
+            temp_dir.path(),
+            "transition-transaction-expired-attempt-budget",
+            &[4],
+        ))
+        .await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: Some(uuid::Uuid::new_v4()),
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Versioned,
+            },
+            tier_name: "UNUSEDABANDONEDTIER".to_string(),
+            backend_fingerprint: [7; 32],
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+
+        let record_name =
+            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
+        let source = observe_recovery_source(
+            store.clone(),
+            &record_name,
+            crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
+        )
+        .await
+        .expect("transaction source generation should be observable");
+        let mut control = IlmRecoveryControl::new(
+            IlmRecoveryControlIdentity {
+                protocol: IlmRecoveryProtocol::TransitionTransaction,
+                canonical_source_path: record_name,
+                stable_operation_identity: transaction.transaction_id.to_string(),
+                record_class: "transition_transaction_v1".to_string(),
+            },
+            source.generation,
+            IlmRecoveryClassification::Retrying,
+            2_000_000_000,
+            IlmRecoveryErrorCode::None,
+        )
+        .expect("recovery control should build");
+        let mut now = 3_000_000_000;
+        for _ in 1..MAX_RECOVERY_ATTEMPTS {
+            now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
+            control
+                .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
+                .expect("abandoned attempt should claim");
+            control
+                .record_expired_attempt(now + 1)
+                .expect("expired attempt should consume retry budget");
+            now += 2;
+        }
+        now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
+        control
+            .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
+            .expect("final abandoned attempt should claim");
+        save_recovery_control_if_absent(store.clone(), &control)
+            .await
+            .expect("claimed recovery control should persist");
+
+        let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
+            .await
+            .expect("recovery should account for the expired attempt");
+        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
+
+        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
+        let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
+            .await
+            .expect("expired recovery control should remain inspectable");
+        assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
+        assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
+        assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
+        assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
+        assert!(persisted.control.owner.is_none());
+        assert_eq!(
+            transition_transaction_record_count(store).await,
+            1,
+            "budget exhaustion must retain the source record"
+        );
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19351,6 +19591,7 @@ mod tests {
             ),
         ];
         let mut expected_removes = Vec::new();
+        let mut recovery_control_ids = Vec::new();
         for (case, put_version, remote_version, source_mode) in cases {
             let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
                 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
@@ -19388,6 +19629,8 @@ mod tests {
             save_transition_transaction_record(store.clone(), &transaction)
                 .await
                 .expect("transaction record should persist");
+            recovery_control_ids
+                .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
             expected_removes.push((transaction.remote_object, put_version));
         }
 
@@ -19403,6 +19646,14 @@ mod tests {
         assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(backend.object_count().await, 0);
+        for control_id in recovery_control_ids {
+            let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
+                .await
+                .expect("completed recovery control should remain inspectable");
+            assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
+            assert_eq!(control.control.attempt_count, 1);
+            assert!(control.control.owner.is_none());
+        }
 
         let replay = recover_transition_transaction_records(store, 100, None)
             .await
@@ -19415,6 +19666,94 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
+                .await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXTERMINALCRASH";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let remote_version = uuid::Uuid::new_v4().to_string();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: Some(uuid::Uuid::new_v4()),
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Versioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(
+                transaction.fence(),
+                TransitionTransactionState::Uploaded,
+                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
+            )
+            .expect("transaction should enter uploaded state");
+        backend.set_put_remote_version(Some(remote_version)).await;
+        let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
+        backend
+            .put(
+                &transaction.remote_object,
+                ReaderImpl::Body(candidate.clone()),
+                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+            )
+            .await
+            .expect("mock backend should accept candidate");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
+
+        let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
+        let recovery_store = store.clone();
+        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
+        barrier.wait_until_paused().await;
+        let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
+            .await
+            .expect("terminal control should persist before source cleanup");
+        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
+        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
+        assert_eq!(backend.object_count().await, 0);
+        assert_eq!(backend.exact_remove_count(), 1);
+
+        recovery.abort();
+        assert!(
+            recovery
+                .await
+                .expect_err("recovery should be cancelled at the crash boundary")
+                .is_cancelled()
+        );
+        drop(barrier);
+
+        let replay = recover_transition_transaction_records(store.clone(), 100, None)
+            .await
+            .expect("terminal control should resume source cleanup without another remote delete");
+        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
+        assert_eq!(transition_transaction_record_count(store).await, 0);
+        assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19472,6 +19811,8 @@ mod tests {
         save_transition_transaction_record(store.clone(), &uploaded)
             .await
             .expect("transaction record should persist");
+        let recovery_control_id =
+            transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
 
         let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
         let recovery_store = store.clone();
@@ -19493,11 +19834,17 @@ mod tests {
             .expect("recovery should treat the lost CAS as a retained transaction");
         assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
         assert_eq!(
-            load_transition_transaction_record(store, uploaded.transaction_id)
+            load_transition_transaction_record(store.clone(), uploaded.transaction_id)
                 .await
                 .expect("newer transaction revision must remain"),
             active
         );
+        let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
+            .await
+            .expect("lost source CAS should retain a retryable recovery control");
+        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
+        assert_eq!(control.control.consecutive_failure_count, 1);
+        assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
         assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
         assert_eq!(backend.remove_count().await, 0);
     }
@@ -19799,27 +20146,15 @@ mod tests {
             not_after_unix_nanos: 1_780_000_000_000_000_000,
         })
         .expect("transaction should build");
-        let uploaded_fence = transaction
+        transaction
             .advance(
                 transaction.fence(),
                 TransitionTransactionState::Uploaded,
-                Some(TransitionRemoteVersion::versioned(remote_version)),
+                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
             )
             .expect("transaction should enter uploaded state");
-        transaction
-            .mark_cleanup_pending(
-                uploaded_fence,
-                TransitionCleanupProof {
-                    transaction_id: transaction.transaction_id,
-                    write_id: transaction.write_id,
-                    remote_object: transaction.remote_object.clone(),
-                    remote_version: transaction.remote_version.clone(),
-                    backend_fingerprint: transaction.backend_fingerprint,
-                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
-                },
-            )
-            .expect("transaction should enter cleanup pending state");
         let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
+        backend.set_put_remote_version(Some(remote_version)).await;
         backend
             .put(
                 &transaction.remote_object,
@@ -19831,6 +20166,8 @@ mod tests {
         save_transition_transaction_record(store.clone(), &transaction)
             .await
             .expect("transaction record should persist");
+        let recovery_control_id =
+            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
 
         backend.set_remove_failure(true);
         let stats = recover_transition_transaction_records(store.clone(), 100, None)
@@ -19846,6 +20183,42 @@ mod tests {
         assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
         assert_eq!(backend.exact_remove_count(), 1);
         assert_eq!(backend.object_count().await, 1);
+        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
+            .await
+            .expect("failed recovery control should persist");
+        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
+        assert_eq!(control.control.attempt_count, 1);
+        assert_eq!(control.control.consecutive_failure_count, 1);
+        assert!(
+            control
+                .control
+                .next_attempt_at_unix_nanos
+                .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
+        );
+
+        backend.set_remove_failure(false);
+        let replay = recover_transition_transaction_records(store.clone(), 100, None)
+            .await
+            .expect("recovery before the persisted deadline should be skipped");
+        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
+        assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
+
+        let retry_at = control
+            .control
+            .next_attempt_at_unix_nanos
+            .expect("retry deadline should persist");
+        let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
+            .await
+            .expect("recovery at the persisted deadline should retry the advanced source generation");
+        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
+        assert_eq!(backend.exact_remove_count(), 2);
+        assert_eq!(backend.object_count().await, 0);
+        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
+        let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
+            .await
+            .expect("completed retry control should remain inspectable");
+        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
+        assert_eq!(terminal.control.attempt_count, 2);
     }
 
     #[cfg(feature = "test-util")]
@@ -20146,6 +20519,10 @@ mod tests {
         local_commit_started
             .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
             .expect("transaction should enter local commit state");
+        let upload_started_control_id =
+            transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
+        let local_commit_control_id =
+            transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
 
         backend.set_put_remote_version(Some(remote_version)).await;
         for transaction in [&upload_started, &local_commit_started] {
@@ -20176,6 +20553,19 @@ mod tests {
         assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
         assert_eq!(backend.remove_count().await, 0);
         assert_eq!(backend.exact_remove_count(), 0);
+        let upload_started_control =
+            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
+                .await
+                .expect("upload-started control should persist");
+        assert_eq!(
+            upload_started_control.control.classification,
+            IlmRecoveryClassification::RetainedAmbiguous
+        );
+        let local_commit_control =
+            load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
+                .await
+                .expect("local-commit control should persist");
+        assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
     }
 
     #[cfg(feature = "test-util")]
@@ -20514,7 +20904,7 @@ mod tests {
             .await;
         let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None)
             .await
-            .expect("unsupported provider recovery should fail closed");
+            .expect("active unknown ownership should remain fenced before provider recovery");
         assert_eq!(
             (
                 unsupported_stats.scanned,
@@ -20523,14 +20913,28 @@ mod tests {
                 unsupported_stats.failed
             ),
             (1, 0, 1, 0),
-            "an unsupported provider probe must retain the unknown upload"
+            "active unknown ownership must retain the upload before the recovery deadline"
         );
         assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
+        let recovery_control_id =
+            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
+        assert!(matches!(
+            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await,
+            Err(Error::ConfigNotFound)
+        ));
         assert!(
             backend.contains(&transaction.remote_object).await,
-            "unsupported recovery must not delete the candidate"
+            "active ownership must not delete the candidate"
+        );
+        assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup");
+        assert!(
+            !backend
+                .op_log()
+                .await
+                .iter()
+                .any(|operation| matches!(operation, MockWarmOp::Probe { .. })),
+            "active ownership must not probe the provider"
         );
-        assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup");
 
         backend.set_transition_candidate_probe_override(None).await;
         let stats =

crates/ecstore/src/store/multipart.rs

@@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation(
 }
 
 impl ECStore {
-    #[cfg(test)]
+    #[cfg(all(test, feature = "test-util"))]
     pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
         data_movement_multipart_discovery_counts()
             .lock()
@@ -246,7 +246,7 @@ impl ECStore {
             .insert(self.id, 0);
     }
 
-    #[cfg(test)]
+    #[cfg(all(test, feature = "test-util"))]
     pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
         data_movement_multipart_discovery_counts()
             .lock()

crates/heal/src/error.rs

@@ -54,6 +54,15 @@ pub enum Error {
     #[error("Heal task execution failed: {message}")]
     TaskExecutionFailed { message: String },
 
+    /// The current page already exhausted its local retry budget. Retrying
+    /// the enclosing bucket would replay pages whose results were counted.
+    #[error("Heal listing failed for bucket {bucket}: {source}")]
+    HealListingFailed {
+        bucket: String,
+        #[source]
+        source: Box<Error>,
+    },
+
     #[error("Invalid heal type: {heal_type}")]
     InvalidHealType { heal_type: String },
 

crates/heal/src/heal/channel.rs

@@ -76,6 +76,8 @@ struct HealTaskStatusPayload<'a> {
     min_seq: u64,
     #[serde(skip_serializing_if = "Option::is_none")]
     progress: Option<&'a HealProgress>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    outcome: Option<&'a super::outcome::HealTaskOutcome>,
 }
 
 fn u64_is_zero(value: &u64) -> bool {
@@ -87,17 +89,18 @@ fn encode_heal_task_status_payload(
     mut items: Vec<HealResultItem>,
     progress: Option<&HealProgress>,
     mut truncated: bool,
-    next_seq: u64,
+    sequence: (u64, u64),
-    min_seq: u64,
+    outcome: Option<&super::outcome::HealTaskOutcome>,
 ) -> Result<(Vec<u8>, bool)> {
     loop {
         let data = serde_json::to_vec(&HealTaskStatusPayload {
             summary,
             items: &items,
             truncated,
-            next_seq,
+            next_seq: sequence.0,
-            min_seq,
+            min_seq: sequence.1,
             progress,
+            outcome,
         })
         .map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
         if data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE {
@@ -111,25 +114,21 @@ fn encode_heal_task_status_payload(
     }
 }
 
-fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
-    if !truncated {
-        return detail;
-    }
-    let truncation = "heal result items were truncated";
-    Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
-}
-
 fn encode_heal_status_response(
     summary: &str,
     items: Vec<HealResultItem>,
     progress: Option<&HealProgress>,
     detail: Option<String>,
     truncated: bool,
-    next_seq: u64,
+    sequence: (u64, u64),
-    min_seq: u64,
+    outcome: Option<&super::outcome::HealTaskOutcome>,
 ) -> Result<(Vec<u8>, Option<String>)> {
-    let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?;
+    let (summary, detail) = match outcome {
-    Ok((data, heal_status_detail(detail, truncated)))
+        Some(outcome) => outcome.legacy_status(summary, detail),
+        None => (summary, detail),
+    };
+    let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, sequence, outcome)?;
+    Ok((data, super::outcome::heal_status_detail(detail, truncated)))
 }
 
 impl HealChannelProcessor {
@@ -439,6 +438,7 @@ impl HealChannelProcessor {
                 .await
         };
 
+        let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
         let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report {
             Ok(HealTaskReport {
                 status: HealTaskStatus::Pending | HealTaskStatus::Running,
@@ -447,6 +447,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "running".to_string(),
                 None,
@@ -463,6 +464,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "running".to_string(),
                 Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
@@ -479,6 +481,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "finished".to_string(),
                 None,
@@ -495,6 +498,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "stopped".to_string(),
                 Some("heal task cancelled".to_string()),
@@ -511,6 +515,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "stopped".to_string(),
                 Some("heal task timed out".to_string()),
@@ -527,6 +532,7 @@ impl HealChannelProcessor {
                 progress,
                 next_seq,
                 min_seq,
+                ..
             }) => (
                 "stopped".to_string(),
                 Some(error),
@@ -570,8 +576,15 @@ impl HealChannelProcessor {
             }
         };
 
-        let (data, detail) =
+        let (data, detail) = encode_heal_status_response(
-            encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?;
+            &summary,
+            items,
+            progress.as_ref(),
+            detail,
+            truncated,
+            (next_seq, min_seq),
+            outcome.as_deref(),
+        )?;
 
         let response = HealChannelResponse {
             request_id: client_token,
@@ -860,7 +873,7 @@ mod tests {
             ..Default::default()
         }];
 
-        let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap();
+        let (data, detail) = encode_heal_status_response("running", items, None, None, false, (0, 0), None).unwrap();
 
         assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
         let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
@@ -869,6 +882,135 @@ mod tests {
         assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
     }
 
+    #[test]
+    fn outcome_v3_fixture_matches_canonical_owner_and_preserves_legacy_terminals() {
+        use crate::heal::outcome::*;
+        let cases: serde_json::Value = serde_json::from_str(include_str!("../../../madmin/tests/fixtures/heal-outcome-v3.json"))
+            .expect("shared client fixtures");
+        for case in cases.as_array().expect("fixture cases") {
+            if case.get("remoteResponse").is_some() {
+                continue;
+            }
+            let mut outcome = HealTaskOutcome::default();
+            let name = case["name"].as_str().expect("case name");
+            if matches!(name, "unknown" | "completed_with_errors") {
+                let disposition = if name == "unknown" {
+                    HealObjectDisposition::Unknown
+                } else {
+                    outcome.attempt_failed();
+                    HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
+                };
+                outcome.record(HealObjectOutcome {
+                    identity: HealObjectIdentity {
+                        kind: HealObjectKind::Object,
+                        bucket: "bucket".into(),
+                        object: "object".into(),
+                        version_id: None,
+                        bucket_incarnation_id: None,
+                        pool_index: None,
+                        set_index: None,
+                    },
+                    disposition,
+                    detail: None,
+                });
+            }
+            let abort = match name {
+                "cancelled" => Some(HealAbortReason::Cancelled),
+                "deadline" => Some(HealAbortReason::Deadline),
+                "untraversable" => Some(HealAbortReason::Untraversable),
+                _ => None,
+            };
+            outcome.finish(abort);
+            let expected = &case["response"];
+            let initial_detail = abort.map(|reason| {
+                match reason {
+                    HealAbortReason::Cancelled => "heal task cancelled",
+                    HealAbortReason::Deadline => "heal task timed out",
+                    HealAbortReason::Untraversable => "heal listing is untraversable",
+                }
+                .to_string()
+            });
+            let (bytes, detail) = encode_heal_status_response(
+                if abort.is_some() { "stopped" } else { "finished" },
+                Vec::new(),
+                None,
+                initial_detail,
+                true,
+                (9, 4),
+                Some(&outcome),
+            )
+            .expect("canonical owner encoding");
+            let decoded: serde_json::Value = serde_json::from_slice(&bytes).expect("wire payload");
+            assert_eq!(decoded["summary"], expected["summary"], "{name}");
+            assert_eq!(detail.unwrap_or_default(), expected["detail"].as_str().expect("detail"), "{name}");
+            assert_eq!(decoded["outcome"], expected["outcome"], "{name}");
+            assert_eq!((decoded["next_seq"].as_u64(), decoded["min_seq"].as_u64()), (Some(9), Some(4)));
+            assert!(decoded["outcome"].get("retainedObjectBytes").is_none());
+            assert!(decoded["outcome"].get("untraversable").is_none());
+        }
+    }
+
+    #[test]
+    fn outcome_v3_abort_cannot_be_hidden_by_a_finished_status() {
+        use crate::heal::outcome::{HealAbortReason, HealTaskOutcome};
+        for reason in [
+            HealAbortReason::Cancelled,
+            HealAbortReason::Deadline,
+            HealAbortReason::Untraversable,
+        ] {
+            let mut outcome = HealTaskOutcome::default();
+            outcome.finish(Some(reason));
+            let (data, detail) = encode_heal_status_response("finished", Vec::new(), None, None, false, (0, 0), Some(&outcome))
+                .expect("canonical abort adapter");
+            let json: serde_json::Value = serde_json::from_slice(&data).expect("public state");
+            assert_eq!(json["summary"], "stopped");
+            assert_eq!(json["outcome"]["execution"]["state"], "aborted");
+            assert!(detail.is_some());
+        }
+    }
+
+    #[test]
+    fn outcome_v3_payload_bound_keeps_cumulative_outcome_and_cursors() {
+        use crate::heal::outcome::*;
+        let mut outcome = HealTaskOutcome::default();
+        outcome.start();
+        for index in 0..256 {
+            outcome.record(HealObjectOutcome {
+                identity: HealObjectIdentity {
+                    kind: HealObjectKind::Object,
+                    bucket: "bucket".into(),
+                    object: format!("object-{index}"),
+                    version_id: None,
+                    bucket_incarnation_id: None,
+                    pool_index: None,
+                    set_index: None,
+                },
+                disposition: HealObjectDisposition::Unknown,
+                detail: Some("\"".repeat(1024)),
+            });
+        }
+        let retained = outcome.objects.len();
+        let items = vec![
+            HealResultItem::default(),
+            HealResultItem {
+                detail: "x".repeat(MAX_HEAL_STATUS_PAYLOAD_SIZE + 1),
+                ..Default::default()
+            },
+        ];
+        let (bytes, detail) = encode_heal_status_response("running", items, None, None, false, (9, 4), Some(&outcome))
+            .expect("bounded status with cumulative outcome");
+        assert!(bytes.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
+        let wire: serde_json::Value = serde_json::from_slice(&bytes).expect("bounded payload");
+        assert_eq!(wire["items"].as_array().expect("items").len(), 1);
+        assert_eq!(wire["truncated"], true);
+        assert_eq!((wire["next_seq"].as_u64(), wire["min_seq"].as_u64()), (Some(9), Some(4)));
+        assert_eq!(wire["outcome"]["counters"]["processed"], 256);
+        assert_eq!(wire["outcome"]["counters"]["healed"], 0);
+        assert_eq!(wire["outcome"]["objects"].as_array().expect("outcome window").len(), retained);
+        assert!(retained < 256 && outcome.objects_truncated);
+        assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
+    }
+
     #[test]
     fn admission_response_preserves_all_admission_outcomes() {
         let cases = [

crates/heal/src/heal/erasure_healer.rs

@@ -108,6 +108,144 @@ pub struct ErasureSetHealer {
     target_endpoints: Arc<[String]>,
     replacement_task_id: Option<String>,
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+}
+
+async fn acquire_page_permit(
+    semaphore: Arc<Semaphore>,
+    pacer: Option<&super::pacing::MainlinePacer>,
+    cancel: &tokio_util::sync::CancellationToken,
+) -> Result<tokio::sync::OwnedSemaphorePermit> {
+    let acquire = || async {
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            permit = semaphore.clone().acquire_owned() => permit.map_err(|err| Error::other(format!("Failed to acquire page concurrency permit: {err}"))),
+        }
+    };
+    let mut paid_pause = false;
+    loop {
+        let permit = acquire().await?;
+        if let Some(pacer) = pacer {
+            // Keep the real permit on the low-pressure path. Every acquisition
+            // gets a fresh decision, including a waiter that queued a second
+            // time. One completed pause is a bounded minimum-progress grant.
+            match pacer.admission_decision() {
+                super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
+                    drop(permit);
+                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
+                    continue;
+                }
+                _ => {}
+            }
+        }
+        return Ok(permit);
+    }
+}
+
+#[cfg(test)]
+mod mainline_pacing_tests {
+    use super::*;
+    use crate::heal::pacing::{MainlinePacer, TestPressure};
+    use rustfs_concurrency::WorkloadClass;
+    use std::sync::atomic::Ordering;
+    use tokio_util::sync::CancellationToken;
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_page_waiters_resample_after_capacity_and_release_permits() {
+        let semaphore = Arc::new(Semaphore::new(1));
+        let occupied = semaphore
+            .clone()
+            .acquire_owned()
+            .await
+            .expect("existing object owns capacity");
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
+        let pacer = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250)).expect("pacer"));
+        let cancel = CancellationToken::new();
+        let waiting = tokio::spawn({
+            let semaphore = semaphore.clone();
+            let pacer = pacer.clone();
+            let cancel = cancel.clone();
+            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
+        });
+        tokio::task::yield_now().await;
+        provider.active.store(100, Ordering::SeqCst);
+        drop(occupied);
+        provider.sampled.notified().await;
+        assert_eq!(semaphore.available_permits(), 1, "running pressure wait cannot retain the page permit");
+        cancel.cancel();
+        assert!(matches!(waiting.await.expect("page waiter"), Err(Error::TaskCancelled)));
+        assert_eq!(semaphore.available_permits(), 1);
+        let deadline = tokio::time::timeout(
+            Duration::from_millis(10),
+            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
+        )
+        .await;
+        assert!(deadline.is_err());
+        assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
+        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
+            .await
+            .expect("unpaced admission");
+        assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
+        drop(permit);
+        assert_eq!(semaphore.available_permits(), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_two_page_waiters_check_pressure_at_final_admission() {
+        use std::task::Poll;
+        for raise_pressure in [false, true] {
+            let semaphore = Arc::new(Semaphore::new(1));
+            let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
+            let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
+            let pause = Duration::from_millis(250);
+            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause).expect("pacer");
+            let cancel = CancellationToken::new();
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            assert!(futures::poll!(first.as_mut()).is_pending());
+            assert!(futures::poll!(second.as_mut()).is_pending());
+            drop(occupied);
+            let first_ready = match futures::poll!(first.as_mut()) {
+                Poll::Ready(result) => Some(result.expect("first admission")),
+                Poll::Pending => None,
+            };
+            assert!(futures::poll!(second.as_mut()).is_pending());
+            let first_permit = match first_ready {
+                Some(permit) => permit,
+                None => tokio::time::timeout(Duration::from_millis(1), first)
+                    .await
+                    .expect("low-pressure waiters must not bounce capacity forever")
+                    .expect("first permit"),
+            };
+            // The first object owns real page capacity while its commit runs.
+            tokio::time::advance(Duration::from_millis(100)).await;
+            if raise_pressure {
+                provider.active.store(100, Ordering::SeqCst);
+            }
+            drop(first_permit);
+            let admitted = if raise_pressure {
+                assert!(
+                    futures::poll!(second.as_mut()).is_pending(),
+                    "a second acquisition cannot reuse the earlier low-pressure sample"
+                );
+                assert_eq!(semaphore.available_permits(), 1, "pressure wait must release page capacity");
+                tokio::time::advance(pause).await;
+                tokio::time::timeout(Duration::from_millis(1), second)
+                    .await
+                    .expect("sustained pressure must allow one unit after its bounded pause")
+                    .expect("second permit")
+            } else {
+                tokio::time::timeout(Duration::from_millis(1), second)
+                    .await
+                    .expect("low pressure must make progress")
+                    .expect("second permit")
+            };
+            assert_eq!(semaphore.available_permits(), 0);
+            drop(admitted);
+            assert_eq!(semaphore.available_permits(), 1);
+        }
+    }
 }
 
 pub(crate) fn target_outcomes_complete(result: &HealResultItem, target_endpoints: &[String]) -> bool {
@@ -219,9 +357,15 @@ impl ErasureSetHealer {
             target_endpoints: Vec::new().into(),
             replacement_task_id: None,
             replacement_target_identities: None,
+            mainline_pacer: None,
         }
     }
 
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+        self.mainline_pacer = pacer;
+        self
+    }
+
     pub(crate) fn with_replacement_targets(
         mut self,
         mut target_endpoints: Vec<String>,
@@ -856,6 +1000,9 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
+            if let Some(pacer) = &self.mainline_pacer {
+                pacer.wait(&self.cancel_token).await?;
+            }
             self.verify_replacement_identity_fence("page scan").await?;
             // Get one page of object versions
             let (objects, next_token, is_truncated) = if use_disk_walk {
@@ -1034,13 +1181,10 @@ impl ErasureSetHealer {
                 let semaphore = semaphore.clone();
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
+                let mainline_pacer = self.mainline_pacer.clone();
 
                 page_tasks.push(async move {
-                    let permit = semaphore
+                    let permit = acquire_page_permit(semaphore, mainline_pacer.as_deref(), &cancel_token).await;
-                        .clone()
-                        .acquire_owned()
-                        .await
-                        .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")));
 
                     let _permit = match permit {
                         Ok(permit) => permit,

crates/heal/src/heal/manager.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::heal::{
+    outcome::HealTaskOutcome,
     progress::{HealProgress, HealStatistics},
     resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils},
     storage::HealStorageAPI,
@@ -185,6 +186,7 @@ fn record_displaced_terminal(
     request: &HealRequest,
 ) -> Arc<CompletedHealStatus> {
     let terminal = Arc::new(CompletedHealStatus {
+        outcome: None,
         progress: None,
         retained_bytes: std::sync::OnceLock::new(),
         heal_type: request.heal_type.clone(),
@@ -268,6 +270,7 @@ async fn publish_completed_heal(
 
 #[derive(Debug, Clone)]
 pub struct HealTaskReport {
+    pub outcome: Option<Arc<HealTaskOutcome>>,
     pub status: HealTaskStatus,
     pub result_items: Vec<HealResultItem>,
     pub result_items_truncated: bool,
@@ -285,6 +288,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
     let window = task.get_result_items_since(since).await;
     HealTaskReport {
         status: task.get_status().await,
+        outcome: Some(Arc::new(task.get_outcome().await)),
         result_items: window.items,
         // The legacy flag stays set once anything was evicted; a lagging
         // incremental cursor additionally marks this response truncated so
@@ -298,6 +302,7 @@ async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskRepo
 
 fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
     HealTaskReport {
+        outcome: None,
         status,
         result_items: Vec::new(),
         result_items_truncated: false,
@@ -325,6 +330,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
     };
     HealTaskReport {
         status: completed.status.clone(),
+        outcome: completed.outcome.clone(),
         result_items,
         result_items_truncated: completed.result_items_truncated || lagged,
         progress: completed.progress.clone(),
@@ -596,13 +602,13 @@ pub struct HealConfig {
     pub set_bulkhead_enable: bool,
     /// Whether erasure-set page parallelism is enabled.
     pub page_parallel_enable: bool,
-    /// Whether foreground read pressure can delay best-effort heal task starts.
+    /// Whether foreground pressure delays best-effort starts and paces running admin work.
     pub mainline_throttle_enable: bool,
-    /// Foreground read permit utilization percentage that delays best-effort heal starts.
+    /// Foreground read utilization high watermark for start admission and admin pacing.
     pub mainline_read_utilization_high_percent: usize,
-    /// Foreground write utilization percentage that delays best-effort heal starts.
+    /// Foreground write utilization high watermark for start admission and admin pacing.
     pub mainline_write_utilization_high_percent: usize,
-    /// Delay before rechecking foreground pressure after delaying heal starts.
+    /// Start recheck interval; running admin pacing caps each holder's pause at one second.
     pub mainline_max_sleep: Duration,
 }
 

crates/heal/src/heal/manager/queue.rs

@@ -83,6 +83,7 @@ pub(super) struct CompletedHealStatus {
     pub(super) heal_type: HealType,
     pub(super) status: HealTaskStatus,
     pub(super) progress: Option<HealProgress>,
+    pub(super) outcome: Option<Arc<HealTaskOutcome>>,
     pub(super) retained_bytes: std::sync::OnceLock<usize>,
     pub(super) result_items_truncated: bool,
     pub(super) completed_at: SystemTime,
@@ -105,6 +106,7 @@ impl CompletedHealStatus {
     fn measure_retained_bytes(&self) -> usize {
         let mut bytes = size_of::<Self>();
         let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
+        add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
         match &self.heal_type {
             HealType::Cluster => {}
             HealType::Bucket { bucket } => add(bucket.capacity()),
@@ -209,6 +211,7 @@ impl CompletedHealStatus {
             heal_type: task.heal_type.clone(),
             status,
             progress: Some(task.get_progress().await),
+            outcome: Some(Arc::new(task.get_outcome().await)),
             retained_bytes: std::sync::OnceLock::new(),
             result_items_truncated: task.result_items_truncated(),
             completed_at: SystemTime::now(),

crates/heal/src/heal/manager/scheduler.rs

@@ -175,11 +175,23 @@ impl HealManager {
                     .unwrap_or_else(|poisoned| poisoned.into_inner())
                     .get(&request.id)
                     .cloned();
-                let task = Arc::new(HealTask::from_replacement_recovery_request(
+                let mainline_pacer = if request.source == HealRequestSource::Admin && config.mainline_throttle_enable {
-                    request,
+                    workload_provider.as_ref().and_then(|provider| {
-                    storage.clone(),
+                        crate::heal::pacing::MainlinePacer::new(
-                    replacement_resume_endpoint,
+                            provider.clone(),
-                ));
+                            config.mainline_read_utilization_high_percent,
+                            config.mainline_write_utilization_high_percent,
+                            config.mainline_max_sleep,
+                        )
+                        .map(Arc::new)
+                    })
+                } else {
+                    None
+                };
+                let task = Arc::new(
+                    HealTask::from_replacement_recovery_request(request, storage.clone(), replacement_resume_endpoint)
+                        .with_mainline_pacer(mainline_pacer),
+                );
                 let task_id = task.id.clone();
                 active_heals_guard.insert(task_id.clone(), task.clone());
                 publish_active_heal_count(&active_heals_guard);
@@ -298,9 +310,9 @@ impl HealManager {
                     if cancelled_completion {
                         completed_status = HealTaskStatus::Cancelled;
                         completed_status_entry.status = HealTaskStatus::Cancelled;
+                        completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
                     }
                     let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
-                    let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
                     // Keep retry ownership continuous: status snapshots acquire
                     // these locks in the same active -> retrying order.
                     let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -362,11 +374,11 @@ impl HealManager {
                         drop(stats);
                         if terminal_completion {
                             let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
-                            if successful_completion {
+                            // Neither task status nor the diagnostic outcome
-                                emit_mrf_repaired_events(notice_targets);
+                            // window supplies a storage-owned repair receipt.
-                            } else {
+                            // Release only the ingress lease for rediscovery;
-                                release_mrf_repair_notice_targets(notice_targets);
+                            // preserve the producer's existing retry hints.
-                            }
+                            release_mrf_repair_notice_targets(notice_targets);
                         }
                     }
 
@@ -693,20 +705,6 @@ fn move_mrf_repair_notice_targets(
     }
 }
 
-fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
-    for target in targets {
-        rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
-        rustfs_common::mrf_channel::release_mrf_identity(
-            target.kind,
-            &target.bucket,
-            &target.object,
-            target.version_id,
-            target.scope,
-            target.lease,
-        );
-    }
-}
-
 fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
     for target in targets {
         rustfs_common::mrf_channel::release_mrf_identity(

crates/heal/src/heal/manager/tests.rs

@@ -14,6 +14,9 @@
 
 use super::*;
 use crate::heal::EcstoreError;
+use crate::heal::outcome::{
+    HealAbortReason, HealDeferredReason, HealExecutionOutcome, HealObjectDisposition, HealTraversalCoverage,
+};
 use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity};
 use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
 use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
@@ -23,6 +26,8 @@ use rustfs_madmin::heal_commands::HealResultItem;
 use std::sync::Mutex as StdMutex;
 use tempfile::TempDir;
 
+mod running_mainline;
+
 use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
 
 #[tokio::test]
@@ -103,6 +108,7 @@ struct MockStorage;
 
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
+        outcome: None,
         heal_type: HealType::Cluster,
         status: HealTaskStatus::Completed,
         progress: Some(HealProgress {
@@ -287,6 +293,59 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat
     }
 }
 
+#[tokio::test]
+async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
+    use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
+    use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
+    let bucket = "canonical-outcome-cancel-before-finish";
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
+    let alias = duplicate.id.clone();
+    let retention_hook = Arc::new(CompletedRetentionHook::default());
+    {
+        let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
+        hooks.insert(bucket.to_string(), retention_hook.clone());
+        hooks.insert(task_id.clone(), retention_hook.clone());
+    }
+    let finish_hook = Arc::new(OutcomeFinishTestHook {
+        task_id: task_id.clone(),
+        reached: Notify::new(),
+        release: Notify::new(),
+    });
+    *OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone());
+    manager.submit_heal_request(request).await.expect("admit original");
+    manager.submit_heal_request(duplicate).await.expect("admit alias");
+    process_manager_queue_once(&manager).await;
+    tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified())
+        .await
+        .expect("storage started");
+    retention_hook.execute.notify_one();
+    tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified())
+        .await
+        .expect("storage returned before outcome finalization");
+    manager.cancel_task(&alias).await.expect("cancel wins publication");
+    finish_hook.release.notify_one();
+    tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified())
+        .await
+        .expect("scheduler completes cancelled handoff");
+    for token in [&task_id, &alias] {
+        let report = manager.get_task_report(token).await.expect("cancelled token retained");
+        assert_eq!(report.status, HealTaskStatus::Cancelled);
+        assert_eq!(
+            report.outcome.as_ref().expect("frozen outcome").execution,
+            HealExecutionOutcome::Aborted(HealAbortReason::Cancelled)
+        );
+    }
+    retention_hook.finish.notify_one();
+    *OUTCOME_FINISH_TEST_HOOK.lock().await = None;
+    COMPLETED_RETENTION_HOOKS
+        .lock()
+        .await
+        .retain(|key, _| key != bucket && key != &task_id);
+}
+
 #[tokio::test]
 async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
     let bucket = "completed-retention-retry-cancel";
@@ -325,6 +384,10 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
     for token in [&task_id, &alias] {
         let report = manager.get_task_report(token).await.expect("cancelled token retained");
         assert_eq!(report.status, HealTaskStatus::Cancelled);
+        assert_eq!(
+            report.outcome.as_ref().expect("cancelled outcome retained").execution,
+            crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled)
+        );
         assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
     }
     assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
@@ -391,6 +454,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
             .expect("scheduler archives terminal");
         assert!(!manager.active_heals.lock().await.contains_key(&task_id));
         let expected = task.get_progress().await;
+        let expected_outcome = task.get_outcome().await;
         for token in [&task_id, &alias] {
             assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
             let report = manager
@@ -398,6 +462,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
                 .await
                 .expect("terminal token remains queryable at handoff");
             assert_eq!(report.progress.as_ref(), Some(&expected));
+            assert_eq!(report.outcome.as_deref(), Some(&expected_outcome));
             assert!(report.result_items.is_empty());
             match outcome {
                 "success" => assert_eq!(report.status, HealTaskStatus::Completed),
@@ -455,10 +520,15 @@ impl HealStorageAPI for MockStorage {
     async fn heal_object(
         &self,
         bucket: &str,
-        _object: &str,
+        object: &str,
         _version_id: Option<&str>,
         _opts: &HealOpts,
     ) -> Result<(HealResultItem, Option<Error>)> {
+        if bucket.starts_with("heal-start-retry-deadline-object-") && object == "blocked" {
+            let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
+            hook.started.notify_one();
+            std::future::pending::<()>().await;
+        }
         if bucket == "completed-retention-failed" {
             return Err(Error::TaskExecutionFailed {
                 message: "retention fixture failure".to_string(),
@@ -520,11 +590,35 @@ impl HealStorageAPI for MockStorage {
 
     async fn list_objects_for_heal_page(
         &self,
-        _bucket: &str,
+        bucket: &str,
         _prefix: &str,
-        _continuation_token: Option<&str>,
+        continuation_token: Option<&str>,
         _include_lifecycle_object_info: bool,
     ) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
+        if bucket.starts_with("heal-start-retry-deadline-") {
+            if continuation_token.is_some() {
+                let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
+                hook.started.notify_one();
+                std::future::pending::<()>().await;
+            }
+            let listing_timeout = bucket.starts_with("heal-start-retry-deadline-listing-");
+            let names = if listing_timeout {
+                vec!["completed"]
+            } else {
+                vec!["completed", "blocked"]
+            };
+            let objects = names
+                .into_iter()
+                .map(|name| crate::heal::storage::HealListItem {
+                    name: name.to_string(),
+                    version_id: None,
+                    mod_time_unix_nanos: None,
+                    lifecycle_object_info: None,
+                    is_delete_marker: false,
+                })
+                .collect();
+            return Ok((objects, listing_timeout.then(|| "next".to_string()), listing_timeout));
+        }
         Ok((Vec::new(), None, false))
     }
 
@@ -547,6 +641,161 @@ impl HealStorageAPI for MockStorage {
     }
 }
 
+async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) {
+    for phase in ["listing", "object"] {
+        let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}");
+        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let mut request = HealRequest::new(
+            HealType::Prefix {
+                bucket: bucket.clone(),
+                prefix: String::new(),
+            },
+            HealOptions {
+                timeout: Some(if cancel {
+                    Duration::from_secs(60)
+                } else {
+                    Duration::from_millis(200)
+                }),
+                ..Default::default()
+            },
+            HealPriority::High,
+        );
+        request.source = HealRequestSource::Admin;
+        let task_id = request.id.clone();
+        let hook = Arc::new(CompletedRetentionHook::default());
+        {
+            let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
+            hooks.insert(bucket.clone(), Arc::clone(&hook));
+            hooks.insert(task_id.clone(), Arc::clone(&hook));
+        }
+        manager.submit_heal_request(request).await.expect("admit deadline task");
+        process_manager_queue_once(&manager).await;
+        tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
+            .await
+            .expect("executor reaches blocked storage");
+        let active = manager.get_task_report(&task_id).await.expect("active report");
+        assert_eq!(active.progress.expect("real completed object progress").objects_healed, 1);
+        if cancel {
+            manager.active_heals.lock().await[&task_id].cancel_token.cancel();
+        }
+        tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+            .await
+            .expect("deadline archives task");
+        let report = manager.get_task_report(&task_id).await.expect("terminal report");
+        assert_eq!(
+            report.status,
+            if cancel {
+                HealTaskStatus::Cancelled
+            } else {
+                HealTaskStatus::Timeout
+            },
+            "blocked {phase}"
+        );
+        let progress = report.progress.expect("terminal progress retained");
+        assert_eq!(progress.objects_healed, 1);
+        assert_eq!(progress.objects_failed, 0, "interrupted object has no terminal storage result");
+        assert_eq!(report.result_items.len(), 1, "completed result retained");
+        let outcome = report.outcome.expect("canonical terminal outcome retained");
+        assert_eq!(
+            outcome.execution,
+            HealExecutionOutcome::Aborted(if cancel {
+                HealAbortReason::Cancelled
+            } else {
+                HealAbortReason::Deadline
+            })
+        );
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(outcome.counters.healed, 0, "legacy success supplies no authoritative repair proof");
+        let completed = outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "completed")
+            .expect("completed object diagnostic retained");
+        assert_eq!(completed.disposition, HealObjectDisposition::Unknown);
+        if phase == "object" {
+            let interrupted = outcome
+                .objects
+                .iter()
+                .find(|item| item.identity.object == "blocked")
+                .expect("interrupted object diagnostic retained");
+            assert_eq!(
+                interrupted.disposition,
+                if cancel {
+                    HealObjectDisposition::Cancelled
+                } else {
+                    HealObjectDisposition::Deferred {
+                        reason: HealDeferredReason::Deadline,
+                        retry_not_before: None,
+                    }
+                }
+            );
+        } else {
+            assert_eq!(outcome.objects.len(), 1, "an unread page cannot supply object identities");
+        }
+        assert!(!manager.active_heals.lock().await.contains_key(&task_id));
+        assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
+        assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
+        hook.finish.notify_one();
+        COMPLETED_RETENTION_HOOKS
+            .lock()
+            .await
+            .retain(|key, _| key != &bucket && key != &task_id);
+    }
+}
+
+#[tokio::test]
+async fn heal_start_retry_deadline_preserves_real_executor_progress() {
+    assert_heal_start_retry_control_preserves_real_executor_progress(false).await;
+}
+
+#[tokio::test]
+async fn heal_start_retry_cancellation_preserves_real_executor_progress() {
+    assert_heal_start_retry_control_preserves_real_executor_progress(true).await;
+}
+
+#[tokio::test]
+async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
+    let manager = HealManager::new(
+        Arc::new(MockStorage),
+        Some(HealConfig {
+            task_timeout: Duration::ZERO,
+            ..Default::default()
+        }),
+    );
+    let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    request.source = HealRequestSource::Admin;
+    request.options.timeout = Some(Duration::from_secs(60));
+    let task_id = request.id.clone();
+    let created_at = request.created_at;
+    let hook = Arc::new(CompletedRetentionHook::default());
+    COMPLETED_RETENTION_HOOKS
+        .lock()
+        .await
+        .insert(task_id.clone(), Arc::clone(&hook));
+    manager
+        .submit_heal_request(request)
+        .await
+        .expect("admit explicit-budget task");
+    process_manager_queue_once(&manager).await;
+    tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+        .await
+        .expect("real read-quorum failure prepares retry");
+    let retry = manager.retrying_heals.lock().await[&task_id].request.clone();
+    assert_eq!(retry.id, task_id);
+    assert_eq!(retry.created_at, created_at);
+    assert_eq!(retry.source, HealRequestSource::Admin);
+    assert_eq!(retry.retry_attempts, 1);
+    let remaining = retry.options.timeout.expect("retry retains explicit budget");
+    assert!(remaining > Duration::ZERO && remaining < Duration::from_secs(60));
+    assert!(matches!(
+        manager.get_task_status(&task_id).await.expect("retry remains queryable"),
+        HealTaskStatus::Retrying { retry_attempt: 1, .. }
+    ));
+    manager.cancel_task(&task_id).await.expect("cancel held retry");
+    hook.finish.notify_one();
+    COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id);
+}
+
 struct ManagerRecoveryTestHook {
     replacement_resume_disk: DiskStore,
     listed: StdMutex<bool>,
@@ -1976,6 +2225,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
         task_id,
         Arc::new(CompletedHealStatus {
             progress: None,
+            outcome: None,
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type,
             status: HealTaskStatus::Retrying {
@@ -2700,6 +2950,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
         task_id.clone(),
         Arc::new(CompletedHealStatus {
             progress: None,
+            outcome: None,
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type.clone(),
             status: HealTaskStatus::Retrying {
@@ -2737,6 +2988,7 @@ async fn test_get_task_status_reads_recent_completed_status() {
         "completed-token".to_string(),
         Arc::new(CompletedHealStatus {
             progress: None,
+            outcome: None,
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Bucket {
                 bucket: "bucket".to_string(),
@@ -2768,6 +3020,7 @@ async fn test_get_task_report_for_path_reads_completed_items() {
         "completed-token".to_string(),
         Arc::new(CompletedHealStatus {
             progress: None,
+            outcome: None,
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Object {
                 bucket: "bucket".to_string(),
@@ -2843,7 +3096,7 @@ async fn test_cancel_task_removes_queued_request() {
 }
 
 #[tokio::test]
-async fn test_mrf_repaired_notice_waits_for_successful_completion() {
+async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
     let bucket = "mrf-completion-success";
     let object = "object";
     let version_id = Some([9u8; 16]);
@@ -2873,21 +3126,81 @@ async fn test_mrf_repaired_notice_waits_for_successful_completion() {
     );
 
     process_manager_queue_once(&manager).await;
-    for _ in 0..100 {
+    tokio::time::timeout(Duration::from_secs(5), async {
-        let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
+        loop {
-        if !events.is_empty() {
+            let stats = manager.get_statistics().await;
-            assert_eq!(events.len(), 1);
+            if stats.successful_tasks + stats.failed_tasks > 0 {
-            assert_eq!(events[0].object.as_ref(), object);
+                break;
-            assert_eq!(events[0].version_id, version_id);
+            }
-            return;
+            tokio::task::yield_now().await;
         }
-        tokio::time::sleep(Duration::from_millis(10)).await;
+    })
-    }
+    .await
-    panic!("successful MRF-owned heal should emit one repaired event");
+    .expect("scheduler completes the task");
+    assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
+    assert!(!lock_mrf_repair_notice_targets(&manager.mrf_repair_notice_targets).contains_key(&receipt.task_id));
 }
 
 #[tokio::test]
-async fn test_mrf_repaired_notice_removed_on_queued_cancel_without_event() {
+async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() {
+    for empty_window in [false, true] {
+        let bucket = if empty_window {
+            "mrf-empty-outcome"
+        } else {
+            "mrf-dry-run-outcome"
+        };
+        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let request = HealRequest::new(
+            if empty_window {
+                HealType::Cluster
+            } else {
+                HealType::Object {
+                    bucket: bucket.to_string(),
+                    object: "object".to_string(),
+                    version_id: None,
+                }
+            },
+            HealOptions {
+                recursive: true,
+                dry_run: !empty_window,
+                recreate_missing: true,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let receipt = manager
+            .submit_mrf_heal_request_with_receipt(request, Arc::from(bucket), Arc::from("object"), None)
+            .await
+            .expect("notice target registered");
+        process_manager_queue_once(&manager).await;
+        tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                let stats = manager.get_statistics().await;
+                if stats.successful_tasks + stats.failed_tasks > 0 {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("scheduler completed");
+        let report = manager.get_task_report(&receipt.task_id).await.expect("completed report");
+        assert_eq!(report.status, HealTaskStatus::Completed);
+        let outcome = report.outcome.expect("canonical outcome");
+        if empty_window {
+            assert!(outcome.objects.is_empty());
+        } else {
+            assert_eq!(
+                outcome.objects[0].disposition,
+                crate::heal::outcome::HealObjectDisposition::DryRunObserved
+            );
+        }
+        assert!(rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket).is_empty());
+    }
+}
+
+#[tokio::test]
+async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
     let bucket = "mrf-completion-cancel";
     let object = "object";
     let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);

crates/heal/src/heal/manager/tests/running_mainline.rs

@@ -0,0 +1,249 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use crate::heal::storage::HealListItem;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use tokio::sync::Semaphore;
+
+#[derive(Default)]
+struct PressureProbe {
+    active: AtomicUsize,
+    commit_open: AtomicBool,
+    high_sampled: Notify,
+}
+
+impl WorkloadAdmissionSnapshotProvider for PressureProbe {
+    fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
+        assert!(
+            !self.commit_open.load(Ordering::SeqCst),
+            "pressure must not be sampled inside an object commit"
+        );
+        let active = self.active.load(Ordering::SeqCst);
+        if active >= 80 {
+            self.high_sampled.notify_one();
+        }
+        WorkloadAdmissionRegistrySnapshot::new(vec![
+            WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
+                Some(active),
+                None,
+                Some(100),
+            ),
+        ])
+    }
+}
+
+struct RunningStorage {
+    provider: Arc<PressureProbe>,
+    namespace: Mutex<()>,
+    io: Arc<Semaphore>,
+    first_started: Notify,
+    release_first: Notify,
+    first_finished: Notify,
+    second_finished: Notify,
+    started: AtomicUsize,
+    committed: AtomicUsize,
+}
+
+#[async_trait::async_trait]
+impl HealStorageAPI for RunningStorage {
+    async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
+        Ok(None)
+    }
+    async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
+        Ok(Vec::new())
+    }
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+        Ok(Some(BucketInfo {
+            name: bucket.into(),
+            ..Default::default()
+        }))
+    }
+    async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
+        Ok(Vec::new())
+    }
+    async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
+        Ok(true)
+    }
+    async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
+        Ok(HealResultItem::default())
+    }
+    async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
+        Ok((HealResultItem::default(), None))
+    }
+    async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
+        Err(Error::other("no resume disk in bucket fixture"))
+    }
+    async fn list_objects_for_heal_page(
+        &self,
+        _: &str,
+        _: &str,
+        _: Option<&str>,
+        _: bool,
+    ) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
+        Ok((
+            ["a", "b"]
+                .into_iter()
+                .map(|name| HealListItem {
+                    name: name.into(),
+                    version_id: None,
+                    mod_time_unix_nanos: None,
+                    lifecycle_object_info: None,
+                    is_delete_marker: false,
+                })
+                .collect(),
+            None,
+            false,
+        ))
+    }
+    async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
+        let permit = self.io.clone().acquire_owned().await.expect("fixture I/O permit");
+        let namespace = self.namespace.lock().await;
+        self.provider.commit_open.store(true, Ordering::SeqCst);
+        let index = self.started.fetch_add(1, Ordering::SeqCst);
+        if index == 0 {
+            self.first_started.notify_one();
+            self.release_first.notified().await;
+        }
+        self.committed.fetch_add(1, Ordering::SeqCst);
+        self.provider.commit_open.store(false, Ordering::SeqCst);
+        drop(namespace);
+        drop(permit);
+        if index == 0 {
+            self.first_finished.notify_one();
+        } else {
+            self.second_finished.notify_one();
+        }
+        Ok((
+            HealResultItem {
+                object_size: 1,
+                ..Default::default()
+            },
+            None,
+        ))
+    }
+}
+
+async fn start_fixture(
+    provider_enabled: bool,
+    pacing_enabled: bool,
+    timeout: Duration,
+) -> (HealManager, Arc<RunningStorage>, Arc<PressureProbe>, Arc<HealTask>) {
+    let provider = Arc::new(PressureProbe::default());
+    let storage = Arc::new(RunningStorage {
+        provider: provider.clone(),
+        namespace: Mutex::new(()),
+        io: Arc::new(Semaphore::new(1)),
+        first_started: Notify::new(),
+        release_first: Notify::new(),
+        first_finished: Notify::new(),
+        second_finished: Notify::new(),
+        started: AtomicUsize::new(0),
+        committed: AtomicUsize::new(0),
+    });
+    let manager = HealManager::new_with_workload_provider(
+        storage.clone(),
+        Some(HealConfig {
+            mainline_throttle_enable: pacing_enabled,
+            mainline_read_utilization_high_percent: 80,
+            mainline_write_utilization_high_percent: 80,
+            mainline_max_sleep: Duration::from_millis(250),
+            max_concurrent_heals: 1,
+            ..HealConfig::default()
+        }),
+        provider_enabled.then(|| provider.clone() as WorkloadSnapshotProviderRef),
+    );
+    let mut request = bucket_request("running-mainline", HealPriority::High, HealRequestSource::Admin);
+    request.options.recursive = true;
+    request.options.timeout = Some(timeout);
+    let task_id = request.id.clone();
+    manager.submit_heal_request(request).await.expect("queue admin heal");
+    process_manager_queue_once(&manager).await;
+    storage.first_started.notified().await;
+    let task = manager
+        .active_heals
+        .lock()
+        .await
+        .get(&task_id)
+        .cloned()
+        .expect("running task");
+    (manager, storage, provider, task)
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_admin_resamples_after_commit_and_yields_without_io_guards() {
+    let (_manager, storage, provider, _task) = start_fixture(true, true, Duration::from_secs(60)).await;
+    provider.active.store(100, Ordering::SeqCst);
+    assert!(storage.provider.commit_open.load(Ordering::SeqCst));
+    assert_eq!(storage.committed.load(Ordering::SeqCst), 0);
+    storage.release_first.notify_one();
+    storage.first_finished.notified().await;
+    tokio::time::timeout(Duration::from_millis(1), provider.high_sampled.notified())
+        .await
+        .expect("running admin heal must re-sample rising pressure before its next object");
+    assert_eq!(storage.started.load(Ordering::SeqCst), 1);
+    assert_eq!(
+        storage.committed.load(Ordering::SeqCst),
+        1,
+        "in-flight commit must finish despite pressure"
+    );
+    assert_eq!(storage.io.available_permits(), 1, "pacing must release I/O permits");
+    assert!(storage.namespace.try_lock().is_ok(), "pacing must not hold the namespace lock");
+    tokio::time::advance(Duration::from_millis(250)).await;
+    storage.second_finished.notified().await;
+    assert_eq!(
+        storage.committed.load(Ordering::SeqCst),
+        2,
+        "sustained pressure must still allow bounded maintenance progress"
+    );
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_missing_provider_or_disabled_pacing_preserves_progress() {
+    for (provider_enabled, pacing_enabled) in [(false, true), (true, false)] {
+        let (_manager, storage, provider, _task) = start_fixture(provider_enabled, pacing_enabled, Duration::from_secs(60)).await;
+        provider.active.store(100, Ordering::SeqCst);
+        let before = tokio::time::Instant::now();
+        storage.release_first.notify_one();
+        storage.second_finished.notified().await;
+        assert_eq!(storage.committed.load(Ordering::SeqCst), 2);
+        assert_eq!(tokio::time::Instant::now(), before);
+        assert_eq!(storage.io.available_permits(), 1);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_cancellation_and_deadline_leave_next_object_unstarted() {
+    for cancelled in [true, false] {
+        let (_manager, storage, provider, task) = start_fixture(true, true, Duration::from_millis(100)).await;
+        provider.active.store(100, Ordering::SeqCst);
+        storage.release_first.notify_one();
+        provider.high_sampled.notified().await;
+        if cancelled {
+            task.cancel_token.cancel();
+        } else {
+            tokio::time::advance(Duration::from_millis(100)).await;
+        }
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while matches!(task.get_status().await, HealTaskStatus::Running) {
+                tokio::time::sleep(Duration::from_millis(1)).await;
+            }
+        })
+        .await
+        .expect("pacing must not mask cancellation or timeout");
+        assert_eq!(storage.started.load(Ordering::SeqCst), 1);
+        assert_eq!(storage.committed.load(Ordering::SeqCst), 1);
+        assert_eq!(storage.io.available_permits(), 1);
+        assert!(storage.namespace.try_lock().is_ok());
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.counters.processed, 1);
+        assert_eq!(
+            task.get_status().await,
+            if cancelled {
+                HealTaskStatus::Cancelled
+            } else {
+                HealTaskStatus::Timeout
+            }
+        );
+    }
+}

crates/heal/src/heal/mod.rs

@@ -16,6 +16,8 @@ pub mod channel;
 pub mod erasure_healer;
 pub mod manager;
 pub mod mrf_queue;
+pub mod outcome;
+pub(crate) mod pacing;
 pub mod progress;
 pub(crate) mod replacement_readiness;
 pub mod resume;

crates/heal/src/heal/mrf_queue.rs

@@ -25,12 +25,13 @@
 //! set, rewritten on a group-commit cadence (every flush interval or flush
 //! threshold new intents). A rewrite is atomic at the record level only — a
 //! torn tail simply truncates during replay because every record carries its
-//! own CRC32. Losing the last flush window (≤500 ms) is acceptable because
+//! own CRC32. Neither ingress nor manager admission is a durable ownership
-//! every producer keeps its own safety net: read-repair re-detects on the
+//! receipt. The last flush window can be lost. Read-repair can rediscover a
-//! next failing read, and the scanner's corrupt-metadata branch leaves a
+//! failed read; the scanner retains bounded, expiring retry hints. Partial
-//! pending-ledger entry behind even when its MRF intent is accepted
+//! writes also use a best-effort in-memory fast path, not a durable successor.
-//! (backlog#1894 axis A), so a lost intent is retried by the ledger rather
+//! These mechanisms must not be reported as verified repair completion.
-//! than waiting for the failed-object TTL to re-scan the path.
+//! The partial-write caller's restart-survival requirement remains unmet by
+//! admission alone; a verified durable handoff is still required.
 
 use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
 use crate::heal::manager::{HealManager, MrfRepairNoticeTarget};
@@ -585,8 +586,8 @@ impl MrfRuntime {
             self.dirty = true;
             match submit_mrf_heal_request(manager, &intent).await {
                 // Accepted intents leave the pending set; the next flush persists the
-                // smaller snapshot. The scanner ledger is cleared later, when the
+                // smaller snapshot. This is not a durable successor receipt and
-                // canonical heal task reaches a successful terminal completion.
+                // does not discharge the producer's existing retry hints.
                 Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
                 Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
                     intent.attempts = intent.attempts.saturating_add(1);

crates/heal/src/heal/mrf_queue/snapshot.rs

@@ -33,6 +33,9 @@ use std::collections::HashMap;
 use tokio::io::AsyncReadExt;
 use uuid::Uuid;
 
+/// Explicit pending migration; never activates the production writer or GC.
+pub mod migration;
+
 // Root-level control files avoid requiring a new directory before the first
 // atomic commit. They remain inside the storage owner's metadata volume.
 const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
@@ -66,6 +69,23 @@ struct Manifest {
 }
 
 impl Manifest {
+    fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
+        let mut bytes = Vec::with_capacity(MANIFEST_LEN);
+        bytes.extend_from_slice(MAGIC);
+        bytes.push(VERSION);
+        bytes.extend_from_slice(owner.as_bytes());
+        bytes.extend_from_slice(&sequence.to_le_bytes());
+        bytes.extend_from_slice(
+            &u64::try_from(payload.len())
+                .map_err(|_| SnapshotError::TooLarge)?
+                .to_le_bytes(),
+        );
+        bytes.extend_from_slice(&Sha256::digest(payload));
+        bytes.extend_from_slice(&Sha256::digest(&bytes));
+        Self::decode(&bytes, payload.len())?;
+        Ok(bytes)
+    }
+
     fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
         if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
             return Err(SnapshotError::Corrupt);

crates/heal/src/heal/outcome.rs

@@ -0,0 +1,473 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Execution results are separate from repair responsibility. A legacy
+//! successful storage call supplies no authoritative repair receipt.
+
+use serde::{Deserialize, Serialize};
+use std::{collections::VecDeque, time::SystemTime};
+use uuid::Uuid;
+
+const MAX_OUTCOME_ITEMS: usize = 128;
+const MAX_OUTCOME_BYTES: usize = 64 * 1024;
+const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealObjectKind {
+    Object,
+    Metadata,
+    Decode,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HealObjectIdentity {
+    pub kind: HealObjectKind,
+    pub bucket: String,
+    pub object: String,
+    /// The requested version; None remains unresolved, never an absence proof.
+    pub version_id: Option<String>,
+    pub bucket_incarnation_id: Option<Uuid>,
+    pub pool_index: Option<usize>,
+    pub set_index: Option<usize>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealDeferredReason {
+    DanglingDeleteGrace,
+    TransientUsageCache,
+    TransientExistenceCheck,
+    Deadline,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealFailureClass {
+    Recoverable,
+    RetryExhausted,
+    Permanent,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(
+    tag = "state",
+    content = "details",
+    rename_all = "snake_case",
+    rename_all_fields = "camelCase"
+)]
+pub enum HealObjectDisposition {
+    /// The legacy storage response does not prove the requested check or commit.
+    Unknown,
+    Repaired,
+    VerifiedHealthy,
+    AuthoritativelyAbsent,
+    Deferred {
+        reason: HealDeferredReason,
+        retry_not_before: Option<SystemTime>,
+    },
+    Failed(HealFailureClass),
+    Cancelled,
+    DryRunObserved,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HealObjectOutcome {
+    pub identity: HealObjectIdentity,
+    pub disposition: HealObjectDisposition,
+    pub detail: Option<String>,
+}
+
+impl HealObjectOutcome {
+    fn retained_bytes(&self) -> usize {
+        size_of::<Self>()
+            .saturating_add(self.identity.bucket.capacity())
+            .saturating_add(self.identity.object.capacity())
+            .saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity))
+            .saturating_add(self.detail.as_ref().map_or(0, String::capacity))
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealTraversalCoverage {
+    #[default]
+    Unknown,
+    Partial,
+    Complete,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum HealAbortReason {
+    Cancelled,
+    Deadline,
+    Untraversable,
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "state", content = "reason", rename_all = "snake_case")]
+pub enum HealExecutionOutcome {
+    #[default]
+    Pending,
+    Running,
+    Completed,
+    CompletedWithErrors,
+    Aborted(HealAbortReason),
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HealOutcomeCounters {
+    pub processed: u64,
+    pub healed: u64,
+    pub unchanged: u64,
+    /// Deferred, cancelled, dry-run and unverified results remain unresolved.
+    pub skipped: u64,
+    pub failed: u64,
+    pub unknown: u64,
+    pub attempt_failures: u64,
+    pub overflowed: bool,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HealTaskOutcome {
+    pub execution: HealExecutionOutcome,
+    pub coverage: HealTraversalCoverage,
+    pub counters: HealOutcomeCounters,
+    /// A bounded diagnostic window, not a complete responsibility ledger.
+    pub objects: VecDeque<HealObjectOutcome>,
+    pub objects_truncated: bool,
+    #[serde(skip)]
+    retained_object_bytes: usize,
+    #[serde(skip)]
+    untraversable: bool,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum HealOutcomeWireError {
+    #[error("heal outcome is missing execution or counters")]
+    MissingFields,
+    #[error("heal outcome has invalid or unsupported execution fields")]
+    InvalidFields(#[from] serde_json::Error),
+    #[error("finished heal summary contradicts its canonical outcome")]
+    ContradictoryCompletion,
+}
+
+/// Reconcile a peer's successful legacy summary using the canonical owner types.
+/// A running retry may legitimately retain the preceding attempt's outcome.
+pub fn legacy_wire_status<'a>(
+    summary: &'a str,
+    wire: &serde_json::Value,
+    truncated: bool,
+) -> Result<(&'a str, Option<String>), HealOutcomeWireError> {
+    if summary != "finished" {
+        return Ok((summary, None));
+    }
+    let execution = HealExecutionOutcome::deserialize(wire.get("execution").ok_or(HealOutcomeWireError::MissingFields)?)?;
+    let counters = HealOutcomeCounters::deserialize(wire.get("counters").ok_or(HealOutcomeWireError::MissingFields)?)?;
+    if matches!(execution, HealExecutionOutcome::Pending | HealExecutionOutcome::Running)
+        || (execution == HealExecutionOutcome::Completed && counters.failed > 0)
+    {
+        return Err(HealOutcomeWireError::ContradictoryCompletion);
+    }
+    let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
+    Ok((
+        adapted,
+        if adapted != summary {
+            heal_status_detail(detail, truncated)
+        } else {
+            None
+        },
+    ))
+}
+
+pub(crate) fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
+    if !truncated {
+        return detail;
+    }
+    let truncation = "heal result items were truncated";
+    Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
+}
+
+fn legacy_execution_status<'a>(
+    summary: &'a str,
+    detail: Option<String>,
+    execution: HealExecutionOutcome,
+    counters: &HealOutcomeCounters,
+) -> (&'a str, Option<String>) {
+    if summary != "finished" {
+        return (summary, detail);
+    }
+    match execution {
+        HealExecutionOutcome::CompletedWithErrors => (
+            "stopped",
+            Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
+        ),
+        HealExecutionOutcome::Aborted(reason) => {
+            let reason = match reason {
+                HealAbortReason::Cancelled => "cancelled",
+                HealAbortReason::Deadline => "timed out",
+                HealAbortReason::Untraversable => "untraversable",
+            };
+            ("stopped", Some(format!("heal task {reason}")))
+        }
+        HealExecutionOutcome::Completed if counters.unknown > 0 => (
+            summary,
+            Some(format!(
+                "heal traversal completed; authoritative storage proof is unavailable for {} objects",
+                counters.unknown
+            )),
+        ),
+        HealExecutionOutcome::Pending | HealExecutionOutcome::Running => {
+            ("running", Some("heal execution has not reached a terminal outcome".to_string()))
+        }
+        HealExecutionOutcome::Completed => (summary, detail),
+    }
+}
+
+impl HealTaskOutcome {
+    pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
+        legacy_execution_status(summary, detail, self.execution, &self.counters)
+    }
+
+    pub(crate) fn start(&mut self) {
+        if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
+            self.execution = HealExecutionOutcome::Running;
+        }
+        self.coverage = HealTraversalCoverage::Partial;
+    }
+
+    pub(crate) fn attempt_failed(&mut self) {
+        self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures);
+    }
+
+    pub(crate) fn mark_untraversable(&mut self) {
+        self.untraversable = true;
+        self.coverage = HealTraversalCoverage::Partial;
+    }
+
+    pub(crate) fn finish(&mut self, abort: Option<HealAbortReason>) {
+        if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
+            return;
+        }
+        let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable));
+        self.execution = match abort {
+            Some(reason) => HealExecutionOutcome::Aborted(reason),
+            None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors,
+            None => HealExecutionOutcome::Completed,
+        };
+        self.coverage = if abort.is_none() && !self.counters.overflowed {
+            HealTraversalCoverage::Complete
+        } else {
+            HealTraversalCoverage::Partial
+        };
+    }
+
+    pub(crate) fn record(&mut self, mut item: HealObjectOutcome) {
+        use super::progress::increment_counter;
+        let counters = &mut self.counters;
+        counters.overflowed |= !increment_counter(&mut counters.processed);
+        let counter = match item.disposition {
+            HealObjectDisposition::Repaired => &mut counters.healed,
+            HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged,
+            HealObjectDisposition::Failed(_) => &mut counters.failed,
+            HealObjectDisposition::Unknown => {
+                counters.overflowed |= !increment_counter(&mut counters.unknown);
+                &mut counters.skipped
+            }
+            _ => &mut counters.skipped,
+        };
+        counters.overflowed |= !increment_counter(counter);
+        if let Some(detail) = &mut item.detail {
+            let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES);
+            while !detail.is_char_boundary(end) {
+                end -= 1;
+            }
+            self.objects_truncated |= end < detail.len();
+            detail.truncate(end);
+            detail.shrink_to_fit();
+        }
+        let bytes = item.retained_bytes();
+        if bytes > MAX_OUTCOME_BYTES {
+            self.objects_truncated = true;
+            return;
+        }
+        while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES {
+            let Some(oldest) = self.objects.pop_front() else { break };
+            self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes());
+            self.objects_truncated = true;
+        }
+        self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes);
+        self.objects.push_back(item);
+    }
+
+    pub(crate) fn retained_bytes(&self) -> usize {
+        size_of::<Self>()
+            .saturating_add(self.retained_object_bytes)
+            .saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
+    }
+}
+
+#[cfg(test)]
+mod canonical_outcome_tests {
+    use super::*;
+
+    fn item(disposition: HealObjectDisposition) -> HealObjectOutcome {
+        HealObjectOutcome {
+            identity: HealObjectIdentity {
+                kind: HealObjectKind::Object,
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: None,
+                bucket_incarnation_id: None,
+                pool_index: None,
+                set_index: None,
+            },
+            disposition,
+            detail: None,
+        }
+    }
+
+    #[test]
+    fn canonical_outcome_categories_have_one_terminal_count() {
+        let mut outcome = HealTaskOutcome::default();
+        for disposition in [
+            HealObjectDisposition::Unknown,
+            HealObjectDisposition::Repaired,
+            HealObjectDisposition::VerifiedHealthy,
+            HealObjectDisposition::AuthoritativelyAbsent,
+            HealObjectDisposition::Deferred {
+                reason: HealDeferredReason::DanglingDeleteGrace,
+                retry_not_before: None,
+            },
+            HealObjectDisposition::Failed(HealFailureClass::Permanent),
+            HealObjectDisposition::Cancelled,
+            HealObjectDisposition::DryRunObserved,
+        ] {
+            outcome.record(item(disposition));
+        }
+        let c = &outcome.counters;
+        assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1));
+        assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed);
+    }
+
+    #[test]
+    fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() {
+        let mut outcome = HealTaskOutcome::default();
+        for _ in 0..MAX_OUTCOME_ITEMS {
+            outcome.record(item(HealObjectDisposition::Unknown));
+        }
+        assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
+        assert!(!outcome.objects_truncated);
+        outcome.record(item(HealObjectDisposition::Unknown));
+        assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS);
+        assert!(outcome.objects_truncated);
+        let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
+        oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES);
+        outcome.record(oversized);
+        assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count"));
+        assert_eq!(outcome.counters.failed, 1);
+        assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
+        for _ in 0..MAX_OUTCOME_ITEMS {
+            let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent));
+            failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES));
+            outcome.record(failed);
+        }
+        assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES);
+        assert!(outcome.objects.iter().all(|item| {
+            item.detail
+                .as_ref()
+                .is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES)
+        }));
+        assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
+    }
+
+    #[test]
+    fn outcome_v3_serialization_keeps_unverified_dispositions_and_window_bounds() {
+        let mut outcome = HealTaskOutcome::default();
+        for disposition in [
+            HealObjectDisposition::Unknown,
+            HealObjectDisposition::Deferred {
+                reason: HealDeferredReason::DanglingDeleteGrace,
+                retry_not_before: None,
+            },
+            HealObjectDisposition::DryRunObserved,
+        ] {
+            outcome.record(item(disposition));
+        }
+        outcome.finish(None);
+        let wire = serde_json::to_value(&outcome).expect("canonical wire view");
+        assert_eq!(wire["execution"]["state"], "completed");
+        assert_eq!(wire["counters"]["healed"], 0);
+        assert_eq!(wire["counters"]["skipped"], 3);
+        assert_eq!(wire["objects"][1]["disposition"]["details"]["reason"], "dangling_delete_grace");
+        assert!(wire["objects"][1]["identity"]["bucketIncarnationId"].is_null());
+        for _ in 0..MAX_OUTCOME_ITEMS + 1 {
+            let mut result = item(HealObjectDisposition::Unknown);
+            result.detail = Some("\"".repeat(MAX_OUTCOME_DETAIL_BYTES));
+            outcome.record(result);
+        }
+        let bytes = serde_json::to_vec(&outcome).expect("bounded canonical samples");
+        assert!(
+            bytes.len() < 8 * MAX_OUTCOME_BYTES,
+            "JSON escaping remains bounded independently of object count"
+        );
+        assert!(outcome.objects_truncated);
+    }
+
+    #[test]
+    fn outcome_v3_wire_consistency_rejects_unknown_success_without_rejecting_extensions() {
+        let mut outcome = HealTaskOutcome::default();
+        outcome.finish(None);
+        let mut wire = serde_json::to_value(&outcome).expect("canonical snapshot");
+        wire["execution"]["futureField"] = serde_json::json!({"new": true});
+        wire["counters"]["futureCounter"] = serde_json::json!(42);
+        assert_eq!(
+            legacy_wire_status("finished", &wire, false).expect("unknown extension fields"),
+            ("finished", None)
+        );
+        wire["execution"]["state"] = serde_json::json!("future_execution");
+        assert!(legacy_wire_status("finished", &wire, false).is_err());
+        assert_eq!(
+            legacy_wire_status("running", &wire, false).expect("unknown nonterminal outcome"),
+            ("running", None)
+        );
+        wire["execution"] = serde_json::json!({"state":"completed"});
+        wire["counters"]["failed"] = serde_json::json!(1);
+        assert!(matches!(
+            legacy_wire_status("finished", &wire, false),
+            Err(HealOutcomeWireError::ContradictoryCompletion)
+        ));
+        wire.as_object_mut().expect("object").remove("execution");
+        assert!(matches!(
+            legacy_wire_status("finished", &wire, false),
+            Err(HealOutcomeWireError::MissingFields)
+        ));
+    }
+
+    #[test]
+    fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
+        let mut outcome = HealTaskOutcome::default();
+        outcome.counters.processed = u64::MAX;
+        outcome.record(item(HealObjectDisposition::Unknown));
+        outcome.finish(None);
+        assert!(outcome.counters.overflowed);
+        assert_eq!(outcome.counters.processed, u64::MAX);
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+    }
+}

crates/heal/src/heal/pacing.rs

@@ -0,0 +1,238 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use crate::{Error, Result};
+use rustfs_concurrency::{
+    WorkloadAdmissionSnapshotProvider,
+    workload::{ForegroundPressure, foreground_pressure},
+};
+use std::{sync::Arc, time::Duration};
+use tokio::{sync::Mutex, time::Instant};
+use tokio_util::sync::CancellationToken;
+
+#[derive(Default)]
+struct PacingState {
+    throttled: bool,
+    low_since: Option<Instant>,
+}
+
+pub(crate) enum PacingDecision {
+    Ready,
+    Wait(Option<ForegroundPressure>),
+}
+
+/// Cooperative pacing for one admin execution, not a storage admission permit.
+pub(crate) struct MainlinePacer {
+    provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
+    read_high: usize,
+    write_high: usize,
+    pause: Duration,
+    state: Mutex<PacingState>,
+}
+
+impl MainlinePacer {
+    pub(crate) fn new(
+        provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
+        read_high: usize,
+        write_high: usize,
+        pause: Duration,
+    ) -> Option<Self> {
+        if (read_high == 0 && write_high == 0) || pause.is_zero() {
+            return None;
+        }
+        Some(Self {
+            provider,
+            read_high: read_high.min(100),
+            write_high: write_high.min(100),
+            pause: pause.min(Duration::from_secs(1)),
+            state: Mutex::new(PacingState::default()),
+        })
+    }
+
+    /// Fresh, nonblocking decision while the caller owns actual page capacity.
+    /// A contended pacing latch is conservative, but never awaited here.
+    pub(crate) fn admission_decision(&self) -> PacingDecision {
+        let snapshot = self.provider.workload_admission_snapshot();
+        let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
+        if pressure.is_none() && self.state.try_lock().is_ok_and(|state| !state.throttled) {
+            PacingDecision::Ready
+        } else {
+            PacingDecision::Wait(pressure)
+        }
+    }
+
+    /// Call only between storage operations, with no namespace lock or I/O
+    /// permit held. The pacing-only mutex serializes starts within this task;
+    /// each holder waits at most one pause so persistent pressure cannot stop
+    /// all maintenance progress. Cancellation also interrupts queued waiters.
+    pub(crate) async fn wait(&self, cancel: &CancellationToken) -> Result<()> {
+        self.wait_after_admission(cancel, None).await.map(|_| ())
+    }
+
+    /// Returns whether this unit paid a bounded pause. That grant permits one
+    /// unit even if pressure persists when page capacity becomes available.
+    pub(crate) async fn wait_after_admission(
+        &self,
+        cancel: &CancellationToken,
+        observed: Option<ForegroundPressure>,
+    ) -> Result<bool> {
+        let mut state = tokio::select! {
+            biased;
+            _ = cancel.cancelled() => return Err(Error::TaskCancelled),
+            state = self.state.lock() => state,
+        };
+        if observed.is_some() {
+            state.throttled = true;
+            state.low_since = None;
+        }
+        let snapshot = self.provider.workload_admission_snapshot();
+        let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
+        if pressure.is_some() {
+            state.throttled = true;
+            state.low_since = None;
+        } else if state.throttled {
+            let low = |high: usize| if high == 0 { 0 } else { (high * 3 / 4).max(1) };
+            if foreground_pressure(&snapshot, low(self.read_high), low(self.write_high)).is_none() {
+                let now = Instant::now();
+                let since = state.low_since.get_or_insert(now);
+                if now.duration_since(*since) >= self.pause.saturating_mul(4) {
+                    state.throttled = false;
+                    state.low_since = None;
+                }
+            } else {
+                state.low_since = None;
+            }
+        }
+        if !state.throttled {
+            return Ok(false);
+        }
+        metrics::counter!(
+            "rustfs_heal_mainline_throttle_total",
+            "source" => "admin",
+            "result" => "delayed",
+            "reason" => pressure.or(observed).map_or("recovery_window", |pressure| pressure.reason())
+        )
+        .increment(1);
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            _ = tokio::time::sleep(self.pause) => Ok(true),
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) struct TestPressure {
+    pub(crate) active: std::sync::atomic::AtomicUsize,
+    pub(crate) sampled: tokio::sync::Notify,
+    class: rustfs_concurrency::WorkloadClass,
+}
+
+#[cfg(test)]
+impl TestPressure {
+    pub(crate) fn new(class: rustfs_concurrency::WorkloadClass, active: usize) -> Self {
+        Self {
+            active: std::sync::atomic::AtomicUsize::new(active),
+            sampled: tokio::sync::Notify::new(),
+            class,
+        }
+    }
+}
+
+#[cfg(test)]
+impl WorkloadAdmissionSnapshotProvider for TestPressure {
+    fn workload_admission_snapshot(&self) -> rustfs_concurrency::WorkloadAdmissionRegistrySnapshot {
+        let active = self.active.load(std::sync::atomic::Ordering::SeqCst);
+        self.sampled.notify_one();
+        rustfs_concurrency::WorkloadAdmissionRegistrySnapshot::new(vec![
+            rustfs_concurrency::WorkloadAdmissionSnapshot::new(self.class, rustfs_concurrency::AdmissionState::Open).with_counts(
+                Some(active),
+                None,
+                Some(100),
+            ),
+        ])
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_concurrency::WorkloadClass;
+    use std::sync::atomic::Ordering;
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
+        for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
+            let provider = Arc::new(TestPressure::new(class, 0));
+            let pause = Duration::from_millis(250);
+            let pacer = MainlinePacer::new(
+                provider.clone(),
+                if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
+                if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
+                pause,
+            )
+            .expect("enabled pacer");
+            let cancel = CancellationToken::new();
+            let now = Instant::now();
+            pacer.wait(&cancel).await.expect("quiet work");
+            assert_eq!(Instant::now(), now);
+            // Low watermark is 30 for the configured high watermark 40.
+            for utilization in [40, 29, 35, 29, 29, 29, 29] {
+                provider.active.store(utilization, Ordering::SeqCst);
+                let before = Instant::now();
+                pacer.wait(&cancel).await.expect("bounded maintenance progress");
+                assert_eq!(Instant::now() - before, pause);
+            }
+            let before = Instant::now();
+            pacer.wait(&cancel).await.expect("stable low pressure restores unpaced work");
+            assert_eq!(Instant::now(), before);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
+        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1)).is_none());
+        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO).is_none());
+        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600)).expect("pacer");
+        let before = Instant::now();
+        pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
+        assert_eq!(Instant::now() - before, Duration::from_secs(1));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
+        let paced = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("pacer"));
+        let cancel_first = CancellationToken::new();
+        let first = tokio::spawn({
+            let paced = paced.clone();
+            let cancel = cancel_first.clone();
+            async move { paced.wait(&cancel).await }
+        });
+        provider.sampled.notified().await;
+        let cancel_second = CancellationToken::new();
+        let second = tokio::spawn({
+            let paced = paced.clone();
+            let cancel = cancel_second.clone();
+            async move { paced.wait(&cancel).await }
+        });
+        tokio::task::yield_now().await;
+        cancel_second.cancel();
+        assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
+        provider.active.store(0, Ordering::SeqCst);
+        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("independent task");
+        let before = Instant::now();
+        other_task
+            .wait(&CancellationToken::new())
+            .await
+            .expect("another task has no inherited latch");
+        assert_eq!(Instant::now(), before, "task/set pacing state must not be global");
+        cancel_first.cancel();
+        assert!(matches!(first.await.expect("sleeping waiter"), Err(Error::TaskCancelled)));
+        tokio::time::timeout(Duration::from_secs(2), paced.wait(&CancellationToken::new()))
+            .await
+            .expect("pacing lock released")
+            .expect("bounded work after cancellation");
+    }
+}

crates/heal/src/heal/task.rs

@@ -15,6 +15,10 @@
 use crate::heal::{
     DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _,
     erasure_healer::target_outcomes_complete,
+    outcome::{
+        HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind,
+        HealObjectOutcome, HealTaskOutcome,
+    },
     progress::HealProgress,
     resume::{
         CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match,
@@ -43,6 +47,26 @@ use uuid::Uuid;
 
 use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET};
 
+#[cfg(test)]
+pub(crate) struct OutcomeFinishTestHook {
+    pub(crate) task_id: String,
+    pub(crate) reached: tokio::sync::Notify,
+    pub(crate) release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
+    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
+
+#[cfg(test)]
+async fn pause_outcome_finish(task_id: &str) {
+    let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
+    if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
+        hook.reached.notify_one();
+        hook.release.notified().await;
+    }
+}
+
 const LOG_COMPONENT_HEAL: &str = "heal";
 const LOG_SUBSYSTEM_TASK: &str = "task";
 const LOG_SUBSYSTEM_OBJECT: &str = "object";
@@ -394,6 +418,7 @@ pub struct HealTask {
     pub status: Arc<RwLock<HealTaskStatus>>,
     /// Progress tracking
     pub progress: Arc<RwLock<HealProgress>>,
+    outcome: Arc<RwLock<HealTaskOutcome>>,
     /// Result items collected from storage heal calls, each stamped with a
     /// monotonically increasing sequence number for incremental consumption
     /// (the client passes the last seen seq back and receives only newer
@@ -421,6 +446,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
 }
 
 impl HealTask {
@@ -460,6 +486,7 @@ impl HealTask {
             result_items_truncated: Arc::new(AtomicBool::new(false)),
             batch_failure: Arc::new(RwLock::new(None)),
             batch_failure_recorded: Arc::new(AtomicBool::new(false)),
+            outcome: Arc::new(RwLock::new(HealTaskOutcome::default())),
             created_at: request.created_at,
             enqueued_at: request.enqueued_at,
             started_at: Arc::new(RwLock::new(None)),
@@ -467,6 +494,7 @@ impl HealTask {
             task_start_instant: Arc::new(RwLock::new(None)),
             cancel_token: tokio_util::sync::CancellationToken::new(),
             storage,
+            mainline_pacer: None,
         }
     }
 
@@ -503,10 +531,82 @@ impl HealTask {
         task
     }
 
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+        self.mainline_pacer = pacer;
+        self
+    }
+
+    async fn pace_mainline(&self) -> Result<()> {
+        if let Some(pacer) = &self.mainline_pacer {
+            self.await_with_control(pacer.wait(&self.cancel_token)).await?;
+        }
+        Ok(())
+    }
+
     pub fn metric_type_label(&self) -> &'static str {
         self.heal_type.kind_label()
     }
 
+    pub async fn get_outcome(&self) -> HealTaskOutcome {
+        self.outcome.read().await.clone()
+    }
+
+    fn outcome_identity(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: Option<&str>,
+        pool_index: Option<usize>,
+        set_index: Option<usize>,
+    ) -> HealObjectIdentity {
+        HealObjectIdentity {
+            kind: match self.heal_type {
+                HealType::Metadata { .. } => HealObjectKind::Metadata,
+                HealType::ECDecode { .. } => HealObjectKind::Decode,
+                _ => HealObjectKind::Object,
+            },
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            version_id: version_id.map(ToOwned::to_owned),
+            bucket_incarnation_id: None,
+            pool_index,
+            set_index,
+        }
+    }
+
+    fn single_object_identity(&self) -> Option<HealObjectIdentity> {
+        let (bucket, object, version) = match &self.heal_type {
+            HealType::Object {
+                bucket,
+                object,
+                version_id,
+            }
+            | HealType::ECDecode {
+                bucket,
+                object,
+                version_id,
+            } => (bucket, object, version_id.as_deref()),
+            HealType::Metadata { bucket, object } => (bucket, object, None),
+            _ => return None,
+        };
+        Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
+    }
+
+    async fn record_deferred_object(&self, reason: HealDeferredReason) {
+        if let Some(identity) = self.single_object_identity() {
+            let mut outcome = self.outcome.write().await;
+            outcome.attempt_failed();
+            outcome.record(HealObjectOutcome {
+                identity,
+                disposition: HealObjectDisposition::Deferred {
+                    reason,
+                    retry_not_before: None,
+                },
+                detail: None,
+            });
+        }
+    }
+
     pub(crate) fn has_batch_failure(&self) -> bool {
         self.batch_failure_recorded.load(Ordering::Acquire)
     }
@@ -634,6 +734,7 @@ impl HealTask {
     }
 
     async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
+        self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await;
         warn!(
             target: "rustfs::heal::task",
             event = EVENT_HEAL_OBJECT_RESULT,
@@ -733,6 +834,8 @@ impl HealTask {
             return false;
         }
 
+        self.record_deferred_object(HealDeferredReason::TransientUsageCache).await;
+
         warn!(
             target: "rustfs::heal::task",
             event = EVENT_HEAL_OBJECT_RESULT,
@@ -755,6 +858,8 @@ impl HealTask {
             return false;
         }
 
+        self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await;
+
         warn!(
             target: "rustfs::heal::task",
             event = EVENT_HEAL_OBJECT_RESULT,
@@ -801,6 +906,7 @@ impl HealTask {
     #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
     #[hotpath::measure]
     pub async fn execute(&self) -> Result<()> {
+        self.outcome.write().await.start();
         // update status and timestamps atomically to avoid race conditions
         let now = SystemTime::now();
         let start_instant = Instant::now();
@@ -841,24 +947,71 @@ impl HealTask {
         });
         self.emit_trace_task_state("started", Duration::ZERO, None);
 
-        let result = match &self.heal_type {
+        let result = async {
-            HealType::Cluster => self.heal_cluster().await,
+            if self.heal_type.is_per_object() {
-            HealType::Object {
+                self.pace_mainline().await?;
-                bucket,
+            }
-                object,
+            match &self.heal_type {
-                version_id,
+                HealType::Cluster => self.heal_cluster().await,
-            } => self.heal_object(bucket, object, version_id.as_deref()).await,
+                HealType::Object {
-            HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
+                    bucket,
-            HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
+                    object,
+                    version_id,
+                } => self.heal_object(bucket, object, version_id.as_deref()).await,
+                HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
+                HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
 
-            HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
+                HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
-            HealType::ECDecode {
+                HealType::ECDecode {
-                bucket,
+                    bucket,
-                object,
+                    object,
-                version_id,
+                    version_id,
-            } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
+                } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
-            HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
+                HealType::ErasureSet { buckets, set_disk_id } => {
-        };
+                    self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await
+                }
+            }
+        }
+        .await;
+
+        #[cfg(test)]
+        pause_outcome_finish(&self.id).await;
+        {
+            let mut outcome = self.outcome.write().await;
+            if outcome.counters.processed == 0
+                && let Some(identity) = self.single_object_identity()
+            {
+                let disposition = match &result {
+                    Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved,
+                    Ok(()) => HealObjectDisposition::Unknown,
+                    Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled,
+                    Err(Error::TaskTimeout) => HealObjectDisposition::Deferred {
+                        reason: HealDeferredReason::Deadline,
+                        retry_not_before: None,
+                    },
+                    Err(error) => {
+                        outcome.attempt_failed();
+                        HealObjectDisposition::Failed(if error.is_recoverable_heal() {
+                            HealFailureClass::Recoverable
+                        } else {
+                            HealFailureClass::Permanent
+                        })
+                    }
+                };
+                outcome.record(HealObjectOutcome {
+                    identity,
+                    disposition,
+                    detail: result.as_ref().err().map(ToString::to_string),
+                });
+            }
+            let abort = match &result {
+                Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled),
+                Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline),
+                Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable),
+                _ => None,
+            };
+            outcome.finish(abort);
+        }
 
         // update completed time and status
         {
@@ -944,6 +1097,7 @@ impl HealTask {
 
     pub async fn cancel(&self) -> Result<()> {
         self.cancel_token.cancel();
+        self.outcome.write().await.finish(Some(HealAbortReason::Cancelled));
         let mut status = self.status.write().await;
         *status = HealTaskStatus::Cancelled;
         debug!(

crates/heal/src/heal/task/heal_bucket.rs

@@ -14,8 +14,107 @@
 /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
 use super::*;
 use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
+use crate::heal::storage::HealListItem;
 use crate::heal::utils::format_set_disk_id;
 
+const MAX_DEFERRED_OBJECTS: usize = 256;
+const MAX_DEFERRED_BYTES: usize = 256 * 1024;
+const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
+const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
+
+struct DeferredObject {
+    name: String,
+    version_id: Option<String>,
+    attempt: u32,
+    page: u64,
+    first_failure: Option<tokio::time::Instant>,
+    due: tokio::time::Instant,
+}
+
+impl DeferredObject {
+    fn new(item: HealListItem, page: u64) -> Self {
+        Self {
+            name: item.name,
+            version_id: item.version_id,
+            attempt: 0,
+            page,
+            first_failure: None,
+            due: tokio::time::Instant::now(),
+        }
+    }
+
+    fn payload_bytes(&self) -> usize {
+        self.name
+            .capacity()
+            .saturating_add(self.version_id.as_ref().map_or(0, String::capacity))
+    }
+
+    fn expired(&self) -> bool {
+        self.first_failure.is_some_and(|first| first.elapsed() >= MAX_DEFERRED_AGE)
+    }
+
+    fn defer(&mut self, delay: Duration) {
+        let now = tokio::time::Instant::now();
+        let first = *self.first_failure.get_or_insert(now);
+        self.attempt += 1;
+        self.due = (now + delay).min(first + MAX_DEFERRED_AGE);
+    }
+}
+
+// Only failed identities are retained. The current listing page remains owned
+// by the caller; capacity pressure stops fetching, never discards that page.
+struct DeferredWindow {
+    objects: VecDeque<DeferredObject>,
+    bytes: usize,
+}
+
+impl Default for DeferredWindow {
+    fn default() -> Self {
+        Self {
+            objects: VecDeque::new(),
+            // Charge every possible slot up front, including spare capacity.
+            bytes: MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>(),
+        }
+    }
+}
+
+impl DeferredWindow {
+    fn push(&mut self, item: DeferredObject) -> std::result::Result<(), DeferredObject> {
+        let bytes = item.payload_bytes();
+        if self.objects.len() >= MAX_DEFERRED_OBJECTS || bytes > MAX_DEFERRED_BYTES.saturating_sub(self.bytes) {
+            return Err(item);
+        }
+        self.bytes += bytes;
+        self.objects.push_back(item);
+        Ok(())
+    }
+
+    fn pop_due(&mut self) -> Option<DeferredObject> {
+        let now = tokio::time::Instant::now();
+        let index = self.objects.iter().position(|item| item.due <= now)?;
+        let item = self.objects.remove(index)?;
+        self.bytes -= item.payload_bytes();
+        Some(item)
+    }
+
+    fn next_due(&self) -> Option<tokio::time::Instant> {
+        self.objects.iter().map(|item| item.due).min()
+    }
+
+    fn can_advance(&self, page: u64) -> bool {
+        self.objects.len() < MAX_DEFERRED_OBJECTS
+            && self.bytes < MAX_DEFERRED_BYTES
+            && self
+                .objects
+                .iter()
+                .all(|item| page.saturating_sub(item.page) < MAX_DEFERRED_FORWARD_PAGES)
+    }
+}
+
+#[cfg(test)]
+#[path = "tests/deferred_retry_window.rs"]
+mod deferred_retry_window;
+
 fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Option<Error> {
     if opts.dry_run || !opts.recreate {
         return None;
@@ -34,6 +133,7 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
 
 impl HealTask {
     pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
+        self.pace_mainline().await?;
         debug!(
             target: "rustfs::heal::task",
             event = EVENT_HEAL_BUCKET_STAGE,
@@ -214,6 +314,7 @@ impl HealTask {
                             continue;
                         }
                         failed = failed.saturating_add(1);
+                        self.outcome.write().await.mark_untraversable();
                         if err.is_recoverable_heal() {
                             retryable = retryable.saturating_add(1);
                         } else {
@@ -260,6 +361,7 @@ impl HealTask {
 
     #[hotpath::measure]
     async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
+        let previous_progress = self.get_progress().await;
         let mut scanned = 0u64;
         let mut healed = 0u64;
         let mut failed = 0u64;
@@ -302,49 +404,122 @@ impl HealTask {
 
         for (set_disk_id, heal_opts) in listing_scopes {
             let mut continuation_token: Option<String> = None;
-            loop {
+            let mut deferred = DeferredWindow::default();
-                self.check_control_flags().await?;
+            let mut inline_retry: Option<DeferredObject> = None;
-                let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() {
+            let mut page_number = 0_u64;
-                    self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
+            let mut aborted_progress_unknown = false;
-                        set_disk_id,
+            let mut pending = Vec::<HealListItem>::new().into_iter();
-                        bucket,
+            let mut listing_finished = false;
-                        prefix,
+            let mut listing_attempt = 0;
-                        continuation_token.as_deref(),
+            let mut listing_due = tokio::time::Instant::now();
-                        false,
+            let scope_result: Result<()> = async {
-                    ))
+                loop {
-                    .await?
+                    self.check_control_flags().await?;
-                } else {
+                    if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none()
-                    self.await_with_control(self.storage.list_objects_for_heal_page(
+                    {
-                        bucket,
+                        break;
-                        prefix,
-                        continuation_token.as_deref(),
-                        false,
-                    ))
-                    .await?
-                };
-
-                let mut pending = objects;
-                let mut retry_attempt = 0_u32;
-                while !pending.is_empty() {
-                    if retry_attempt > 0 {
-                        self.await_with_control(async {
-                            tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
-                            Ok(())
-                        })
-                        .await?;
                     }
-                    let mut retry = Vec::with_capacity(pending.len());
+                    self.pace_mainline().await?;
-                    for item in pending {
+                    // Listing and object retries share this safe boundary. A
-                        self.check_control_flags().await?;
+                    // failed listing never hides an already-due object retry.
-                        let mut telemetry_unknown = false;
+                    let item = deferred.pop_due().or_else(|| {
-                        let object = item.name.as_str();
+                        if inline_retry
+                            .as_ref()
+                            .is_some_and(|item| item.due <= tokio::time::Instant::now())
                         {
-                            let mut progress = self.progress.write().await;
+                            inline_retry.take()
-                            progress.set_current_object(Some(format!("{bucket}/{object}")));
+                        } else if inline_retry.is_none() {
+                            pending.next().map(|item| DeferredObject::new(item, page_number))
+                        } else {
+                            None
                         }
+                    });
+                    let Some(mut item) = item else {
+                        let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
+                        if can_list && listing_due <= tokio::time::Instant::now() {
+                            let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
+                                self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
+                                    set_disk_id,
+                                    bucket,
+                                    prefix,
+                                    continuation_token.as_deref(),
+                                    false,
+                                ))
+                                .await
+                            } else {
+                                self.await_with_control(self.storage.list_objects_for_heal_page(
+                                    bucket,
+                                    prefix,
+                                    continuation_token.as_deref(),
+                                    false,
+                                ))
+                                .await
+                            };
+                            match page {
+                                Ok((objects, next_token, is_truncated)) => {
+                                    page_number = page_number.saturating_add(1);
+                                    continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
+                                    listing_finished = continuation_token.is_none();
+                                    listing_attempt = 0;
+                                    listing_due = tokio::time::Instant::now();
+                                    pending = objects.into_iter();
+                                }
+                                Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error),
+                                Err(error) => {
+                                    self.outcome.write().await.attempt_failed();
+                                    if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                                        listing_attempt += 1;
+                                        listing_due =
+                                            tokio::time::Instant::now() + self.bucket_object_retry_delay(listing_attempt);
+                                        continue;
+                                    }
+                                    self.outcome.write().await.mark_untraversable();
+                                    return Err(Error::HealListingFailed {
+                                        bucket: bucket.to_string(),
+                                        source: Box::new(error),
+                                    });
+                                }
+                            }
+                            continue;
+                        } else {
+                            let due = deferred
+                                .next_due()
+                                .into_iter()
+                                .chain(inline_retry.as_ref().map(|item| item.due))
+                                .chain(can_list.then_some(listing_due))
+                                .min();
+                            if let Some(due) = due {
+                                self.await_with_control(async {
+                                    tokio::time::sleep_until(due).await;
+                                    Ok(())
+                                })
+                                .await?;
+                            }
+                        }
+                        continue;
+                    };
+                    let retry_attempt = item.attempt;
+                    let mut telemetry_unknown = false;
+                    let object = item.name.as_str();
+                    let identity =
+                        self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
+                    let mut disposition = if heal_opts.dry_run {
+                        HealObjectDisposition::DryRunObserved
+                    } else {
+                        HealObjectDisposition::Unknown
+                    };
+                    let mut detail = None;
+                    {
+                        let mut progress = self.progress.write().await;
+                        progress.set_current_object(Some(format!("{bucket}/{object}")));
+                    }
 
-                        let mut terminal_outcome = true;
+                    let mut terminal_outcome = true;
-                        let error = match self
+                    let age_exhausted = item.expired();
+                    let error = if age_exhausted {
+                        Some(Error::other("heal object retry age exhausted"))
+                    } else {
+                        match self
                             .await_with_control(
                                 self.storage
                                     .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
@@ -377,107 +552,188 @@ impl HealTask {
                                 None
                             }
                             Ok((_, Some(err))) | Err(err) => Some(err),
-                        };
+                        }
+                    };
 
-                        if let Some(err) = error {
+                    if let Some(err) = error {
-                            if Self::is_dangling_delete_grace_error(&err) {
+                        match err {
-                                telemetry_unknown |= !increment_counter(&mut skipped);
+                            Error::TaskCancelled | Error::TaskTimeout => {
-                                warn!(
+                                let disposition = if matches!(err, Error::TaskCancelled) {
-                                    target: "rustfs::heal::task",
+                                    HealObjectDisposition::Cancelled
-                                    event = EVENT_HEAL_BUCKET_RESULT,
-                                    component = LOG_COMPONENT_HEAL,
-                                    subsystem = LOG_SUBSYSTEM_TASK,
-                                    task_id = %self.id,
-                                    bucket,
-                                    object,
-                                    result = "dangling_delete_grace_skip",
-                                    error = %err,
-                                    "Heal bucket object dangling cleanup deferred by grace window"
-                                );
-                            } else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
-                                telemetry_unknown |= !increment_counter(&mut skipped);
-                                warn!(
-                                    target: "rustfs::heal::task",
-                                    event = EVENT_HEAL_BUCKET_RESULT,
-                                    component = LOG_COMPONENT_HEAL,
-                                    subsystem = LOG_SUBSYSTEM_TASK,
-                                    task_id = %self.id,
-                                    bucket,
-                                    object,
-                                    result = "transient_skip",
-                                    error = %err,
-                                    "Heal bucket object repair skipped due to transient metadata error"
-                                );
-                            } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
-                                terminal_outcome = false;
-                                debug!(
-                                    target: "rustfs::heal::task",
-                                    event = EVENT_HEAL_BUCKET_RESULT,
-                                    component = LOG_COMPONENT_HEAL,
-                                    subsystem = LOG_SUBSYSTEM_TASK,
-                                    task_id = %self.id,
-                                    bucket,
-                                    object,
-                                    retry_attempt = retry_attempt.saturating_add(1),
-                                    error = %err,
-                                    result = "object_retry_scheduled",
-                                    "Heal bucket object retry scheduled"
-                                );
-                                retry.push(item);
-                            } else {
-                                telemetry_unknown |= !increment_counter(&mut failed);
-                                if err.is_recoverable_heal() {
-                                    retryable_failed = retryable_failed.saturating_add(1);
                                 } else {
-                                    permanent_failed = permanent_failed.saturating_add(1);
+                                    HealObjectDisposition::Deferred {
-                                }
+                                        reason: HealDeferredReason::Deadline,
-                                first_failed_object.get_or_insert_with(|| object.to_string());
+                                        retry_not_before: None,
-                                first_error.get_or_insert_with(|| err.to_string());
+                                    }
-                                if take_failure_log_sample(&mut failure_samples_logged) {
+                                };
-                                    warn!(
+                                self.outcome.write().await.record(HealObjectOutcome {
-                                        target: "rustfs::heal::task",
+                                    identity,
-                                        event = EVENT_HEAL_BUCKET_RESULT,
+                                    disposition,
-                                        component = LOG_COMPONENT_HEAL,
+                                    detail: None,
-                                        subsystem = LOG_SUBSYSTEM_TASK,
+                                });
-                                        task_id = %self.id,
+                                aborted_progress_unknown |= !increment_counter(&mut scanned);
-                                        bucket,
+                                aborted_progress_unknown |= !increment_counter(&mut skipped);
-                                        object,
+                                return Err(err);
-                                        retry_attempt,
+                            }
-                                        error = %err,
+                            _ if !age_exhausted => self.outcome.write().await.attempt_failed(),
-                                        result = "object_failed",
+                            _ => {}
-                                        "Heal bucket object repair failed"
+                        }
-                                    );
+                        detail = Some(err.to_string());
-                                }
+                        if Self::is_dangling_delete_grace_error(&err) {
+                            disposition = HealObjectDisposition::Deferred {
+                                reason: HealDeferredReason::DanglingDeleteGrace,
+                                retry_not_before: None,
+                            };
+                            telemetry_unknown |= !increment_counter(&mut skipped);
+                            warn!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object,
+                                result = "dangling_delete_grace_skip",
+                                error = %err,
+                                "Heal bucket object dangling cleanup deferred by grace window"
+                            );
+                        } else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                            disposition = HealObjectDisposition::Deferred {
+                                reason: HealDeferredReason::TransientUsageCache,
+                                retry_not_before: None,
+                            };
+                            telemetry_unknown |= !increment_counter(&mut skipped);
+                            warn!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object,
+                                result = "transient_skip",
+                                error = %err,
+                                "Heal bucket object repair skipped due to transient metadata error"
+                            );
+                        } else if !age_exhausted && err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                            terminal_outcome = false;
+                            debug!(
+                                target: "rustfs::heal::task",
+                                event = EVENT_HEAL_BUCKET_RESULT,
+                                component = LOG_COMPONENT_HEAL,
+                                subsystem = LOG_SUBSYSTEM_TASK,
+                                task_id = %self.id,
+                                bucket,
+                                object,
+                                retry_attempt = retry_attempt.saturating_add(1),
+                                error = %err,
+                                result = "object_retry_scheduled",
+                                "Heal bucket object retry scheduled"
+                            );
+                            item.defer(self.bucket_object_retry_delay(retry_attempt + 1));
+                            if let Err(item) = deferred.push(item) {
+                                inline_retry = Some(item);
+                            }
+                        } else {
+                            disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_heal() {
+                                HealFailureClass::RetryExhausted
+                            } else {
+                                HealFailureClass::Permanent
+                            });
+                            telemetry_unknown |= !increment_counter(&mut failed);
+                            if age_exhausted || err.is_recoverable_heal() {
+                                retryable_failed = retryable_failed.saturating_add(1);
+                            } else {
+                                permanent_failed = permanent_failed.saturating_add(1);
+                            }
+                            first_failed_object.get_or_insert_with(|| object.to_string());
+                            first_error.get_or_insert_with(|| err.to_string());
+                            if take_failure_log_sample(&mut failure_samples_logged) {
+                                warn!(
+                                    target: "rustfs::heal::task",
+                                    event = EVENT_HEAL_BUCKET_RESULT,
+                                    component = LOG_COMPONENT_HEAL,
+                                    subsystem = LOG_SUBSYSTEM_TASK,
+                                    task_id = %self.id,
+                                    bucket,
+                                    object,
+                                    retry_attempt,
+                                    error = %err,
+                                    result = "object_failed",
+                                    "Heal bucket object repair failed"
+                                );
                             }
                         }
-
-                        if terminal_outcome {
-                            telemetry_unknown |= !increment_counter(&mut scanned);
-                        }
-
-                        if !terminal_outcome {
-                            continue;
-                        }
-
-                        let mut progress = self.progress.write().await;
-                        progress.update_object_progress(scanned, healed, failed, skipped, bytes);
-                        if telemetry_unknown {
-                            progress.mark_unknown();
-                        }
                     }
-                    pending = retry;
-                    retry_attempt = retry_attempt.saturating_add(1);
-                }
 
-                if !is_truncated {
+                    if terminal_outcome {
-                    break;
+                        telemetry_unknown |= !increment_counter(&mut scanned);
-                }
+                    }
 
-                continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
+                    if !terminal_outcome {
-                if continuation_token.is_none() {
+                        continue;
-                    // Truncated but no continuation token: end of listing.
+                    }
-                    break;
+
+                    self.outcome.write().await.record(HealObjectOutcome {
+                        identity,
+                        disposition,
+                        detail,
+                    });
+
+                    let mut progress = self.progress.write().await;
+                    progress.update_object_progress(
+                        previous_progress.objects_scanned.saturating_add(scanned),
+                        previous_progress.objects_healed.saturating_add(healed),
+                        previous_progress.objects_failed.saturating_add(failed),
+                        previous_progress.skipped_objects.saturating_add(skipped),
+                        previous_progress.bytes_processed.saturating_add(bytes),
+                    );
+                    if telemetry_unknown {
+                        progress.mark_unknown();
+                    }
                 }
+                Ok(())
+            }
+            .await;
+            if let Err(error) = scope_result {
+                let disposition = match error {
+                    Error::TaskCancelled => HealObjectDisposition::Cancelled,
+                    Error::TaskTimeout => HealObjectDisposition::Deferred {
+                        reason: HealDeferredReason::Deadline,
+                        retry_not_before: None,
+                    },
+                    _ => HealObjectDisposition::Unknown,
+                };
+                // Only attempted identities have terminal outcomes. Unstarted
+                // page tails remain unprocessed under the task's partial coverage.
+                // No detached sleepers survive abort.
+                for item in deferred.objects.into_iter().chain(inline_retry) {
+                    self.outcome.write().await.record(HealObjectOutcome {
+                        identity: self.outcome_identity(
+                            bucket,
+                            &item.name,
+                            item.version_id.as_deref(),
+                            heal_opts.pool,
+                            heal_opts.set,
+                        ),
+                        disposition: disposition.clone(),
+                        detail: None,
+                    });
+                    aborted_progress_unknown |= !increment_counter(&mut scanned);
+                    aborted_progress_unknown |= !increment_counter(&mut skipped);
+                }
+                let mut progress = self.progress.write().await;
+                progress.update_object_progress(
+                    previous_progress.objects_scanned.saturating_add(scanned),
+                    previous_progress.objects_healed.saturating_add(healed),
+                    previous_progress.objects_failed.saturating_add(failed),
+                    previous_progress.skipped_objects.saturating_add(skipped),
+                    previous_progress.bytes_processed.saturating_add(bytes),
+                );
+                if aborted_progress_unknown {
+                    progress.mark_unknown();
+                }
+                return Err(error);
             }
         }
 

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -422,7 +422,8 @@ impl HealTask {
             self.source,
         )
         .with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
-        .with_replacement_identity_fence(replacement_target_identities.clone());
+        .with_replacement_identity_fence(replacement_target_identities.clone())
+        .with_mainline_pacer(self.mainline_pacer.clone());
 
         {
             let mut progress = self.progress.write().await;

crates/heal/src/heal/task/heal_metadata.rs

@@ -261,8 +261,8 @@ impl HealTask {
             update_parity: true,
             no_lock: self.options.no_lock,
             read_repair: false,
-            pool: None,
+            pool: self.options.pool_index,
-            set: None,
+            set: self.options.set_index,
         };
 
         let heal_result = self

crates/heal/src/heal/task/tests.rs

@@ -14,6 +14,366 @@
 
 use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
 use super::*;
+
+mod deferred_retry;
+
+mod canonical_outcome {
+    use super::*;
+    use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
+
+    fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
+        HealTask::from_request(
+            HealRequest::new(
+                HealType::Bucket {
+                    bucket: "bucket-a".to_string(),
+                },
+                HealOptions {
+                    recursive: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            ),
+            storage,
+        )
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn cluster_retries_only_the_failed_listing_page() {
+        let storage = Arc::new(MockStorage {
+            recoverable_second_page_failures: Mutex::new(Some(1)),
+            ..Default::default()
+        });
+        let task = HealTask::from_request(
+            HealRequest::new(
+                HealType::Cluster,
+                HealOptions {
+                    recursive: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            ),
+            storage.clone(),
+        );
+        task.execute().await.expect("second-page retry succeeds");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+        assert_eq!(outcome.counters.processed, 2);
+        assert_eq!(outcome.counters.attempt_failures, 1);
+        assert_eq!(task.get_progress().await.objects_scanned, 2);
+        assert_eq!(
+            storage.heal_object_calls.lock().expect("object calls").as_slice(),
+            ["object-a", "object-b"]
+        );
+        assert_eq!(
+            storage.listing_tokens.lock().expect("listing tokens").as_slice(),
+            [None, Some("second".to_string()), Some("second".to_string())]
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn exhausted_listing_page_cannot_restart_the_bucket() {
+        let storage = Arc::new(MockStorage {
+            recoverable_second_page_failures: Mutex::new(Some(4)),
+            ..Default::default()
+        });
+        let task = HealTask::from_request(
+            HealRequest::new(
+                HealType::Cluster,
+                HealOptions {
+                    recursive: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            ),
+            storage.clone(),
+        );
+        task.execute().await.expect_err("listing page budget exhausted");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(outcome.counters.processed, 1);
+        assert_eq!(outcome.counters.attempt_failures, 4);
+        assert_eq!(task.get_progress().await.objects_scanned, 1);
+        assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]);
+        assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
+    }
+
+    #[tokio::test]
+    async fn listing_failure_preserves_processed_objects_and_partial_coverage() {
+        let storage = Arc::new(MockStorage {
+            fail_second_listing_page: true,
+            ..Default::default()
+        });
+        let task = bucket_task(storage);
+        task.execute().await.expect_err("second page cannot be traversed");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(outcome.counters.processed, 1);
+        assert_eq!(outcome.objects[0].identity.object, "object-a");
+        assert_eq!(task.get_progress().await.objects_scanned, 1);
+    }
+
+    #[tokio::test]
+    async fn cluster_preserves_cumulative_progress_across_buckets() {
+        let storage = Arc::new(MockStorage {
+            list_each_bucket: true,
+            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+            ..Default::default()
+        });
+        let task = HealTask::from_request(
+            HealRequest::new(
+                HealType::Cluster,
+                HealOptions {
+                    recursive: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            ),
+            storage,
+        );
+        task.execute().await.expect("both buckets complete");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.counters.processed, 4);
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+        let progress = task.get_progress().await;
+        assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4));
+        assert_eq!(
+            outcome
+                .objects
+                .iter()
+                .filter(|item| item.identity.bucket == "bucket-b")
+                .count(),
+            2
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
+        let storage = Arc::new(MockStorage::default());
+        storage.heal_object_outcomes.lock().expect("outcomes").insert(
+            "object-a".to_string(),
+            (0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(),
+        );
+        let task = bucket_task(storage.clone());
+        task.execute().await.expect_err("legacy adapter retains batch failure");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+        assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1));
+        assert_eq!(outcome.counters.attempt_failures, 4);
+        let failed = outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "object-a")
+            .expect("failed object");
+        assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted));
+        let object_b_calls = {
+            let calls = storage.heal_object_calls.lock().expect("calls");
+            calls.iter().filter(|object| object.as_str() == "object-b").count()
+        };
+        assert_eq!(object_b_calls, 1);
+        let progress = task.get_progress().await;
+        assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn retry_success_counts_one_terminal_outcome() {
+        let storage = Arc::new(MockStorage::default());
+        storage
+            .heal_object_outcomes
+            .lock()
+            .expect("outcomes")
+            .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum]));
+        let task = bucket_task(storage);
+        task.execute().await.expect("retry should recover");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
+        assert_eq!(outcome.counters.processed, 2);
+        assert_eq!(outcome.counters.failed, 0);
+        assert_eq!(outcome.counters.attempt_failures, 1);
+        assert_eq!(
+            outcome
+                .objects
+                .iter()
+                .filter(|item| item.identity.object == "object-a")
+                .count(),
+            1
+        );
+        assert_eq!(
+            outcome.counters.processed,
+            outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed
+        );
+    }
+
+    #[tokio::test]
+    async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() {
+        let storage = Arc::new(MockStorage::default());
+        storage
+            .heal_object_outcomes
+            .lock()
+            .expect("outcomes")
+            .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]));
+        let task = bucket_task(storage);
+        task.execute().await.expect("grace permits traversal completion");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+        assert_eq!(outcome.counters.processed, 2);
+        assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt");
+        assert!(matches!(
+            outcome.objects[0].disposition,
+            HealObjectDisposition::Deferred {
+                reason: HealDeferredReason::DanglingDeleteGrace,
+                ..
+            }
+        ));
+        assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown);
+        assert!(
+            outcome
+                .objects
+                .iter()
+                .all(|item| item.identity.bucket_incarnation_id.is_none())
+        );
+        assert_eq!(
+            task.get_progress().await.objects_healed,
+            1,
+            "legacy display count remains distinct from proof"
+        );
+    }
+
+    #[tokio::test]
+    async fn grace_single_object_is_completed_but_deferred() {
+        let storage = Arc::new(MockStorage {
+            heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)),
+            ..Default::default()
+        });
+        let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage);
+        task.execute().await.expect("grace is deferred");
+        let outcome = task.get_outcome().await;
+        assert_eq!(task.get_status().await, HealTaskStatus::Completed);
+        assert_eq!(outcome.counters.processed, 1);
+        assert!(matches!(
+            outcome.objects[0].disposition,
+            HealObjectDisposition::Deferred {
+                reason: HealDeferredReason::DanglingDeleteGrace,
+                ..
+            }
+        ));
+        assert_eq!(outcome.counters.attempt_failures, 1);
+    }
+
+    #[tokio::test]
+    async fn dry_run_and_transient_existence_do_not_prove_repair() {
+        for transient in [false, true] {
+            let storage = Arc::new(MockStorage::default());
+            if transient {
+                storage
+                    .object_exists_by_name
+                    .lock()
+                    .expect("existence fixture")
+                    .insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
+            }
+            let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
+            request.options.dry_run = !transient;
+            let task = HealTask::from_request(request, storage);
+            task.execute().await.expect("observation may complete");
+            let outcome = task.get_outcome().await;
+            assert_eq!(outcome.counters.healed, 0);
+            if transient {
+                assert!(matches!(
+                    outcome.objects[0].disposition,
+                    HealObjectDisposition::Deferred {
+                        reason: HealDeferredReason::TransientExistenceCheck,
+                        ..
+                    }
+                ));
+            } else {
+                assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
+        let storage = Arc::new(MockStorage {
+            listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
+            ..Default::default()
+        });
+        let task = HealTask::from_request(
+            HealRequest::new(
+                HealType::Cluster,
+                HealOptions {
+                    recursive: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            ),
+            storage.clone(),
+        );
+        task.execute().await.expect_err("structural bucket error");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(
+            storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
+            ["bucket-a", "bucket-b"]
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn cancellation_and_deadline_leave_partial_coverage() {
+        for cancel in [false, true] {
+            let storage = Arc::new(MockStorage {
+                block_heal_object: Mutex::new(true),
+                ..Default::default()
+            });
+            let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
+            request.options.timeout = Some(Duration::from_secs(1));
+            let task = HealTask::from_request(request, storage);
+            if cancel {
+                task.cancel().await.expect("cancel request");
+            }
+            task.execute().await.expect_err("control interruption");
+            let outcome = task.get_outcome().await;
+            assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+            assert_eq!(
+                outcome.execution,
+                HealExecutionOutcome::Aborted(if cancel {
+                    HealAbortReason::Cancelled
+                } else {
+                    HealAbortReason::Deadline
+                })
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn decode_keeps_the_requested_pool_and_set() {
+        let storage = Arc::new(MockStorage::default());
+        let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string()));
+        request.options.pool_index = Some(2);
+        request.options.set_index = Some(3);
+        let task = HealTask::from_request(request, storage.clone());
+        task.execute().await.expect("decode fixture");
+        let pool_and_set = {
+            let options = storage.object_heal_opts.lock().expect("storage options");
+            (options[0].pool, options[0].set)
+        };
+        assert_eq!(pool_and_set, (Some(2), Some(3)));
+        let outcome = task.get_outcome().await;
+        let identity = &outcome.objects[0].identity;
+        assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3)));
+        assert_eq!(identity.version_id.as_deref(), Some("version-a"));
+        assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
+    }
+}
 use crate::heal::storage::{HealListItem, HealObjectInfo};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events};
 use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
@@ -581,7 +941,15 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
 
 #[derive(Default)]
 struct MockStorage {
+    retry_test_pages: Option<Vec<Vec<HealListItem>>>,
+    retry_test_delays: HashMap<String, Duration>,
+    retry_test_listing_delays: Mutex<VecDeque<Duration>>,
+    retry_test_events: Mutex<Vec<String>>,
     listed: Mutex<bool>,
+    list_each_bucket: bool,
+    fail_second_listing_page: bool,
+    recoverable_second_page_failures: Mutex<Option<usize>>,
+    listing_tokens: Mutex<Vec<Option<String>>>,
     healed_objects: Mutex<Vec<String>>,
     heal_object_calls: Mutex<Vec<String>>,
     heal_object_version_ids: Mutex<Vec<Option<String>>>,
@@ -747,6 +1115,7 @@ fn replacement_identity(
 }
 
 enum MockHealObjectOutcome {
+    RetryableLock,
     OkWithOtherError(&'static str),
     ErrOther(&'static str),
     DanglingGraceDeferred,
@@ -851,6 +1220,10 @@ impl HealStorageAPI for MockStorage {
         opts: &HealOpts,
     ) -> Result<(HealResultItem, Option<Error>)> {
         self.heal_object_calls.lock().unwrap().push(object.to_string());
+        self.retry_test_events.lock().expect("events").push(format!("heal:{object}"));
+        if let Some(delay) = self.retry_test_delays.get(object) {
+            tokio::time::sleep(*delay).await;
+        }
         self.heal_object_version_ids
             .lock()
             .unwrap()
@@ -879,6 +1252,13 @@ impl HealStorageAPI for MockStorage {
                     bucket.to_string(),
                     object.to_string(),
                 ))),
+                MockHealObjectOutcome::RetryableLock => Ok((
+                    HealResultItem::default(),
+                    Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
+                        resource: object.to_string(),
+                        owner: "competing-writer".to_string(),
+                    }))),
+                )),
                 MockHealObjectOutcome::RetryableSlowDown => {
                     Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
                 }
@@ -904,6 +1284,13 @@ impl HealStorageAPI for MockStorage {
                     bucket.to_string(),
                     object.to_string(),
                 ))),
+                MockHealObjectOutcome::RetryableLock => Ok((
+                    HealResultItem::default(),
+                    Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::AlreadyLocked {
+                        resource: object.to_string(),
+                        owner: "competing-writer".to_string(),
+                    }))),
+                )),
                 MockHealObjectOutcome::RetryableSlowDown => {
                     Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
                 }
@@ -995,12 +1382,54 @@ impl HealStorageAPI for MockStorage {
         _include_lifecycle_object_info: bool,
     ) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
         self.listed_prefixes.lock().unwrap().push(prefix.to_string());
+        self.listing_tokens
+            .lock()
+            .expect("listing tokens")
+            .push(continuation_token.map(ToOwned::to_owned));
+        self.retry_test_events
+            .lock()
+            .expect("events")
+            .push(format!("list:{}", continuation_token.unwrap_or("first")));
+        let delay = self.retry_test_listing_delays.lock().expect("listing delays").pop_front();
+        if let Some(delay) = delay {
+            tokio::time::sleep(delay).await;
+        }
+        if let Some(pages) = &self.retry_test_pages {
+            let page = continuation_token.map_or(0, |token| token.parse::<usize>().expect("test page token"));
+            let next = (page + 1 < pages.len()).then(|| (page + 1).to_string());
+            return Ok((pages[page].clone(), next.clone(), next.is_some()));
+        }
+        if let Some(remaining) = self
+            .recoverable_second_page_failures
+            .lock()
+            .expect("listing failures")
+            .as_mut()
+        {
+            if continuation_token.is_none() {
+                return Ok((vec![heal_item("object-a")], Some("second".to_string()), true));
+            }
+            if *remaining > 0 {
+                *remaining -= 1;
+                return Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
+                    bucket.to_string(),
+                    "page".to_string(),
+                )));
+            }
+            return Ok((vec![heal_item("object-b")], None, false));
+        }
+        if self.fail_second_listing_page {
+            return if continuation_token.is_none() {
+                Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true))
+            } else {
+                Err(Error::other("listing unavailable"))
+            };
+        }
         if *self.truncate_without_token.lock().unwrap() {
             return Ok((vec![heal_item("object-a")], None, true));
         }
 
         let mut listed = self.listed.lock().unwrap();
-        if continuation_token.is_none() && !*listed {
+        if continuation_token.is_none() && (!*listed || self.list_each_bucket) {
             *listed = true;
             let objects = if bucket == RUSTFS_META_BUCKET {
                 vec![
@@ -1393,6 +1822,8 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
 
 #[tokio::test]
 async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
+    use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
+
     // A version listing can report the final page as truncated with no
     // continuation token. That is treated as end-of-listing (not an error),
     // so the returned page is healed and the pass terminates cleanly instead
@@ -1414,10 +1845,16 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
     );
     let task = HealTask::from_request(request, storage.clone());
 
-    task.heal_bucket("bucket-a")
+    task.execute()
         .await
         .expect("truncated-without-token must terminate cleanly, not loop or error");
 
+    assert_eq!(task.get_status().await, HealTaskStatus::Completed);
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
+    assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+    assert_eq!(outcome.counters.processed, 1);
+
     assert_eq!(
         storage.healed_objects.lock().unwrap().as_slice(),
         ["object-a".to_string()],

crates/heal/src/heal/task/tests/deferred_retry.rs

@@ -0,0 +1,487 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
+    HealTask::from_request(
+        HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        ),
+        storage,
+    )
+}
+
+fn pages_storage(pages: &[&[&str]]) -> MockStorage {
+    MockStorage {
+        retry_test_pages: Some(
+            pages
+                .iter()
+                .map(|page| page.iter().map(|name| heal_item(name)).collect())
+                .collect(),
+        ),
+        ..Default::default()
+    }
+}
+
+fn fail_once(storage: &MockStorage, name: &str) {
+    storage
+        .heal_object_outcomes
+        .lock()
+        .expect("outcomes")
+        .insert(name.to_string(), VecDeque::from([MockHealObjectOutcome::RetryableLock]));
+}
+
+#[tokio::test(start_paused = true)]
+async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
+    let storage = Arc::new(MockStorage {
+        recoverable_second_page_failures: Mutex::new(Some(1)),
+        retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])),
+        ..Default::default()
+    });
+    storage.heal_object_outcomes.lock().expect("outcomes").insert(
+        "object-a".to_string(),
+        VecDeque::from([
+            MockHealObjectOutcome::RetryableSlowDown,
+            MockHealObjectOutcome::RetryableSlowDown,
+        ]),
+    );
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(
+        tokio::time::timeout(Duration::from_millis(30_500), &mut execution)
+            .await
+            .is_err()
+    );
+    assert_eq!(
+        storage.retry_test_events.lock().expect("events").as_slice(),
+        ["list:first", "heal:object-a", "list:second", "heal:object-a"]
+    );
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 1);
+    assert_eq!(
+        outcome.objects[0].disposition,
+        HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
+    );
+    execution.await.expect_err("age exhausted object must remain a batch failure");
+    assert_eq!(
+        storage.retry_test_events.lock().expect("events").as_slice(),
+        [
+            "list:first",
+            "heal:object-a",
+            "list:second",
+            "heal:object-a",
+            "list:second",
+            "heal:object-b"
+        ]
+    );
+    let outcome = task.get_outcome().await;
+    assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3));
+}
+
+#[tokio::test(start_paused = true)]
+async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() {
+    let storage = Arc::new(MockStorage {
+        recoverable_second_page_failures: Mutex::new(Some(1)),
+        retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
+        ..Default::default()
+    });
+    fail_once(&storage, "object-a");
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(
+        tokio::time::timeout(Duration::from_millis(31_500), &mut execution)
+            .await
+            .is_err()
+    );
+    assert_eq!(
+        storage.retry_test_events.lock().expect("events").as_slice(),
+        ["list:first", "heal:object-a", "list:second"]
+    );
+    assert_eq!(task.get_outcome().await.counters.failed, 1);
+    execution.await.expect_err("age exhaustion remains a failure");
+    assert_eq!(
+        storage.retry_test_events.lock().expect("events").as_slice(),
+        ["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"]
+    );
+}
+
+#[tokio::test(start_paused = true)]
+async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unprocessed() {
+    for cancel in [true, false] {
+        let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
+        let mut page: Vec<HealListItem> = names.iter().map(|name| heal_item(name)).collect();
+        page[256].version_id = Some("inline-version".to_string());
+        let storage = Arc::new(MockStorage {
+            retry_test_pages: Some(vec![page, vec![heal_item("healthy")]]),
+            ..Default::default()
+        });
+        for name in &names {
+            fail_once(&storage, name);
+        }
+        let mut task = bucket_task(storage.clone());
+        if !cancel {
+            task.options.timeout = Some(Duration::from_secs(1));
+        }
+        let execution = task.execute();
+        tokio::pin!(execution);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(500), &mut execution)
+                .await
+                .is_err()
+        );
+        assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
+        assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
+        if cancel {
+            task.cancel().await.expect("cancel");
+        }
+        let result = execution.await;
+        assert!(matches!(
+            (&result, cancel),
+            (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
+        ));
+        let outcome = task.get_outcome().await;
+        assert_eq!(
+            (
+                outcome.counters.processed,
+                outcome.counters.skipped,
+                outcome.counters.failed,
+                outcome.counters.healed
+            ),
+            (257, 257, 0, 0)
+        );
+        assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
+        assert_eq!(
+            outcome.execution,
+            crate::heal::outcome::HealExecutionOutcome::Aborted(if cancel {
+                HealAbortReason::Cancelled
+            } else {
+                HealAbortReason::Deadline
+            })
+        );
+        let inline: Vec<_> = outcome
+            .objects
+            .iter()
+            .filter(|item| item.identity.object == "blocked-256")
+            .collect();
+        assert_eq!(inline.len(), 1);
+        assert_eq!(inline[0].identity.version_id.as_deref(), Some("inline-version"));
+        assert_eq!(
+            inline[0].disposition,
+            if cancel {
+                HealObjectDisposition::Cancelled
+            } else {
+                HealObjectDisposition::Deferred {
+                    reason: HealDeferredReason::Deadline,
+                    retry_not_before: None,
+                }
+            }
+        );
+        let progress = task.get_progress().await;
+        assert_eq!(
+            (
+                progress.objects_scanned,
+                progress.skipped_objects,
+                progress.objects_failed,
+                progress.objects_healed
+            ),
+            (257, 257, 0, 0)
+        );
+        assert!(
+            !outcome
+                .objects
+                .iter()
+                .any(|item| item.identity.object == "blocked-257" || item.identity.object == "healthy")
+        );
+        tokio::time::advance(Duration::from_secs(60)).await;
+        assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
+        assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
+    let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
+    for name in ["a", "b", "c", "d"] {
+        storage
+            .heal_object_outcomes
+            .lock()
+            .expect("outcomes")
+            .insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
+    }
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
+    assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
+    execution.await.expect_err("all four objects exhaust retries");
+    let outcome = task.get_outcome().await;
+    assert_eq!(
+        (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
+        (4, 4, 16)
+    );
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
+    for name in ["a", "b", "c", "d"] {
+        assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn listing_retry_keeps_cursor_and_does_not_replay_successful_objects() {
+    let storage = Arc::new(MockStorage {
+        recoverable_second_page_failures: Mutex::new(Some(1)),
+        ..Default::default()
+    });
+    fail_once(&storage, "object-a");
+    let task = bucket_task(storage.clone());
+    task.execute().await.expect("both retries complete");
+    assert_eq!(
+        storage.listing_tokens.lock().expect("tokens").as_slice(),
+        [None, Some("second".to_string()), Some("second".to_string())]
+    );
+    assert_eq!(
+        storage.heal_object_calls.lock().expect("calls").as_slice(),
+        ["object-a", "object-a", "object-b"]
+    );
+    let outcome = task.get_outcome().await;
+    assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 2));
+}
+
+#[tokio::test(start_paused = true)]
+async fn typed_lock_contention_allows_only_two_forward_pages() {
+    let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
+    fail_once(&storage, "a");
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c"]);
+    assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 3);
+    execution.await.expect("all objects complete");
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "c", "a", "d"]);
+    assert_eq!(task.get_outcome().await.counters.processed, 4);
+}
+
+#[tokio::test(start_paused = true)]
+async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() {
+    let mut storage = pages_storage(&[&["a"], &["b", "c"]]);
+    storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(3));
+    fail_once(&storage, "a");
+    let storage = Arc::new(storage);
+    bucket_task(storage.clone()).execute().await.expect("all objects complete");
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "c"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
+    let mut storage = pages_storage(&[&["a"], &["b"]]);
+    storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
+    fail_once(&storage, "a");
+    let storage = Arc::new(storage);
+    let task = bucket_task(storage.clone());
+    task.execute().await.expect_err("aged pending responsibility is not success");
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 2);
+    assert_eq!(outcome.counters.attempt_failures, 1);
+    assert_eq!(outcome.counters.failed, 1);
+    assert_eq!(
+        outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "a")
+            .expect("a outcome")
+            .disposition,
+        HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
+    );
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancellation_drains_owned_retries_once() {
+    let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
+    fail_once(&storage, "a");
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
+    task.cancel().await.expect("cancel");
+    assert!(matches!(execution.await, Err(Error::TaskCancelled)));
+    tokio::time::advance(Duration::from_secs(60)).await;
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 2);
+    assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == "a").count(), 1);
+    assert_eq!(
+        outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "a")
+            .expect("a")
+            .disposition,
+        HealObjectDisposition::Cancelled
+    );
+}
+
+#[tokio::test(start_paused = true)]
+async fn deadline_drains_owned_retries_without_false_completion() {
+    let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
+    fail_once(&storage, "a");
+    let mut task = bucket_task(storage.clone());
+    task.options.timeout = Some(Duration::from_secs(1));
+    assert!(matches!(task.execute().await, Err(Error::TaskTimeout)));
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 2);
+    assert_eq!(
+        outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "a")
+            .expect("a")
+            .disposition,
+        HealObjectDisposition::Deferred {
+            reason: HealDeferredReason::Deadline,
+            retry_not_before: None
+        }
+    );
+    tokio::time::advance(Duration::from_secs(60)).await;
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn full_window_backpressures_without_losing_the_current_page_tail() {
+    let names: Vec<String> = (0..258).map(|index| format!("blocked-{index}")).collect();
+    let mut storage = MockStorage {
+        retry_test_pages: Some(vec![names.iter().map(|name| heal_item(name)).collect(), vec![heal_item("healthy")]]),
+        ..Default::default()
+    };
+    for name in &names {
+        fail_once(&storage, name);
+    }
+    // The last item has a version, proving the current-page tail is not rebuilt
+    // from names alone when the window fills.
+    storage.retry_test_pages.as_mut().expect("pages")[0][257].version_id = Some("version-tail".to_string());
+    let storage = Arc::new(storage);
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 257);
+    assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
+    execution.await.expect("every owned item eventually completes");
+    assert_eq!(task.get_outcome().await.counters.processed, 259);
+    let calls = storage.heal_object_calls.lock().expect("calls");
+    for name in &names {
+        assert_eq!(calls.iter().filter(|called| *called == name).count(), 2);
+    }
+    let versions = storage.heal_object_version_ids.lock().expect("versions");
+    for (name, version) in calls.iter().zip(versions.iter()) {
+        if name == "blocked-257" {
+            assert_eq!(version.as_deref(), Some("version-tail"));
+        }
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn oversized_identity_stays_inline_without_losing_version() {
+    let name = "k".repeat(256 * 1024);
+    let mut item = heal_item(&name);
+    item.version_id = Some("v".repeat(1024));
+    let storage = Arc::new(MockStorage {
+        retry_test_pages: Some(vec![vec![item], vec![heal_item("healthy")]]),
+        ..Default::default()
+    });
+    fail_once(&storage, &name);
+    let task = bucket_task(storage.clone());
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
+    assert_eq!(storage.listing_tokens.lock().expect("tokens").len(), 1);
+    execution.await.expect("oversized identity retries inline");
+    assert_eq!(task.get_outcome().await.counters.processed, 2);
+    let versions = storage.heal_object_version_ids.lock().expect("versions");
+    assert_eq!(versions[0], versions[1]);
+    assert_eq!(versions[0].as_ref().expect("version").len(), 1024);
+}
+
+#[tokio::test(start_paused = true)]
+async fn terminal_listing_failure_keeps_deferred_identity_unknown() {
+    let storage = Arc::new(MockStorage {
+        fail_second_listing_page: true,
+        ..Default::default()
+    });
+    fail_once(&storage, "object-a");
+    let task = bucket_task(storage.clone());
+    task.execute().await.expect_err("listing cannot continue");
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 1);
+    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
+    assert_eq!(outcome.coverage, crate::heal::outcome::HealTraversalCoverage::Partial);
+    assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn healthy_second_page_advances_before_first_retry_is_due() {
+    let storage = Arc::new(MockStorage {
+        recoverable_second_page_failures: Mutex::new(Some(0)),
+        ..Default::default()
+    });
+    storage
+        .heal_object_outcomes
+        .lock()
+        .expect("outcomes")
+        .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableSlowDown]));
+    let task = HealTask::from_request(
+        HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        ),
+        storage.clone(),
+    );
+    let execution = task.execute();
+    tokio::pin!(execution);
+    assert!(
+        tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err(),
+        "the deferred first object must remain pending before its retry is due"
+    );
+    assert_eq!(
+        storage.heal_object_calls.lock().expect("calls").as_slice(),
+        ["object-a", "object-b"],
+        "a retryable page head must not hold the healthy second page behind its backoff"
+    );
+    execution.await.expect("retry eventually succeeds");
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.counters.processed, 2);
+    assert_eq!(outcome.counters.attempt_failures, 1);
+    assert_eq!(
+        storage.heal_object_calls.lock().expect("calls").as_slice(),
+        ["object-a", "object-b", "object-a"]
+    );
+}

crates/heal/src/heal/task/tests/deferred_retry_window.rs

@@ -0,0 +1,76 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+fn item(name: String, version_id: Option<String>) -> DeferredObject {
+    DeferredObject::new(
+        HealListItem {
+            name,
+            version_id,
+            mod_time_unix_nanos: None,
+            lifecycle_object_info: None,
+            is_delete_marker: false,
+        },
+        1,
+    )
+}
+
+#[tokio::test(start_paused = true)]
+async fn count_cap_and_next_item_preserve_ownership() {
+    let mut window = DeferredWindow::default();
+    for _ in 0..MAX_DEFERRED_OBJECTS {
+        assert!(window.push(item("key".to_string(), None)).is_ok());
+    }
+    let rejected = window
+        .push(item("next".to_string(), Some("version".to_string())))
+        .expect_err("count cap");
+    assert_eq!(rejected.name, "next");
+    assert_eq!(rejected.version_id.as_deref(), Some("version"));
+    assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS);
+    assert!(window.bytes <= MAX_DEFERRED_BYTES);
+    assert!(!window.can_advance(1));
+    assert!(window.pop_due().is_some());
+    assert!(window.push(rejected).is_ok());
+}
+
+#[tokio::test(start_paused = true)]
+async fn byte_cap_counts_key_version_and_reserved_slots() {
+    let mut window = DeferredWindow::default();
+    let available = MAX_DEFERRED_BYTES - window.bytes;
+    let key = "k".repeat(available / 2);
+    let version = "v".repeat(available - key.capacity());
+    assert_eq!(key.capacity() + version.capacity(), available);
+    assert!(window.push(item(key, Some(version))).is_ok());
+    assert_eq!(window.bytes, MAX_DEFERRED_BYTES);
+    assert!(!window.can_advance(1));
+    assert!(window.push(item("x".to_string(), None)).is_err());
+    assert!(window.pop_due().is_some());
+    assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>());
+    assert!(window.push(item("x".to_string(), None)).is_ok());
+}
+
+#[tokio::test(start_paused = true)]
+async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() {
+    let mut entry = item("a".to_string(), None);
+    entry.defer(Duration::from_secs(2));
+    let first = entry.first_failure.expect("first failure");
+    tokio::time::advance(Duration::from_secs(29)).await;
+    entry.defer(Duration::from_secs(8));
+    assert_eq!(entry.first_failure, Some(first));
+    assert_eq!(entry.due, first + MAX_DEFERRED_AGE);
+    assert!(!entry.expired());
+    tokio::time::advance(Duration::from_secs(1)).await;
+    assert!(entry.expired());
+}

crates/iam/src/manager.rs

@@ -429,6 +429,27 @@ where
         }
     }
 
+    /// The cached mapping record for one user or group, looked up in the same
+    /// cache partition `policy_db_set` writes it to (group / STS / regular+service
+    /// user). `None` when no mapping is stored.
+    pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
+        let cache = self.cache.snapshot();
+        if is_group {
+            cache.group_policies.get(name).cloned()
+        } else if user_type == UserType::Sts {
+            cache.sts_policies.get(name).cloned()
+        } else {
+            cache.user_policies.get(name).cloned()
+        }
+    }
+
+    /// The cached group record (members, status, own timestamp) without the
+    /// mapped-policy overlay `get_group_description` applies. `None` when the
+    /// group does not exist.
+    pub async fn get_group_info(&self, name: &str) -> Option<GroupInfo> {
+        self.cache.snapshot().groups.get(name).cloned()
+    }
+
     pub async fn get_policy(&self, name: &str) -> Result<Policy> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
@@ -534,6 +555,17 @@ where
     }
 
     pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
+        self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_policy`] stamping the document with `updated_at` instead
+    /// of the local clock.
+    ///
+    /// A site-replication receiver passes the edit's source time: the next
+    /// incoming revision is judged against the stored `UpdateDate`, so a
+    /// local stamp would reject a newer source edit that was merely delivered
+    /// later (backlog#2291). The returned stamp is the one persisted.
+    pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if name.is_empty() || policy.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -544,18 +576,17 @@ where
             .get(name)
             .map(|v| {
                 let mut p = v.clone();
-                p.update(policy.clone());
+                p.update_at(policy.clone(), updated_at);
                 p
             })
-            .unwrap_or_else(|| PolicyDoc::new(policy));
+            .unwrap_or_else(|| PolicyDoc::new_at(policy, updated_at));
 
         self.api.save_policy_doc(name, policy_doc.clone()).await?;
 
-        let now = OffsetDateTime::now_utc();
+        self.cache
+            .add_or_update_policy_doc(name, &policy_doc, OffsetDateTime::now_utc());
 
-        self.cache.add_or_update_policy_doc(name, &policy_doc, now);
+        Ok(updated_at)
-
-        Ok(now)
     }
 
     pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
@@ -789,6 +820,12 @@ where
 
     /// create a service account and update cache
     pub async fn add_service_account(&self, cred: Credentials) -> Result<OffsetDateTime> {
+        self.add_service_account_at(cred, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_service_account`] stamping the identity with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_service_account_at(&self, cred: Credentials, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if cred.access_key.is_empty() || cred.parent_user.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -800,7 +837,8 @@ where
         }
         drop(cache);
 
-        let u = UserIdentity::new(cred);
+        let mut u = UserIdentity::new(cred);
+        u.update_at = Some(updated_at);
 
         self.api
             .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
@@ -808,10 +846,22 @@ where
 
         self.update_user_with_claims(&u.credentials.access_key, u.clone())?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
+        self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::update_service_account`] stamping the identity with
+    /// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
+    /// (backlog#2291).
+    pub async fn update_service_account_at(
+        &self,
+        name: &str,
+        opts: UpdateServiceAccountOpts,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         let _mutation_guard = self.cache.service_account_mutation_lock().lock().await;
         let cache = self.cache.snapshot();
         let Some(ui) = cache.users.get(name).cloned() else {
@@ -858,13 +908,7 @@ where
         }
 
         if let Some(status) = opts.status {
-            match status.as_str() {
+            cr.status = account_status_flag(&status).to_owned();
-                val if val == AccountStatus::Enabled.as_ref() => cr.status = auth::ACCOUNT_ON.to_owned(),
-                val if val == AccountStatus::Disabled.as_ref() => cr.status = auth::ACCOUNT_OFF.to_owned(),
-                auth::ACCOUNT_ON => cr.status = auth::ACCOUNT_ON.to_owned(),
-                auth::ACCOUNT_OFF => cr.status = auth::ACCOUNT_OFF.to_owned(),
-                _ => cr.status = auth::ACCOUNT_OFF.to_owned(),
-            }
         }
 
         let mut m: HashMap<String, Value> = if token_without_expiration {
@@ -916,8 +960,8 @@ where
 
         cr.session_token = jwt_sign(&m, &cr.secret_key)?;
 
-        let u = UserIdentity::new(cr);
+        let mut u = UserIdentity::new(cr);
-        let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc);
+        u.update_at = Some(updated_at);
         self.api
             .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
             .await?;
@@ -1149,6 +1193,20 @@ where
         Ok((policies.into_iter().collect(), update_at))
     }
     pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
+        self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::policy_db_set`] stamping the mapping with `updated_at` instead
+    /// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn policy_db_set_at(
+        &self,
+        name: &str,
+        user_type: UserType,
+        is_group: bool,
+        policy: &str,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1168,10 +1226,11 @@ where
                 self.cache.delete_user_policy(name, OffsetDateTime::now_utc());
             }
 
-            return Ok(OffsetDateTime::now_utc());
+            return Ok(updated_at);
         }
 
-        let mp = MappedPolicy::new(policy);
+        let mut mp = MappedPolicy::new(policy);
+        mp.update_at = updated_at;
 
         let cache = self.cache.snapshot();
         let policy_docs_cache = Arc::clone(&cache.policy_docs);
@@ -1194,7 +1253,7 @@ where
             self.cache.add_or_update_user_policy(name, &mp, OffsetDateTime::now_utc());
         }
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn set_temp_user(&self, access_key: &str, cred: &Credentials, policy_name: Option<&str>) -> Result<OffsetDateTime> {
@@ -1391,6 +1450,17 @@ where
     }
 
     pub async fn add_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
+        self.add_user_at(access_key, args, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_user`] stamping the identity with `updated_at` instead of
+    /// the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_user_at(
+        &self,
+        access_key: &str,
+        args: &AddOrUpdateUserReq,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         let cache = self.cache.snapshot();
         let users = Arc::clone(&cache.users);
         if let Some(x) = users.get(access_key) {
@@ -1408,12 +1478,13 @@ where
                 _ => auth::ACCOUNT_OFF,
             }
         };
-        let user_entry = UserIdentity::from(Credentials {
+        let mut user_entry = UserIdentity::from(Credentials {
             access_key: access_key.to_string(),
             secret_key: args.secret_key.to_string(),
             status: status.to_owned(),
             ..Default::default()
         });
+        user_entry.update_at = Some(updated_at);
 
         self.api
             .save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
@@ -1421,7 +1492,7 @@ where
 
         self.update_user_with_claims(access_key, user_entry)?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
@@ -1599,6 +1670,17 @@ where
     }
 
     pub async fn set_user_status(&self, access_key: &str, status: AccountStatus) -> Result<OffsetDateTime> {
+        self.set_user_status_at(access_key, status, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_user_status`] stamping the identity with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn set_user_status_at(
+        &self,
+        access_key: &str,
+        status: AccountStatus,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if access_key.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1625,12 +1707,13 @@ where
             }
         };
 
-        let user_entry = UserIdentity::from(Credentials {
+        let mut user_entry = UserIdentity::from(Credentials {
             access_key: access_key.to_string(),
             secret_key: u.credentials.secret_key.clone(),
             status: status.to_owned(),
             ..Default::default()
         });
+        user_entry.update_at = Some(updated_at);
         drop(cache);
         drop(users);
 
@@ -1640,7 +1723,7 @@ where
 
         self.update_user_with_claims(access_key, user_entry)?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     fn update_user_with_claims(&self, k: &str, u: UserIdentity) -> Result<()> {
@@ -1676,6 +1759,17 @@ where
     }
 
     pub async fn add_users_to_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
+        self.add_users_to_group_at(group, members, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_users_to_group`] stamping the group with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_users_to_group_at(
+        &self,
+        group: &str,
+        members: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if group.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1693,6 +1787,14 @@ where
             }
         }
 
+        // The group's own timestamp moves with every membership or status
+        // change: site replication judges an incoming group item against it
+        // (backlog#2291), so it must reflect the last change, not creation.
+        // `updated_at` is the record's stamp only; the cache is published
+        // with the local clock, because `LockedCache::exec` drops a write
+        // whose time predates the entity's load time — a replicated edit
+        // whose source time is older than this node's startup would
+        // otherwise never reach the cache.
         let gi = match cache.groups.get(group) {
             Some(res) => {
                 let mut gi = res.clone();
@@ -1701,15 +1803,20 @@ where
                 uniq_set.extend(members.iter().cloned());
 
                 gi.members = uniq_set.into_iter().collect();
+                gi.update_at = Some(updated_at);
+                gi
+            }
+            None => {
+                let mut gi = GroupInfo::new(members.clone());
+                gi.update_at = Some(updated_at);
                 gi
             }
-            None => GroupInfo::new(members.clone()),
         };
         drop(cache);
 
         self.api.save_group_info(group, gi.clone()).await?;
 
-        let now = self.cache.with_write_lock(|cache| {
+        self.cache.with_write_lock(|cache| {
             let now = OffsetDateTime::now_utc();
             cache.add_or_update_group(group, &gi, now);
 
@@ -1719,13 +1826,18 @@ where
                 m.insert(group.to_string());
                 cache.add_or_update_user_group_membership(member, &m, now);
             });
-            now
         });
 
-        Ok(now)
+        Ok(updated_at)
     }
 
     pub async fn set_group_status(&self, name: &str, enable: bool) -> Result<OffsetDateTime> {
+        self.set_group_status_at(name, enable, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_group_status`] stamping the group with `updated_at` instead
+    /// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn set_group_status_at(&self, name: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1743,12 +1855,15 @@ where
         } else {
             gi.status = STATUS_DISABLED.to_owned();
         }
+        gi.update_at = Some(updated_at);
 
         self.api.save_group_info(name, gi.clone()).await?;
 
+        // Cache publication time is the local clock, not the record stamp
+        // (see `add_users_to_group_at`).
         self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn get_group_description(&self, name: &str) -> Result<GroupDesc> {
@@ -1818,6 +1933,20 @@ where
         name: &str,
         members: Vec<String>,
         update_cache_only: bool,
+    ) -> Result<OffsetDateTime> {
+        self.remove_members_from_group_at(name, members, update_cache_only, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::remove_members_from_group`] stamping the group with
+    /// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
+    /// (backlog#2291).
+    pub async fn remove_members_from_group_at(
+        &self,
+        name: &str,
+        members: Vec<String>,
+        update_cache_only: bool,
+        updated_at: OffsetDateTime,
     ) -> Result<OffsetDateTime> {
         let cache = self.cache.snapshot();
         let mut gi = cache
@@ -1830,12 +1959,14 @@ where
         let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
         let d: HashSet<&String> = HashSet::from_iter(members.iter());
         gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
-
+        gi.update_at = Some(updated_at);
         if !update_cache_only {
             self.api.save_group_info(name, gi.clone()).await?;
         }
 
-        let now = self.cache.with_write_lock(|cache| {
+        self.cache.with_write_lock(|cache| {
+            // Sample after storage completes so a concurrent reload cannot
+            // make this publication older than the cache it must update.
             let now = OffsetDateTime::now_utc();
             cache.add_or_update_group(name, &gi, now);
 
@@ -1847,13 +1978,25 @@ where
                     cache.add_or_update_user_group_membership(member, &m, now);
                 }
             });
-            now
         });
 
-        Ok(now)
+        Ok(updated_at)
     }
 
     pub async fn remove_users_from_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
+        self.remove_users_from_group_at(group, members, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::remove_users_from_group`] stamping the group with `updated_at`
+    /// instead of the local clock; a group delete (no members) leaves no
+    /// record and returns the stamp unchanged (backlog#2291).
+    pub async fn remove_users_from_group_at(
+        &self,
+        group: &str,
+        members: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if group.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1902,18 +2045,17 @@ where
                 return Err(err);
             }
 
-            let now = self.cache.with_write_lock(|cache| {
+            self.cache.with_write_lock(|cache| {
                 let now = OffsetDateTime::now_utc();
                 self.remove_group_from_memberships_map_unlocked(cache, group, now);
                 cache.delete_group(group, now);
                 cache.delete_group_policy(group, now);
-                now
             });
 
-            return Ok(now);
+            return Ok(updated_at);
         }
 
-        self.remove_members_from_group(group, members, false).await
+        self.remove_members_from_group_at(group, members, false, updated_at).await
     }
 
     fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
@@ -2235,6 +2377,19 @@ where
     }
 }
 
+/// The stored `status` flag for a service-account status given on the admin
+/// or replication wire: the madmin `enabled` / `disabled` words and the stored
+/// `on` / `off` flags are both accepted; anything else disables the account.
+pub(crate) fn account_status_flag(status: &str) -> &'static str {
+    match status {
+        val if val == AccountStatus::Enabled.as_ref() => auth::ACCOUNT_ON,
+        val if val == AccountStatus::Disabled.as_ref() => auth::ACCOUNT_OFF,
+        auth::ACCOUNT_ON => auth::ACCOUNT_ON,
+        auth::ACCOUNT_OFF => auth::ACCOUNT_OFF,
+        _ => auth::ACCOUNT_OFF,
+    }
+}
+
 pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
     let default_policies = &DEFAULT_POLICIES;
     default_policies

crates/iam/src/sys.rs

@@ -385,7 +385,14 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_policy(name, policy).await?;
+        self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_policy`] stamping the document with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock; see
+    /// `IamCache::set_policy_at` (backlog#2291).
+    pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_policy_at(name, policy, updated_at).await?;
 
         if !self.has_watcher() {
             for r in notify_iam_load_policy(name).await {
@@ -643,7 +650,18 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_user_status(name, status).await?;
+        self.set_user_status_at(name, status, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_user_status`] stamping the identity with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn set_user_status_at(
+        &self,
+        name: &str,
+        status: rustfs_madmin::AccountStatus,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_user_status_at(name, status, updated_at).await?;
 
         self.notify_for_user(name, false).await;
 
@@ -655,6 +673,20 @@ impl<T: Store> IamSys<T> {
         parent_user: &str,
         groups: Option<Vec<String>>,
         opts: NewServiceAccountOpts,
+    ) -> Result<(Credentials, OffsetDateTime)> {
+        self.new_service_account_at(parent_user, groups, opts, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::new_service_account`] stamping the identity with `updated_at`
+    /// (a replicated edit's source time) instead of the local clock
+    /// (backlog#2291).
+    pub async fn new_service_account_at(
+        &self,
+        parent_user: &str,
+        groups: Option<Vec<String>>,
+        opts: NewServiceAccountOpts,
+        updated_at: OffsetDateTime,
     ) -> Result<(Credentials, OffsetDateTime)> {
         if parent_user.is_empty() {
             return Err(IamError::InvalidArgument);
@@ -724,11 +756,18 @@ impl<T: Store> IamSys<T> {
         let mut cred = create_new_credentials_with_metadata(&access_key, &secret_key, &m, &secret_key)?;
         cred.parent_user = parent_user.to_owned();
         cred.groups = groups;
-        cred.status = ACCOUNT_ON.to_owned();
+        // The status is part of the created identity: a replicated disabled
+        // account must never exist enabled, not even between a create and a
+        // follow-up status write (backlog#2289).
+        cred.status = opts
+            .status
+            .as_deref()
+            .map_or(ACCOUNT_ON, crate::manager::account_status_flag)
+            .to_owned();
         cred.name = opts.name;
         cred.description = opts.description;
 
-        let create_at = self.store.add_service_account(cred.clone()).await?;
+        let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?;
 
         self.notify_for_service_account(&cred.access_key).await;
 
@@ -736,11 +775,23 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
+        self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::update_service_account`] stamping the identity with
+    /// `updated_at` (a replicated edit's source time) instead of the local
+    /// clock (backlog#2291).
+    pub async fn update_service_account_at(
+        &self,
+        name: &str,
+        opts: UpdateServiceAccountOpts,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if name == SITE_REPLICATOR_SERVICE_ACCOUNT && !opts.allow_site_replicator_account {
             return Err(IamError::IAMActionNotAllowed);
         }
 
-        let updated_at = self.store.update_service_account(name, opts).await?;
+        let updated_at = self.store.update_service_account_at(name, opts, updated_at).await?;
 
         self.notify_for_service_account(name).await;
 
@@ -940,6 +991,17 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn create_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
+        self.create_user_at(access_key, args, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::create_user`] stamping the identity with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn create_user_at(
+        &self,
+        access_key: &str,
+        args: &AddOrUpdateUserReq,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if !is_access_key_valid(access_key) {
             return Err(IamError::InvalidAccessKeyLength);
         }
@@ -952,7 +1014,7 @@ impl<T: Store> IamSys<T> {
             return Err(IamError::InvalidSecretKeyLength);
         }
 
-        let updated_at = self.store.add_user(access_key, args).await?;
+        let updated_at = self.store.add_user_at(access_key, args, updated_at).await?;
         self.load_user(access_key, UserType::Reg).await?;
 
         self.notify_for_user(access_key, false).await;
@@ -1026,10 +1088,21 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn add_users_to_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
+        self.add_users_to_group_at(group, users, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_users_to_group`] stamping the group with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn add_users_to_group_at(
+        &self,
+        group: &str,
+        users: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if contains_reserved_chars(group) {
             return Err(IamError::GroupNameContainsReservedChars);
         }
-        let updated_at = self.store.add_users_to_group(group, users).await?;
+        let updated_at = self.store.add_users_to_group_at(group, users, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1037,7 +1110,19 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn remove_users_from_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
-        let updated_at = self.store.remove_users_from_group(group, users).await?;
+        self.remove_users_from_group_at(group, users, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::remove_users_from_group`] stamping the group with `updated_at`
+    /// (a replicated edit's source time) instead of the local clock
+    /// (backlog#2291).
+    pub async fn remove_users_from_group_at(
+        &self,
+        group: &str,
+        users: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self.store.remove_users_from_group_at(group, users, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1045,7 +1130,13 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_group_status(&self, group: &str, enable: bool) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_group_status(group, enable).await?;
+        self.set_group_status_at(group, enable, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_group_status`] stamping the group with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn set_group_status_at(&self, group: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_group_status_at(group, enable, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1055,6 +1146,22 @@ impl<T: Store> IamSys<T> {
         self.store.get_group_description(group).await
     }
 
+    /// The stored group record itself (see `IamCache::get_group_info`).
+    pub async fn get_group_info(&self, group: &str) -> Option<GroupInfo> {
+        self.store.get_group_info(group).await
+    }
+
+    /// The stored policy document, `Error::NoSuchPolicy` when absent.
+    pub async fn get_policy_doc(&self, name: &str) -> Result<PolicyDoc> {
+        self.store.get_policy_doc(name).await
+    }
+
+    /// The stored mapping record for one user or group (see
+    /// `IamCache::get_mapped_policy_record`).
+    pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
+        self.store.get_mapped_policy_record(name, user_type, is_group).await
+    }
+
     pub async fn list_groups_load(&self) -> Result<Vec<String>> {
         self.store.update_groups().await
     }
@@ -1064,7 +1171,24 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
-        let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
+        self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::policy_db_set`] stamping the mapping with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn policy_db_set_at(
+        &self,
+        name: &str,
+        user_type: UserType,
+        is_group: bool,
+        policy: &str,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self
+            .store
+            .policy_db_set_at(name, user_type, is_group, policy, updated_at)
+            .await?;
 
         if !self.has_watcher() {
             for r in notify_iam_load_policy_mapping(name, user_type.to_u64(), is_group).await {
@@ -1846,6 +1970,11 @@ pub struct NewServiceAccountOpts {
     pub expiration: Option<OffsetDateTime>,
     pub allow_site_replicator_account: bool,
     pub claims: Option<HashMap<String, Value>>,
+    /// Status the account is created with (`enabled` / `disabled` or the
+    /// stored `on` / `off` flags); `None` creates it enabled. Site
+    /// replication passes the source account's status so a disabled account
+    /// is never enabled on the peer, not even transiently (backlog#2289).
+    pub status: Option<String>,
 }
 
 pub struct UpdateServiceAccountOpts {
@@ -2081,6 +2210,9 @@ mod tests {
         block_delete: Arc<std::sync::atomic::AtomicBool>,
         delete_started: Arc<tokio::sync::Notify>,
         release_delete: Arc<tokio::sync::Notify>,
+        block_group_save: Arc<std::sync::atomic::AtomicBool>,
+        group_save_started: Arc<tokio::sync::Notify>,
+        group_save_release: Arc<tokio::sync::Notify>,
     }
 
     impl StsTestMockStore {
@@ -2094,6 +2226,9 @@ mod tests {
                 block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 delete_started: Arc::new(tokio::sync::Notify::new()),
                 release_delete: Arc::new(tokio::sync::Notify::new()),
+                block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                group_save_started: Arc::new(tokio::sync::Notify::new()),
+                group_save_release: Arc::new(tokio::sync::Notify::new()),
             }
         }
 
@@ -2197,11 +2332,15 @@ mod tests {
         }
 
         async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
-            Err(Error::InvalidArgument)
+            if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
+                self.group_save_started.notify_one();
+                self.group_save_release.notified().await;
+            }
+            Ok(())
         }
 
         async fn delete_group_info(&self, _name: &str) -> Result<()> {
-            Err(Error::InvalidArgument)
+            Ok(())
         }
 
         async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()> {
@@ -2378,6 +2517,140 @@ mod tests {
         IamSys::new(cache)
     }
 
+    async fn assert_group_write_during_reload_is_published(remove: bool) {
+        let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
+        let member = "sts-fallback-test-parent";
+        let group = if remove { "testgroup" } else { "new-published-group" };
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        iam_sys
+            .store
+            .api
+            .block_group_save
+            .store(true, std::sync::atomic::Ordering::SeqCst);
+        let before = iam_sys.store.cache.snapshot();
+        let writer_iam = iam_sys.clone();
+        let writer = tokio::spawn(async move {
+            if remove {
+                writer_iam
+                    .remove_users_from_group_at(group, vec![member.to_string()], source_time)
+                    .await
+            } else {
+                writer_iam
+                    .add_users_to_group_at(group, vec![member.to_string()], source_time)
+                    .await
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), iam_sys.store.api.group_save_started.notified())
+            .await
+            .expect("group save should reach the barrier");
+        // The pending store write has not changed the cache, so the production
+        // full-reload snapshot guard permits this replacement.
+        assert!(iam_sys.store.cache.with_write_lock(|cache| cache.matches_snapshot(&before)));
+        iam_sys
+            .store
+            .api
+            .load_all(&iam_sys.store.cache)
+            .await
+            .expect("reload while group save is pending");
+        iam_sys.store.api.group_save_release.notify_one();
+        assert_eq!(writer.await.expect("join group writer").expect("group write should succeed"), source_time);
+        let info = iam_sys
+            .get_group_info(group)
+            .await
+            .expect("successful group write must remain readable after reload");
+        assert_eq!(info.update_at, Some(source_time), "source timestamp must remain on the record");
+        assert_eq!(info.members, if remove { Vec::new() } else { vec![member.to_string()] });
+        let groups = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert_eq!(
+            groups.is_some_and(|groups| groups.contains(group)),
+            !remove,
+            "membership index must reflect the write"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn add_group_write_during_reload_publishes_after_store_save() {
+        assert_group_write_during_reload_is_published(false).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn remove_group_write_during_reload_publishes_after_store_save() {
+        assert_group_write_during_reload_is_published(true).await;
+    }
+
+    /// Review finding on rustfs#7195: a replicated group edit carries a source
+    /// stamp that may predate this node's cache load time. The stamp belongs on
+    /// the record only; publishing the cache with it makes `LockedCache::exec`
+    /// drop the write, so the group is written to the store but unreadable
+    /// here and the receiver's next `set_group_status_at` fails with
+    /// `NoSuchGroup`. Add, status and removal must all publish with the local
+    /// clock while keeping the source stamp on `GroupInfo::update_at`.
+    #[tokio::test]
+    async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
+        let iam_sys = test_iam_sys().await;
+        let member = "group-stamp-member";
+        let identity = UserIdentity {
+            version: 1,
+            credentials: Credentials {
+                access_key: member.to_string(),
+                secret_key: "longenoughsecret".to_string(),
+                status: "on".to_string(),
+                ..Default::default()
+            },
+            update_at: Some(OffsetDateTime::now_utc()),
+        };
+        iam_sys.store.cache.with_write_lock(|cache| {
+            cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
+            // The startup load publishes every entity with the load time.
+            cache.replace_groups(CacheEntity::new(HashMap::new()));
+            cache.replace_user_group_memberships(CacheEntity::new(HashMap::new()));
+        });
+
+        let group = "group-stamp";
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        let stamped = iam_sys
+            .add_users_to_group_at(group, vec![member.to_string()], source_time)
+            .await
+            .expect("add members with a source stamp older than the cache load");
+        assert_eq!(stamped, source_time, "the returned stamp is the source time");
+        let info = iam_sys
+            .get_group_info(group)
+            .await
+            .expect("the group must be readable right after the add");
+        assert_eq!(info.members, vec![member.to_string()]);
+        assert_eq!(info.update_at, Some(source_time), "the record keeps the source stamp");
+        let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert!(
+            memberships.is_some_and(|groups| groups.contains(group)),
+            "the membership index is published too"
+        );
+
+        let disabled_at = source_time + time::Duration::seconds(1);
+        iam_sys
+            .set_group_status_at(group, false, disabled_at)
+            .await
+            .expect("status change with a source stamp older than the cache load");
+        let info = iam_sys.get_group_info(group).await.expect("group after status change");
+        assert_eq!(info.status, "disabled");
+        assert_eq!(info.update_at, Some(disabled_at));
+
+        let removed_at = source_time + time::Duration::seconds(2);
+        iam_sys
+            .remove_users_from_group_at(group, vec![member.to_string()], removed_at)
+            .await
+            .expect("removal with a source stamp older than the cache load");
+        let info = iam_sys.get_group_info(group).await.expect("group after removal");
+        assert!(info.members.is_empty(), "the removal must be visible in the cache");
+        assert_eq!(info.update_at, Some(removed_at));
+        let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert!(
+            !memberships.is_some_and(|groups| groups.contains(group)),
+            "the membership index follows the removal"
+        );
+    }
+
     fn service_account_opts(access_key: &str, secret_key: &str) -> NewServiceAccountOpts {
         NewServiceAccountOpts {
             access_key: access_key.to_string(),

crates/lifecycle/src/core.rs

@@ -18,7 +18,7 @@ use s3s::dto::{
     BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter,
     NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
 };
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use time::macros::offset;
 use time::{self, Duration, OffsetDateTime};
@@ -65,6 +65,66 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
     "Rule with ExpiredObjectDeleteMarker cannot have tags based filtering";
 const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
 const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
+const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer";
+const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str =
+    "Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And";
+const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates";
+const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key";
+const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters";
+const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative";
+const ERR_LIFECYCLE_FILTER_SIZE_RANGE: &str = "ObjectSizeGreaterThan must be smaller than ObjectSizeLessThan";
+/// Longest tag key S3 accepts.
+const MAX_TAG_KEY_LEN: usize = 128;
+/// Longest tag value S3 accepts.
+const MAX_TAG_VALUE_LEN: usize = 256;
+
+/// A validation failure that the S3 boundary must answer with `MalformedXML`
+/// rather than `InvalidArgument`: the document does not match the published
+/// schema shape (wrong number of `Filter` predicates, a one-member `And`).
+///
+/// Everything else stays [`std::io::ErrorKind::Other`], which the boundary
+/// already maps to `InvalidArgument`.
+pub const LIFECYCLE_MALFORMED_XML_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
+
+/// A persisted rule that could never have passed validation. Callers that can
+/// report an error surface it; evaluation itself stays fail-closed and takes
+/// no action for the rule.
+pub const LIFECYCLE_CORRUPT_RULE_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData;
+
+fn malformed_xml_error(message: &'static str) -> std::io::Error {
+    std::io::Error::new(LIFECYCLE_MALFORMED_XML_ERROR_KIND, message)
+}
+
+/// The retention count a rule keeps, or `None` when the persisted value is
+/// negative — a shape PUT validation rejects, so reaching it means the rule
+/// came from older persistence or an import.
+///
+/// A negative count must never be read as "retain everything": that is how an
+/// invalid configuration silently stopped deleting versions (backlog#2201).
+pub fn retained_noncurrent_versions(count: i32) -> Option<usize> {
+    usize::try_from(count).ok()
+}
+
+/// Does any rule carry a retention count that validation would have rejected?
+pub fn lifecycle_has_corrupt_retention_count(lc: &BucketLifecycleConfiguration) -> bool {
+    lc.rules.iter().any(rule_has_corrupt_retention_count)
+}
+
+fn rule_has_corrupt_retention_count(rule: &LifecycleRule) -> bool {
+    let expiration_count = rule
+        .noncurrent_version_expiration
+        .as_ref()
+        .and_then(|expiration| expiration.newer_noncurrent_versions);
+    let transition_counts = rule
+        .noncurrent_version_transitions
+        .iter()
+        .flatten()
+        .filter_map(|transition| transition.newer_noncurrent_versions);
+    expiration_count
+        .into_iter()
+        .chain(transition_counts)
+        .any(|count| retained_noncurrent_versions(count).is_none())
+}
 
 pub use rustfs_scanner_metrics::metrics::IlmAction;
 
@@ -141,6 +201,17 @@ impl RuleValidate for LifecycleRule {
             return Err(std::io::Error::other(ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT));
         }
 
+        if let Some(filter) = self.filter.as_ref() {
+            validate_lifecycle_filter(filter)?;
+        }
+
+        // A negative retention count was accepted and then read as "retain
+        // (almost) everything" during evaluation, so an HTTP-accepted rule
+        // silently stopped deleting versions (backlog#2201).
+        if rule_has_corrupt_retention_count(self) {
+            return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS));
+        }
+
         // Rule with DelMarkerExpiration cannot have tags based filtering
         let has_tag_filter = self
             .filter
@@ -173,11 +244,14 @@ impl RuleValidate for LifecycleRule {
         // Rule must have at least one action
         let has_expiration = self.expiration.is_some();
         let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
-        let has_noncurrent_expiration = self
+        // `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS
-            .noncurrent_version_expiration
+        // form: it keeps the newest N noncurrent versions and expires the rest
-            .as_ref()
+        // with no age condition. RustFS accepts it for MinIO compatibility, so
-            .and_then(|e| e.noncurrent_days)
+        // it has to count as an action here — otherwise a count-only rule was
-            .is_some();
+        // rejected as actionless (backlog#2201).
+        let has_noncurrent_expiration = self.noncurrent_version_expiration.as_ref().is_some_and(|expiration| {
+            expiration.noncurrent_days.is_some() || expiration.newer_noncurrent_versions.is_some_and(|count| count > 0)
+        });
         let has_noncurrent_transition = self
             .noncurrent_version_transitions
             .as_ref()
@@ -203,6 +277,81 @@ impl RuleValidate for LifecycleRule {
     }
 }
 
+/// Structural validation for `LifecycleRuleFilter`.
+///
+/// The generated DTO is all-`Option`, so the S3 schema constraints have to be
+/// checked here: at most one top-level predicate, an `And` that actually
+/// combines at least two, no repeated tag key, tag key/value limits, and a
+/// coherent non-negative size range (backlog#2201).
+///
+/// A filter with no predicate at all stays valid: AWS documents an empty
+/// `Filter` as "applies to every object in the bucket", and rejecting it would
+/// break the most common way to write an unconditional rule.
+fn validate_lifecycle_filter(filter: &LifecycleRuleFilter) -> Result<(), std::io::Error> {
+    let top_level_predicates = usize::from(filter.prefix.is_some())
+        + usize::from(filter.tag.is_some())
+        + usize::from(filter.object_size_greater_than.is_some())
+        + usize::from(filter.object_size_less_than.is_some())
+        + usize::from(filter.and.is_some());
+    if top_level_predicates > 1 {
+        return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES));
+    }
+
+    if let Some(tag) = filter.tag.as_ref() {
+        validate_lifecycle_tag(tag)?;
+    }
+
+    if let Some(and) = filter.and.as_ref() {
+        let tags = and.tags.as_deref().unwrap_or(&[]);
+        let and_predicates = usize::from(and.prefix.is_some())
+            + tags.len()
+            + usize::from(and.object_size_greater_than.is_some())
+            + usize::from(and.object_size_less_than.is_some());
+        if and_predicates < 2 {
+            return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES));
+        }
+        let mut seen_keys = HashSet::with_capacity(tags.len());
+        for tag in tags {
+            validate_lifecycle_tag(tag)?;
+            let key = tag.key.as_deref().unwrap_or_default();
+            if !seen_keys.insert(key) {
+                return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY));
+            }
+        }
+        validate_lifecycle_size_bounds(and.object_size_greater_than, and.object_size_less_than)?;
+    }
+
+    validate_lifecycle_size_bounds(filter.object_size_greater_than, filter.object_size_less_than)?;
+
+    Ok(())
+}
+
+/// S3 requires a tag to carry a key and value; both are length-bounded.
+/// The DTO makes both optional, so incomplete tags have to be rejected here
+/// rather than silently matching nothing.
+fn validate_lifecycle_tag(tag: &s3s::dto::Tag) -> Result<(), std::io::Error> {
+    let key = tag.key.as_deref().unwrap_or_default();
+    let Some(value) = tag.value.as_deref() else {
+        return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
+    };
+    if key.is_empty() || key.chars().count() > MAX_TAG_KEY_LEN || value.chars().count() > MAX_TAG_VALUE_LEN {
+        return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG));
+    }
+    Ok(())
+}
+
+fn validate_lifecycle_size_bounds(greater_than: Option<i64>, less_than: Option<i64>) -> Result<(), std::io::Error> {
+    if greater_than.is_some_and(|size| size < 0) || less_than.is_some_and(|size| size < 0) {
+        return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE));
+    }
+    if let (Some(greater_than), Some(less_than)) = (greater_than, less_than)
+        && greater_than >= less_than
+    {
+        return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_SIZE_RANGE));
+    }
+    Ok(())
+}
+
 fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> {
     // Prefer a non-empty legacy prefix; treat an empty legacy prefix as if it were not set
     if let Some(p) = rule.prefix.as_deref()
@@ -293,6 +442,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 {
                     return true;
                 }
+                // A positive count is an action on its own (the MinIO count-only
+                // form). Zero means "no count constraint" here, exactly as the
+                // batch limit path reads it, and a negative count is corrupt —
+                // neither makes the rule active (backlog#2201).
                 if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions
                     && newer_noncurrent_versions > 0
                 {
@@ -563,6 +716,23 @@ impl Lifecycle for BucketLifecycleConfiguration {
 
         if let Some(ref lc_rules) = self.filter_rules(obj).await {
             for rule in lc_rules.iter() {
+                // A retention count that PUT validation would have rejected can
+                // only come from older persistence or an import. Take no action
+                // for the rule instead of allowing another action on the same
+                // corrupt rule to delete or transition an object (backlog#2201).
+                if rule_has_corrupt_retention_count(rule) {
+                    debug!(
+                        event = EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                        object = %obj.name,
+                        rule_id = %rule.id.clone().unwrap_or_default(),
+                        reason = "corrupt_newer_noncurrent_versions",
+                        "Skipped lifecycle evaluation for a rule with an invalid retention count"
+                    );
+                    continue;
+                }
+
                 if obj.is_latest && obj.expired_object_deletemarker() {
                     if let Some(expiration) = rule.expiration.as_ref()
                         && expiration.expired_object_delete_marker.is_some_and(|v| v)
@@ -619,11 +789,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
 
                 if !obj.is_latest
                     && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
-                    && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
+                    && (noncurrent_version_expiration.noncurrent_days.is_some()
+                        || noncurrent_version_expiration
+                            .newer_noncurrent_versions
+                            .is_some_and(|count| count > 0))
                     && noncurrent_version_expiration
                         .newer_noncurrent_versions
                         .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
                 {
+                    // A count-only rule (MinIO extension) has no age condition:
+                    // every version past the retained count is due as soon as it
+                    // became noncurrent, i.e. zero days after the successor.
+                    let noncurrent_days = noncurrent_version_expiration.noncurrent_days.unwrap_or(0);
                     if let Some(successor_mod_time) = obj.successor_mod_time {
                         let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
                         if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
@@ -791,15 +968,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
             for rule in filter_rules.iter() {
                 if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
                     return if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
-                        if newer_noncurrent_versions == 0 {
+                        // Zero means "no count constraint"; a negative count is
+                        // corrupt and must not be read as "retain everything"
+                        // (backlog#2201). Neither yields a limit event.
+                        let Some(retained) = retained_noncurrent_versions(newer_noncurrent_versions).filter(|c| *c > 0) else {
                             continue;
-                        }
+                        };
                         Event {
                             action: IlmAction::DeleteVersionAction,
                             rule_id: rule.id.clone().unwrap_or_default(),
                             noncurrent_days: u32::try_from(noncurrent_version_expiration.noncurrent_days.unwrap_or(0))
                                 .unwrap_or(u32::MAX),
-                            newer_noncurrent_versions: usize::try_from(newer_noncurrent_versions).unwrap_or(usize::MAX),
+                            newer_noncurrent_versions: retained,
                             due: Some(OffsetDateTime::UNIX_EPOCH),
                             storage_class: "".into(),
                         }
@@ -1162,7 +1342,11 @@ mod tests {
     use super::*;
     use metrics_util::MetricKind;
     use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-    use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass};
+    use s3s::dto::{
+        LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, NoncurrentVersionTransition,
+        TransitionStorageClass,
+    };
+    use s3s::xml::{Deserialize as XmlDeserialize, SerializeContent as XmlSerializeContent};
     use serial_test::serial;
     use std::sync::Arc;
     use time::macros::datetime;
@@ -4183,6 +4367,583 @@ mod tests {
         assert_eq!(event.action, IlmAction::NoneAction);
     }
 
+    // ---- backlog#2201: retention-count and Filter invariants -----------------
+
+    fn rule_with_noncurrent_expiration(expiration: NoncurrentVersionExpiration) -> LifecycleRule {
+        LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            expiration: None,
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: None,
+            filter: None,
+            id: Some("noncurrent".to_string()),
+            noncurrent_version_expiration: Some(expiration),
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }
+    }
+
+    fn rule_with_filter(filter: LifecycleRuleFilter) -> LifecycleRule {
+        LifecycleRule {
+            status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+            expiration: Some(LifecycleExpiration {
+                days: Some(1),
+                ..Default::default()
+            }),
+            abort_incomplete_multipart_upload: None,
+            del_marker_expiration: None,
+            filter: Some(filter),
+            id: Some("filtered".to_string()),
+            noncurrent_version_expiration: None,
+            noncurrent_version_transitions: None,
+            prefix: None,
+            transitions: None,
+        }
+    }
+
+    fn config_with_rules(rules: Vec<LifecycleRule>) -> BucketLifecycleConfiguration {
+        BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules,
+        }
+    }
+
+    fn tag(key: &str, value: &str) -> s3s::dto::Tag {
+        s3s::dto::Tag {
+            key: Some(key.to_string()),
+            value: Some(value.to_string()),
+        }
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_negative_newer_noncurrent_versions() {
+        // A negative retention count used to be accepted and then read as
+        // usize::MAX during evaluation, so the rule silently stopped deleting
+        // versions (backlog#2201).
+        let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(30),
+            newer_noncurrent_versions: Some(-1),
+        })]);
+
+        let err = lc
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("a negative retention count must be rejected");
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS);
+        assert_ne!(err.kind(), LIFECYCLE_MALFORMED_XML_ERROR_KIND, "value errors stay InvalidArgument");
+    }
+
+    #[tokio::test]
+    async fn validate_rejects_negative_newer_noncurrent_versions_on_transition() {
+        let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(30),
+            newer_noncurrent_versions: None,
+        });
+        rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+            newer_noncurrent_versions: Some(-3),
+            noncurrent_days: Some(1),
+            storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
+        }]);
+
+        // The transition validator already refuses a negative count, and it runs
+        // first, so this pins the rejection rather than the message. The gap
+        // this PR closes is the expiration side, which had no such check.
+        config_with_rules(vec![rule])
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("a negative retention count on a transition must be rejected");
+    }
+
+    #[tokio::test]
+    async fn zero_newer_noncurrent_versions_means_no_count_constraint() {
+        // Zero carries no constraint, matching how the batch limit path has
+        // always read it. Alongside an age condition the rule is valid; on its
+        // own it says nothing, so the rule has no action.
+        config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(30),
+            newer_noncurrent_versions: Some(0),
+        })])
+        .validate(&ObjectLockConfiguration::default())
+        .await
+        .expect("zero count alongside NoncurrentDays is valid");
+
+        let err = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: None,
+            newer_noncurrent_versions: Some(0),
+        })])
+        .validate(&ObjectLockConfiguration::default())
+        .await
+        .expect_err("a zero count on its own is not an action");
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION);
+    }
+
+    #[tokio::test]
+    async fn validate_accepts_count_only_noncurrent_expiration() {
+        // MinIO extension: NewerNoncurrentVersions with no NoncurrentDays. It
+        // used to be rejected as an actionless rule (backlog#2201).
+        let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: None,
+            newer_noncurrent_versions: Some(2),
+        })]);
+
+        lc.validate(&ObjectLockConfiguration::default())
+            .await
+            .expect("a count-only noncurrent expiration rule is accepted");
+    }
+
+    #[tokio::test]
+    async fn eval_inner_expires_versions_beyond_count_only_retention() {
+        // Count-only rules have no age condition: everything past the retained
+        // count is due as soon as it became noncurrent.
+        let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: None,
+            newer_noncurrent_versions: Some(2),
+        })]);
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
+            successor_mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)),
+            is_latest: false,
+            num_versions: 5,
+            ..Default::default()
+        };
+
+        // Rank 2 is the third-newest noncurrent version: past a retention of 2.
+        let expired = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 2).await;
+        assert_eq!(expired.action, IlmAction::DeleteVersionAction);
+        assert_eq!(expired.rule_id, "noncurrent");
+
+        // Rank 1 is still within the retained count.
+        let retained = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 1).await;
+        assert_eq!(retained.action, IlmAction::NoneAction);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn eval_inner_keeps_age_condition_when_count_and_days_are_set() {
+        // With both set, the count gates which versions are candidates and the
+        // age condition still decides when they are due.
+        with_default_ilm_process_time(|| {});
+        let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(10),
+            newer_noncurrent_versions: Some(1),
+        })]);
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            is_latest: false,
+            num_versions: 3,
+            ..Default::default()
+        };
+
+        let too_young = lc.eval_inner(&opts, datetime!(2025-01-05 00:00:00 UTC), 2).await;
+        assert_eq!(too_young.action, IlmAction::NoneAction, "the age condition still applies");
+
+        let due = lc.eval_inner(&opts, datetime!(2025-01-20 00:00:00 UTC), 2).await;
+        assert_eq!(due.action, IlmAction::DeleteVersionAction);
+    }
+
+    #[tokio::test]
+    async fn eval_inner_takes_no_action_for_a_corrupt_retention_count() {
+        // Reachable only from older persistence or an import; it must not be
+        // read as "retain everything", and it must not delete either.
+        let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(-1),
+        })]);
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            is_latest: false,
+            num_versions: 3,
+            ..Default::default()
+        };
+
+        let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 2).await;
+
+        assert_eq!(event.action, IlmAction::NoneAction);
+    }
+
+    #[tokio::test]
+    async fn eval_inner_does_not_expire_latest_object_for_a_corrupt_retention_rule() {
+        let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(-1),
+        });
+        rule.expiration = Some(LifecycleExpiration {
+            days: Some(1),
+            ..Default::default()
+        });
+        let lc = config_with_rules(vec![rule]);
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            is_latest: true,
+            ..Default::default()
+        };
+
+        let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
+
+        assert_eq!(event.action, IlmAction::NoneAction);
+    }
+
+    #[tokio::test]
+    async fn eval_inner_does_not_delete_latest_marker_for_a_corrupt_retention_rule() {
+        let mut expired_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(-1),
+        });
+        expired_marker_rule.expiration = Some(LifecycleExpiration {
+            expired_object_delete_marker: Some(true),
+            ..Default::default()
+        });
+
+        let mut aged_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(-1),
+        });
+        aged_marker_rule.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(1) });
+
+        for rule in [expired_marker_rule, aged_marker_rule] {
+            let lc = config_with_rules(vec![rule]);
+            let opts = ObjectOpts {
+                name: "obj".to_string(),
+                mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+                version_id: Some(Uuid::new_v4()),
+                is_latest: true,
+                delete_marker: true,
+                num_versions: 1,
+                ..Default::default()
+            };
+
+            let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await;
+
+            assert_eq!(event.action, IlmAction::NoneAction);
+        }
+    }
+
+    #[test]
+    fn corrupt_retention_count_is_detected_on_either_action() {
+        let mut transition_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(0),
+        });
+        transition_rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+            newer_noncurrent_versions: Some(-1),
+            noncurrent_days: Some(1),
+            storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)),
+        }]);
+
+        assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
+            rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+                noncurrent_days: Some(1),
+                newer_noncurrent_versions: Some(-1),
+            })
+        ])));
+        assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![transition_rule])));
+        assert!(!lifecycle_has_corrupt_retention_count(&config_with_rules(vec![
+            rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+                noncurrent_days: Some(1),
+                newer_noncurrent_versions: Some(3),
+            })
+        ])));
+    }
+
+    #[test]
+    fn count_only_rules_are_active_only_for_a_positive_count() {
+        let positive = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: None,
+            newer_noncurrent_versions: Some(2),
+        })]);
+        assert!(positive.has_active_rules(""));
+
+        let corrupt = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: None,
+            newer_noncurrent_versions: Some(-1),
+        })]);
+        assert!(!corrupt.has_active_rules(""), "a corrupt retention count must not make a rule active");
+    }
+
+    #[tokio::test]
+    async fn noncurrent_versions_expiration_limit_ignores_a_corrupt_count() {
+        // The batch path must not read a negative count as "retain everything".
+        let lc = Arc::new(config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: Some(-1),
+        })]));
+        let opts = ObjectOpts {
+            name: "obj".to_string(),
+            mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)),
+            is_latest: false,
+            ..Default::default()
+        };
+
+        let event = lc.noncurrent_versions_expiration_limit(&opts).await;
+
+        assert_eq!(event.action, IlmAction::NoneAction);
+        assert_eq!(event.newer_noncurrent_versions, 0);
+    }
+
+    #[tokio::test]
+    async fn validate_covers_filter_invariants() {
+        struct Case {
+            name: &'static str,
+            filter: LifecycleRuleFilter,
+            expected: Option<(&'static str, std::io::ErrorKind)>,
+        }
+
+        let cases = vec![
+            Case {
+                // AWS documents an empty Filter as "every object in the bucket".
+                name: "empty filter applies to all objects",
+                filter: LifecycleRuleFilter::default(),
+                expected: None,
+            },
+            Case {
+                name: "single prefix predicate",
+                filter: LifecycleRuleFilter {
+                    prefix: Some("logs/".to_string()),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+            Case {
+                name: "two top-level predicates",
+                filter: LifecycleRuleFilter {
+                    prefix: Some("logs/".to_string()),
+                    tag: Some(tag("env", "prod")),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
+            },
+            Case {
+                name: "prefix alongside And",
+                filter: LifecycleRuleFilter {
+                    prefix: Some("logs/".to_string()),
+                    and: Some(LifecycleRuleAndOperator {
+                        prefix: Some("logs/".to_string()),
+                        tags: Some(vec![tag("env", "prod")]),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
+            },
+            Case {
+                name: "And with a single member",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        prefix: Some("logs/".to_string()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)),
+            },
+            Case {
+                name: "And with two members",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        prefix: Some("logs/".to_string()),
+                        tags: Some(vec![tag("env", "prod")]),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+            Case {
+                name: "And with two tags",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        tags: Some(vec![tag("env", "prod"), tag("team", "storage")]),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+            Case {
+                name: "And repeating a tag key",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        tags: Some(vec![tag("env", "prod"), tag("env", "dev")]),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "empty tag key",
+                filter: LifecycleRuleFilter {
+                    tag: Some(tag("", "prod")),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "missing tag key",
+                filter: LifecycleRuleFilter {
+                    tag: Some(s3s::dto::Tag {
+                        key: None,
+                        value: Some("prod".to_string()),
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "missing tag value",
+                filter: LifecycleRuleFilter {
+                    tag: Some(s3s::dto::Tag {
+                        key: Some("env".to_string()),
+                        value: None,
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "empty tag value",
+                filter: LifecycleRuleFilter {
+                    tag: Some(tag("env", "")),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+            Case {
+                name: "tag key at the limit",
+                filter: LifecycleRuleFilter {
+                    tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN), "prod")),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+            Case {
+                name: "tag key past the limit",
+                filter: LifecycleRuleFilter {
+                    tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN + 1), "prod")),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "tag value past the limit",
+                filter: LifecycleRuleFilter {
+                    tag: Some(tag("env", &"v".repeat(MAX_TAG_VALUE_LEN + 1))),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "negative ObjectSizeGreaterThan",
+                filter: LifecycleRuleFilter {
+                    object_size_greater_than: Some(-1),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "negative ObjectSizeLessThan",
+                filter: LifecycleRuleFilter {
+                    object_size_less_than: Some(-5),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "inverted size range inside And",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        object_size_greater_than: Some(100),
+                        object_size_less_than: Some(100),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: Some((ERR_LIFECYCLE_FILTER_SIZE_RANGE, std::io::ErrorKind::Other)),
+            },
+            Case {
+                name: "valid size range inside And",
+                filter: LifecycleRuleFilter {
+                    and: Some(LifecycleRuleAndOperator {
+                        object_size_greater_than: Some(1),
+                        object_size_less_than: Some(2),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                expected: None,
+            },
+        ];
+
+        for case in cases {
+            let result = config_with_rules(vec![rule_with_filter(case.filter)])
+                .validate(&ObjectLockConfiguration::default())
+                .await;
+            match (case.expected, result) {
+                (None, Ok(())) => {}
+                (None, Err(err)) => panic!("{}: expected acceptance, got {err}", case.name),
+                (Some((message, _)), Ok(())) => panic!("{}: expected rejection with {message}", case.name),
+                (Some((message, kind)), Err(err)) => {
+                    assert_eq!(err.to_string(), message, "{}", case.name);
+                    assert_eq!(err.kind(), kind, "{}: wrong S3 error category", case.name);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn validate_keeps_legacy_prefix_and_filter_mutually_exclusive() {
+        let mut rule = rule_with_filter(LifecycleRuleFilter {
+            prefix: Some("logs/".to_string()),
+            ..Default::default()
+        });
+        rule.prefix = Some("legacy/".to_string());
+
+        let err = config_with_rules(vec![rule])
+            .validate(&ObjectLockConfiguration::default())
+            .await
+            .expect_err("legacy Prefix and Filter cannot both be present");
+
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT);
+    }
+
+    #[test]
+    fn count_only_rule_round_trips_through_xml() {
+        // The MinIO count-only form has to survive the wire codec, or the rule
+        // this PR now accepts could not be persisted and read back.
+        let xml = br#"<LifecycleConfiguration><Rule><ID>count-only</ID><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>2</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
+        let mut deserializer = s3s::xml::Deserializer::new(xml);
+        let parsed =
+            <BucketLifecycleConfiguration as XmlDeserialize>::deserialize(&mut deserializer).expect("count-only XML parses");
+
+        let expiration = parsed.rules[0]
+            .noncurrent_version_expiration
+            .as_ref()
+            .expect("noncurrent expiration is present");
+        assert_eq!(expiration.newer_noncurrent_versions, Some(2));
+        assert_eq!(expiration.noncurrent_days, None);
+
+        let mut buf = Vec::new();
+        let mut serializer = s3s::xml::Serializer::new(&mut buf);
+        XmlSerializeContent::serialize_content(&parsed, &mut serializer).expect("count-only config serializes");
+        let serialized = String::from_utf8(buf).expect("serialized XML is UTF-8");
+        assert!(
+            serialized.contains("<NewerNoncurrentVersions>2</NewerNoncurrentVersions>"),
+            "retention count survives the round trip: {serialized}"
+        );
+        assert!(
+            !serialized.contains("<NoncurrentDays>"),
+            "a count-only rule must not gain an age condition: {serialized}"
+        );
+    }
+
     mod adversarial_regressions {
         use super::*;
         use s3s::dto::NoncurrentVersionExpiration;

crates/lifecycle/src/evaluator.rs

@@ -22,7 +22,10 @@ use rustfs_replication::ReplicationStatusType;
 use rustfs_scanner_metrics::metrics::IlmAction;
 
 use crate::object_lock;
-use crate::{Event, Lifecycle, ObjectOpts, expiration_action_has_valid_target};
+use crate::{
+    Event, LIFECYCLE_CORRUPT_RULE_ERROR_KIND, Lifecycle, ObjectOpts, expiration_action_has_valid_target,
+    lifecycle_has_corrupt_retention_count,
+};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
@@ -152,6 +155,17 @@ impl Evaluator {
                 format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()),
             ));
         }
+        // PUT validation rejects a negative retention count, so a rule that
+        // carries one came from older persistence or an import. Report it
+        // instead of evaluating a configuration that cannot be honoured;
+        // `eval_inner` independently takes no action for such a rule
+        // (backlog#2201).
+        if lifecycle_has_corrupt_retention_count(&self.policy) {
+            return Err(std::io::Error::new(
+                LIFECYCLE_CORRUPT_RULE_ERROR_KIND,
+                "lifecycle configuration carries a negative 'NewerNoncurrentVersions'",
+            ));
+        }
         Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await)
     }
 }