test(tier): pin candidate package digest

Problem: the nightly functional chain has not completed end-to-end.
Evidence from recent runs:
- workflow_run events are fire-and-forget: after KMS finished at 17:09Z
  on 8/31 no tier run was created; rustfs-storage-test.yml has never run.
- 'if: conclusion == success' gates skip downstream suites on any
  failure (security was skipped after pool failed on 9/1 01:48Z).
- rustfs-pool-expand-test.yml embedded a heal pass without
  continue-on-error, so a heal failure failed the whole workflow.

Fixes:
- Add rustfs-functional-chain.yml: entry point that dispatches the first
  suite via repository_dispatch; each suite hands off to the next with an
  explicit, re-drivable API call instead of workflow_run triggers.
- Split heal out of the pool workflow (renamed to RustFS Pool Expansion
  Test): heal now runs exactly once per chain, in rustfs-heal-test.yml
  (storage -> heal -> pool).
- Every suite job gets continue-on-error so a failing test never fails
  the workflow; failures are filed as issues in rustfs/backlog (report
  + redacted log tail) and the chain moves on.
- Clone rustfs/auto-testing with the PF token via 'gh repo clone' plus a
  5-attempt retry loop (transient clone failures aborted whole suites).
- Stop rewriting functional/index.html from every suite (divergent
  copies raced each other with stale SHAs); the canonical index now
  lives in the dashboard repo.
- Standalone workflow_dispatch runs are unchanged and never forward the
  chain; performance runs on its own runner, dispatched in parallel.

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
-sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2
+sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
+sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4

.config/e2e-nightly-selection.txt

@@ -1 +1 @@
-sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105
+sha256=51da41c54167602f2bd6c45921b39a44562bf3cfcdf468d992bb992c62cad7fd

.config/make/coverage.mak

@@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow,
 	@mkdir -p target/llvm-cov
 	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
 	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
+	$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json

.config/make/lint-fmt.mak

@@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 .PHONY: test-wiring-check
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
-	python3 ./scripts/check_test_wiring.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -35,13 +35,14 @@ script-tests: ## Run shell script tests
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/test_fuzz_runner.sh
+	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
-	python3 ./scripts/check_test_wiring.py --self-test
-	python3 ./scripts/check_security_coverage.py --self-test
-	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-	python3 ./scripts/s3-tests/test_report_compat.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
-	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
 
 .PHONY: test

.config/nextest.toml

@@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
 
 # --- default profile (local): serialize the flaky groups, never retry --------
 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.default.scripts]]
@@ -89,6 +89,29 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests. Every test in the module builds a
+# real-disk (TempDir-backed) hermetic erasure set and drives MiB-scale writes
+# plus deep-scan heal — the same load-sensitive cross-disk IO shape as the
+# crash_consistency scenarios above. Under a heavily parallel run a single
+# disk's IO can fail while write quorum still holds, which flips per-disk
+# readback and aggregate-outcome assertions nondeterministically (different
+# tests each round; all pass standalone). Preventive serialization only, no
+# retries. The matching ci-profile override is after [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair. Both carry
+# #[serial(metadata_cache_invalidation_probe)] — a no-op across nextest's
+# process boundary — and assert get_object_metadata_cache generation
+# semantics on a 4-disk hermetic set, the same load-sensitive shape that
+# forced the transition matrix tests into this group. Preventive
+# serialization only, no retries. The matching ci-profile override is after
+# [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # The production-handler relocation regression builds an isolated 8-disk,
 # 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
 # from overlapping the ecstore commit fixtures above.
@@ -183,7 +206,7 @@ fail-fast = false
 path = "junit.xml"
 
 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.ci.scripts]]
@@ -250,6 +273,20 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests under the ci profile too (see the
+# matching default-profile override near the top). Not a quarantine: no
+# retries, just serialized real-disk heal IO.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair under the ci
+# profile too (see the matching default-profile override near the top). Not a
+# quarantine: no retries.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # Match the default-profile embedded test isolation without quarantining or
 # retrying failures in CI.
 [[profile.ci.overrides]]
@@ -440,7 +477,7 @@ path = "junit.xml"
 [profile.e2e-nightly]
 default-filter = """
     package(e2e_test)
-    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
 """
 fail-fast = false
 
@@ -496,7 +533,7 @@ path = "junit.xml"
 default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
-    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
     & !test(/^replication_extension_test::/)
 """
 fail-fast = false

.github/workflows/audit.yml

@@ -24,8 +24,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
@@ -37,8 +40,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
     # Daily, not weekly. This schedule exists to catch RustSec advisories
@@ -146,6 +152,12 @@ jobs:
       - name: Check performance A/B workflow trust boundary
         run: ./scripts/security/check_performance_ab_workflow.sh
 
+      - name: Check tier evidence workflow isolation
+        run: ./scripts/security/check_tier_artifact_workflow.sh
+
+      - name: Check package version contract
+        run: ./scripts/test_package_versions.sh
+
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -244,7 +244,7 @@ jobs:
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 150
+    timeout-minutes: 180
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Release binaries ship without dial9 telemetry and therefore do not need
@@ -408,9 +408,9 @@ jobs:
 
           if [[ "${{ matrix.cross }}" == "true" ]]; then
             # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           fi
 
       - name: Create release package

.github/workflows/e2e-upgrade.yml

@@ -49,8 +49,20 @@ env:
   UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
 
 jobs:
-  direct-upgrade:
-    name: Direct upgrade from rc.2
+  upgrade:
+    name: ${{ matrix.name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Direct upgrade from rc.2
+            cache_key: e2e-direct-upgrade
+            test: direct_upgrade_from_rc2_preserves_object_contracts
+            artifact: direct-upgrade
+          - name: Mixed-version rolling upgrade from rc.2
+            cache_key: e2e-mixed-version-upgrade
+            test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
+            artifact: mixed-version-upgrade
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -64,7 +76,7 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: e2e-direct-upgrade
+          cache-shared-key: ${{ matrix.cache_key }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: "false"
 
@@ -89,17 +101,17 @@ jobs:
           cargo build --locked -p rustfs --bin rustfs
           : > target/debug/rustfs.features
 
-      - name: Run direct-upgrade compatibility test
+      - name: Run upgrade compatibility test
         run: |
           cargo test --locked -p e2e_test \
-            upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \
+            "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: direct-upgrade-server-logs-${{ github.run_number }}
+          name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs
           if-no-files-found: warn
           retention-days: 14

.github/workflows/package.yml

@@ -21,10 +21,10 @@
 # - workflow_run: automatically package after "Build and Release" completes
 #   for a release tag (the mac/windows/linux binaries are already uploaded
 #   to the GitHub release before packaging starts)
-# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
+# - workflow_dispatch: manual fallback with a release tag and/or exact build run ID
 #
 # Flow:
-# 1. Resolve the triggering Build workflow run for the release tag
+# 1. Resolve and validate the selected Build workflow run and source identity
 # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
 # 3. Build DEB packages for amd64 and arm64
 # 4. Build RPM packages for x86_64 and aarch64
@@ -51,7 +51,7 @@ on:
         required: false
         type: string
       build_run_id:
-        description: "Build workflow run ID (overrides tag lookup)"
+        description: "Build workflow run ID (when combined with tag, both must identify the same release commit)"
         required: false
         type: string
 
@@ -82,6 +82,9 @@ jobs:
       version: ${{ steps.resolve.outputs.version }}
       build_type: ${{ steps.resolve.outputs.build_type }}
       build_run_id: ${{ steps.resolve.outputs.build_run_id }}
+      build_run_number: ${{ steps.resolve.outputs.build_run_number }}
+      head_sha: ${{ steps.resolve.outputs.head_sha }}
+      dev_sequence: ${{ steps.resolve.outputs.dev_sequence }}
       tag: ${{ steps.resolve.outputs.tag }}
     steps:
       - name: Resolve build run
@@ -89,90 +92,129 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
+          EVENT_NAME: ${{ github.event_name }}
+          REPOSITORY: ${{ github.repository }}
           INPUT_TAG: ${{ github.event.inputs.tag }}
           INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
         run: |
           set -euo pipefail
 
-          # Determine tag
-          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            TAG="${HEAD_BRANCH}"
-          elif [[ -n "$INPUT_TAG" ]]; then
-            TAG="$INPUT_TAG"
+          fail() {
+            echo "❌ $1" >&2
+            exit 1
+          }
+
+          TAG=""
+          BUILD_RUN_ID=""
+          case "$EVENT_NAME" in
+            workflow_run)
+              TAG="$HEAD_BRANCH"
+              BUILD_RUN_ID="$WORKFLOW_RUN_ID"
+              ;;
+            workflow_dispatch)
+              TAG="$INPUT_TAG"
+              BUILD_RUN_ID="$INPUT_RUN_ID"
+              ;;
+            *) fail "unsupported event: $EVENT_NAME" ;;
+          esac
+
+          # Validate and classify tags before using them in API paths or logs.
+          semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)'
+          prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)'
+          if [[ -n "$TAG" ]]; then
+            if [[ "$TAG" =~ ^${semver_core}-${prerelease_id}-preview\.(0|[1-9][0-9]*)$ ]]; then
+              BUILD_TYPE=preview
+            elif [[ "$TAG" =~ ^${semver_core}-${prerelease_id}$ ]]; then
+              BUILD_TYPE=prerelease
+            elif [[ "$TAG" =~ ^${semver_core}$ ]]; then
+              BUILD_TYPE=release
+            else
+              fail "tag is not a supported strict package version"
+            fi
           else
-            TAG=""
+            BUILD_TYPE=development
           fi
 
-          echo "Tag: ${TAG:-<none>}"
-
-          # Determine build run ID
-          BUILD_RUN_ID=""
-
-          if [[ -n "$INPUT_RUN_ID" ]]; then
-            # Explicit run ID takes priority
-            BUILD_RUN_ID="$INPUT_RUN_ID"
-            echo "Using explicit build run ID: $BUILD_RUN_ID"
-
-          elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            # Use the Build and Release run that triggered this workflow
-            BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
-            echo "Using triggering workflow run: $BUILD_RUN_ID"
-
+          if [[ -n "$BUILD_RUN_ID" ]]; then
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "build run ID must be a positive decimal integer"
+            echo "Using selected build run: $BUILD_RUN_ID"
           elif [[ -n "$TAG" ]]; then
-            # Find the build run that produced this tag
             echo "Looking for build run for tag: $TAG"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
 
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              # Tag might not be a branch; try event=push with head_branch matching
-              BUILD_RUN_ID=$(gh api \
-                "repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
-                --jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
-            fi
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful build run found for tag: $TAG"
-              exit 1
+            if [[ -z "$BUILD_RUN_ID" ]]; then
+              BUILD_RUN_ID=$(gh api --method GET \
+                "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+                -f event=push -f status=success -F per_page=100 2>/dev/null |
+                jq -r --arg tag "$TAG" \
+                  '[.workflow_runs[] | select(.head_branch == $tag)][0].id // empty' || true)
             fi
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful build run found for tag"
             echo "Found build run: $BUILD_RUN_ID"
-
           else
-            # No tag — latest successful main build
             echo "No tag specified, looking for latest main build"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful main build found"
-              exit 1
-            fi
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch=main -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful main build found"
             echo "Latest main build: $BUILD_RUN_ID"
           fi
 
-          # Determine version and build type
+          # Fetch once and use the same immutable run metadata for identity,
+          # ordering, workflow provenance, and release-channel validation.
+          RUN_JSON=$(gh api "repos/${REPOSITORY}/actions/runs/${BUILD_RUN_ID}") ||
+            fail "cannot read selected build run"
+          RUN_ID=$(jq -r '.id // empty' <<<"$RUN_JSON")
+          RUN_NUMBER=$(jq -r '.run_number // empty' <<<"$RUN_JSON")
+          RUN_STATUS=$(jq -r '.status // empty' <<<"$RUN_JSON")
+          RUN_CONCLUSION=$(jq -r '.conclusion // empty' <<<"$RUN_JSON")
+          RUN_PATH=$(jq -r '.path // empty' <<<"$RUN_JSON")
+          HEAD_SHA=$(jq -r '.head_sha // empty' <<<"$RUN_JSON")
+          RUN_HEAD_BRANCH=$(jq -r '.head_branch // empty' <<<"$RUN_JSON")
+
+          [[ "$RUN_ID" == "$BUILD_RUN_ID" ]] || fail "run metadata ID mismatch"
+          [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] || fail "build run number must be a positive decimal integer"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]] || fail "selected build run is not successful"
+          [[ "$RUN_PATH" == .github/workflows/build.yml ]] || fail "selected run is not Build and Release"
+          [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "selected build run has an invalid head SHA"
+          [[ "$RUN_HEAD_BRANCH" != *$'\n'* && -n "$RUN_HEAD_BRANCH" ]] || fail "selected build run has an invalid head branch"
+
           if [[ -n "$TAG" ]]; then
+            [[ "$RUN_HEAD_BRANCH" == "$TAG" ]] || fail "tag and build run head branch do not match"
+
+            TAG_REF_JSON=$(gh api "repos/${REPOSITORY}/git/ref/tags/${TAG}") ||
+              fail "cannot resolve release tag ref"
+            TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_REF_JSON")
+            TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_REF_JSON")
+            depth=0
+            while [[ "$TAG_OBJECT_TYPE" == tag && $depth -lt 5 ]]; do
+              TAG_OBJECT_JSON=$(gh api "repos/${REPOSITORY}/git/tags/${TAG_OBJECT_SHA}") ||
+                fail "cannot peel annotated release tag"
+              TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_OBJECT_JSON")
+              TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_OBJECT_JSON")
+              depth=$((depth + 1))
+            done
+            [[ "$TAG_OBJECT_TYPE" == commit && "$TAG_OBJECT_SHA" =~ ^[0-9a-f]{40}$ ]] ||
+              fail "release tag does not resolve to a commit"
+            [[ "$TAG_OBJECT_SHA" == "$HEAD_SHA" ]] || fail "release tag commit and build run head SHA do not match"
             VERSION="$TAG"
-            if [[ "$TAG" == *"-preview"* ]]; then
-              BUILD_TYPE="preview"
-            elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
-              BUILD_TYPE="prerelease"
-            else
-              BUILD_TYPE="release"
-            fi
+            DEV_SEQUENCE=""
           else
-            SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
-              --jq '.head_sha' 2>/dev/null | head -c 7)
-            VERSION="dev-${SHORT_SHA}"
-            BUILD_TYPE="development"
+            VERSION="dev-${HEAD_SHA}"
+            DEV_SEQUENCE="$RUN_NUMBER"
           fi
 
           {
             echo "version=$VERSION"
             echo "build_type=$BUILD_TYPE"
             echo "build_run_id=$BUILD_RUN_ID"
+            echo "build_run_number=$RUN_NUMBER"
+            echo "head_sha=$HEAD_SHA"
+            echo "dev_sequence=$DEV_SEQUENCE"
             echo "tag=${TAG}"
           } >> "$GITHUB_OUTPUT"
 
@@ -180,6 +222,7 @@ jobs:
           echo "  Version: $VERSION"
           echo "  Build type: $BUILD_TYPE"
           echo "  Build run ID: $BUILD_RUN_ID"
+          echo "  Build run number: $RUN_NUMBER"
 
   # Build DEB and RPM packages for each architecture
   package:
@@ -206,6 +249,22 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Normalize package metadata
+        id: versions
+        shell: bash
+        env:
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
+          DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+        run: |
+          set -euo pipefail
+
+          normalized=$(./scripts/release/package_versions.sh \
+            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
+          printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
+
       - name: Download binary artifact from build run
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
@@ -245,18 +304,16 @@ jobs:
       - name: Build DEB package
         id: deb
         shell: bash
+        env:
+          DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          DEB_FILE: ${{ steps.versions.outputs.deb_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          DEB_ARCH="${{ matrix.deb_arch }}"
-          # DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
-          # Use a variable for ~ to prevent tilde expansion by bash
-          TILDE='~'
-          DEB_VERSION="${VERSION/-/$TILDE}"
-          PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
+          PKG_DIR="${DEB_FILE%.deb}"
 
-          echo "Building DEB: ${PKG_DIR}.deb"
+          echo "Building DEB: ${DEB_FILE}"
 
           mkdir -p "${PKG_DIR}/DEBIAN"
           mkdir -p "${PKG_DIR}/usr/bin"
@@ -333,9 +390,12 @@ jobs:
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
           cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
 
-          fakeroot dpkg-deb --build "${PKG_DIR}"
+          fakeroot dpkg-deb --build "${PKG_DIR}" "$DEB_FILE"
 
-          DEB_FILE="${PKG_DIR}.deb"
+          [[ $(dpkg-deb -f "$DEB_FILE" Package) == rustfs ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Version) == "$DEB_VERSION" ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Architecture) == "$DEB_ARCH" ]]
+          dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -tf - | grep -Fx './usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
@@ -343,16 +403,19 @@ jobs:
       - name: Build RPM package
         id: rpm
         shell: bash
+        env:
+          RPM_VERSION: ${{ steps.versions.outputs.rpm_version }}
+          RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+          RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          RPM_ARCH="${{ matrix.rpm_arch }}"
-
           echo "Building RPM for ${RPM_ARCH}"
 
-          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
+          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
           sudo gem install fpm
+          ./scripts/test_package_versions.sh --require-package-managers
 
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
@@ -367,8 +430,10 @@ jobs:
 
           fpm -s dir -t rpm \
             --name rustfs \
-            --version "$VERSION" \
+            --version "$RPM_VERSION" \
+            --iteration "$RPM_RELEASE" \
             --architecture "$RPM_ARCH" \
+            --package "$RPM_FILE" \
             --depends "glibc >= 2.31" \
             --maintainer "RustFS Team <support@rustfs.com>" \
             --description "High-performance distributed object storage" \
@@ -410,13 +475,15 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
-            RPM_FILE="${RPM_FILE#./}"
-          if [[ -z "$RPM_FILE" ]]; then
+          if [[ ! -f "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
+          RPM_METADATA=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$RPM_FILE")
+          EXPECTED_METADATA=$(printf 'rustfs\n%s\n%s\n%s' "$RPM_VERSION" "$RPM_RELEASE" "$RPM_ARCH")
+          [[ "$RPM_METADATA" == "$EXPECTED_METADATA" ]]
+          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
@@ -438,6 +505,9 @@ jobs:
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
           R2_BUCKET: ${{ secrets.R2_BUCKET }}
           AWS_EC2_METADATA_DISABLED: true
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
@@ -455,7 +525,6 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
@@ -465,9 +534,6 @@ jobs:
 
           echo "📤 Uploading to $R2_PATH"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           for f in "$DEB_FILE" "$RPM_FILE"; do
             if [[ -n "$f" && -f "$f" ]]; then
               echo "Uploading: $f"
@@ -493,14 +559,13 @@ jobs:
         if: needs.resolve.outputs.tag != ''
         env:
           GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.resolve.outputs.tag }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
 
-          TAG="${{ needs.resolve.outputs.tag }}"
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           # Upload the packages, then refresh the release checksums so the new
           # assets are covered, matching the binary release flow.
           for f in "$DEB_FILE" "$RPM_FILE"; do
@@ -552,14 +617,19 @@ jobs:
     steps:
       - name: Print summary
         shell: bash
+        env:
+          SUMMARY_VERSION: ${{ needs.resolve.outputs.version }}
+          SUMMARY_BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SUMMARY_BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          SUMMARY_PACKAGE_STATUS: ${{ needs.package.result }}
         run: |
           {
             echo "## 📦 Package Summary"
             echo ""
             echo "| Item | Value |"
             echo "|------|-------|"
-            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
-            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
-            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
-            echo "| Package Status | ${{ needs.package.result }} |"
+            echo "| Version | \`${SUMMARY_VERSION}\` |"
+            echo "| Build Type | ${SUMMARY_BUILD_TYPE} |"
+            echo "| Build Run | #${SUMMARY_BUILD_RUN_ID} |"
+            echo "| Package Status | ${SUMMARY_PACKAGE_STATUS} |"
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-functional-chain.yml

@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Functional chain driver: runs the nine functional suites in a fixed order
+# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security, with
+# performance on its own runner in parallel) and guarantees the chain keeps
+# moving even when individual suites fail.
+#
+# Each suite workflow can still be dispatched standalone (workflow_dispatch);
+# only chain-triggered runs forward to the next suite via repository_dispatch,
+# so a standalone run never drags the rest of the chain behind it.
+#
+# Why not workflow_run chaining: GitHub does not guarantee delivery of
+# workflow_run events (they are fire-and-forget), and the head-SHA filter made
+# newly added suites (storage) unable to trigger at all. Explicit
+# repository_dispatch handoffs are verifiable and re-drivable.
+
+name: RustFS Functional Chain
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    # Entry point: start the chain after the nightly build completes. The
+    # build's own conclusion does not gate the chain; each suite reports its
+    # own result to rustfs/backlog and the dashboard.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  start-chain:
+    name: Start functional chain (upgrade first)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    steps:
+      - name: Dispatch first suite (upgrade)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-upgrade' \
+            -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -23,6 +23,11 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the storage suite finishes. Heal runs
+    # exactly once per chain; the pool expansion workflow no longer embeds
+    # its own heal pass.
+    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -49,19 +54,33 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 480
-    # Manual-only standalone run. Nightly chain already runs heal in
-    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
-    if: ${{ github.event_name == 'workflow_dispatch' }}
+    # Standalone manual run, or one link of the nightly functional chain
+    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -116,8 +135,8 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
+            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
+            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
             --log-file /tmp/rustfs-heal-test.log
 
       - name: Generate report
@@ -175,155 +194,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'heal'
+          SUITE_LABEL: 'Heal'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-heal-report.md'
+          LOG_FILE: '/tmp/rustfs-heal-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -354,6 +282,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Pool expansion)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Pool expansion"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-pool' \
+            -F 'client_payload[from_suite]=heal'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-kms-test.yml

@@ -23,10 +23,9 @@ on:
         description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after S3 compatibility test completes.
-    workflows: ["RustFS S3 Compatibility Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the S3 compatibility suite finishes.
+    types: [rustfs-chain-kms]
 
 permissions:
   contents: read
@@ -52,16 +51,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -240,105 +250,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'kms'
+          SUITE_LABEL: 'KMS'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-kms-report.md'
+          LOG_FILE: '/tmp/rustfs-kms.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -369,6 +338,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Tier)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Tier"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-tier' \
+            -F 'client_payload[from_suite]=kms'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-performance-test.yml

@@ -48,10 +48,10 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Run after the nightly build completes; the nightly deb is what the test installs.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
+    types: [rustfs-chain-performance]
 
 permissions:
   contents: read
@@ -84,19 +84,33 @@ env:
 jobs:
   performance-test:
     runs-on: pf-testing
+    # Requirement: a failing benchmark must not fail the workflow;
+    # failures are filed to rustfs/backlog.
+    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -212,6 +226,65 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'performance'
+          SUITE_LABEL: 'Performance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-perf-report.md'
+          LOG_FILE: '/tmp/rustfs-perf-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
       - name: Upload test logs & results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion / Heal Test
+name: RustFS Pool Expansion Test
 
 on:
   workflow_dispatch:
@@ -33,14 +33,6 @@ on:
         description: 'Run the pool decommission step (3-pool topology only)'
         type: boolean
         default: true
-      stop_node_gb:
-        description: 'Heal: stop the outage node when surviving nodes reach N GiB'
-        required: false
-        default: '15'
-      warp_stop_gb:
-        description: 'Heal: stop warp when surviving nodes reach N GiB'
-        required: false
-        default: '40'
       cleanup_before:
         description: 'Reset the nodes before the test (DESTROYS existing data/config)'
         type: boolean
@@ -49,17 +41,16 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the heal suite finishes.
+    types: [rustfs-chain-pool]
 
 permissions:
   contents: read
 
-# Only one test run at a time: every job mutates the same shared test
+# Only one test run at a time: the job mutates the same shared test
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
-# other. Jobs inside a run are chained with needs to serialize them.
+# other.
 concurrency:
   group: rustfs-shared-functional-tests
   cancel-in-progress: false
@@ -80,330 +71,16 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  heal-test:
-    name: Heal test
-    runs-on: smoke-testing
-    timeout-minutes: 480
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        if: ${{ inputs.cleanup_before != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Install RustFS package & start cluster
-        run: |
-          ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Preflight checks
-        run: |
-          ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Run heal test (write -> outage -> heal -> verify)
-        id: test
-        run: |
-          ARGS=(--steps "3,4,5,6,7" -y \
-            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          {
-            echo "# RustFS heal test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-          SUITE: heal
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          fi
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-heal-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-heal-test.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
-
-      - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS heal test failed"
-          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
-          echo "See the uploaded log artifact for details."
-
-  # Pool expansion runs after heal regardless of heal outcome.
+  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
+  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 360
-    needs: heal-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run') }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -411,14 +88,25 @@ jobs:
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Initialize pool test artifacts
         run: |
@@ -810,155 +498,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'pool'
+          SUITE_LABEL: 'Pool expansion'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
+          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -997,6 +594,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Security)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Security"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-security' \
+            -F 'client_payload[from_suite]=pool'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-s3-compat-test.yml

@@ -11,10 +11,9 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Run after upgrade compatibility completes; the nightly deb is what the test installs.
-    workflows: ["RustFS Upgrade Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the upgrade suite finishes.
+    types: [rustfs-chain-s3]
 
 permissions:
   contents: read
@@ -40,16 +39,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -220,155 +230,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 's3'
+          SUITE_LABEL: 'S3 compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+          LOG_FILE: '/tmp/rustfs-s3-compat.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -399,6 +318,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: KMS)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: KMS"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-kms' \
+            -F 'client_payload[from_suite]=s3'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-security-test.yml

@@ -46,10 +46,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs last in the functional chain, after pool/heal, on the shared VMs.
-    workflows: ["RustFS Pool Expansion / Heal Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the pool expansion suite finishes (last link).
+    types: [rustfs-chain-security]
 
 permissions:
   contents: read
@@ -77,16 +76,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Checkout repository (for the OIDC live gate script)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -193,155 +203,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'security'
+          SUITE_LABEL: 'Security'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-security-report.md'
+          LOG_FILE: ''
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()

.github/workflows/rustfs-storage-test.yml

@@ -20,10 +20,9 @@ on:
           - single-multi
           - multi-multi
         default: all
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the tier suite finishes.
+    types: [rustfs-chain-storage]
 
 permissions:
   contents: read
@@ -49,16 +48,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -235,156 +245,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'storage', label: 'Storage Engine' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'storage'
+          SUITE_LABEL: 'Storage engine'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-storage-report.md'
+          LOG_FILE: '/tmp/rustfs-storage.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -415,6 +333,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Heal)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Heal"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-heal' \
+            -F 'client_payload[from_suite]=storage'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-tier-test.yml

@@ -11,10 +11,22 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after KMS test completes.
-    workflows: ["RustFS KMS Test"]
-    types: [completed]
+      package_sha256:
+        description: 'Optional SHA-256 for package_url; mismatch is an infrastructure failure.'
+        required: false
+        type: string
+      rc_sha256:
+        description: 'Optional SHA-256 for the preinstalled rc binary; mismatch is an infrastructure failure.'
+        required: false
+        type: string
+      force_case_failure:
+        description: 'Diagnostic only: rewrite single-single/TIER-101 to FAIL after execution to verify artifact and final-gate behavior.'
+        required: false
+        default: false
+        type: boolean
+  repository_dispatch:
+    # Chain handoff: dispatched when the KMS suite finishes.
+    types: [rustfs-chain-tier]
 
 permissions:
   contents: read
@@ -33,23 +45,90 @@ env:
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  RUSTFS_EXPECTED_RC_SHA256: ${{ inputs.rc_sha256 || vars.RUSTFS_TIER_RC_SHA256 }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
 
 jobs:
   tier-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - name: Initialize run evidence directory
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then
+            echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2
+            exit 1
+          fi
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          AUTO_TESTING_REF: cxymds/fix-2132-tier-log-isolation
+          AUTO_TESTING_COMMIT: 02da54dd62110649dc2860fc5fcd9e08d2e9a1ca
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- \
+              --branch "${AUTO_TESTING_REF}" --single-branch --depth 1 --quiet; then
+              actual_commit="$(git -C auto-testing rev-parse HEAD)"
+              if [[ "${actual_commit}" == "${AUTO_TESTING_COMMIT}" ]]; then
+                echo "auto-testing ${actual_commit} cloned (attempt ${attempt})"
+                exit 0
+              fi
+              echo "auto-testing commit mismatch: expected ${AUTO_TESTING_COMMIT}, got ${actual_commit}" >&2
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
+
+      - name: Download exact rc candidate
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          repository: rustfs/rustfs-release-validation
+          run-id: '33465191972'
+          name: rc-under-test-33465191972-1
+          path: ${{ runner.temp }}/issue-2128-rc
+          github-token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Verify exact rc candidate
+        env:
+          RC_BIN: ${{ runner.temp }}/issue-2128-rc/rc
+          RC_PROVENANCE: ${{ runner.temp }}/issue-2128-rc/rc-build.json
+          RC_EXPECTED_COMMIT: f6b9b509a60ef172a2b037d638c2cac46e762129
+          RC_EXPECTED_SHA256: 3d128d99f05403f4028c7c9ae24b03d66a3e98f2090e66cb7f9f45a9e11fdce1
+        run: |
+          set -euo pipefail
+          test -s "${RC_BIN}"
+          test -s "${RC_PROVENANCE}"
+          jq -e \
+            --arg commit "${RC_EXPECTED_COMMIT}" \
+            --arg digest "${RC_EXPECTED_SHA256}" \
+            '.repository == "rustfs/cli"
+             and .requestedCommit == $commit
+             and .resolvedCommit == $commit
+             and .binarySha256 == $digest
+             and .target == "x86_64-unknown-linux-gnu"' \
+            "${RC_PROVENANCE}" >/dev/null
+          actual_sha256="$(sha256sum -- "${RC_BIN}" | awk '{print $1}')"
+          test "${actual_sha256}" = "${RC_EXPECTED_SHA256}"
+          chmod 0555 "${RC_BIN}"
+          "${RC_BIN}" --version
 
       - name: Show environment
         run: |
@@ -110,13 +189,30 @@ jobs:
         id: test
         continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-tier.log
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          PACKAGE_SHA256_INPUT: ${{ inputs.package_sha256 }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
         run: |
           set -euo pipefail
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
           chmod +x auto-testing/rustfs-tier-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          RC_BIN="${RUNNER_TEMP}/issue-2128-rc/rc"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          PACKAGE_SHA256="${PACKAGE_SHA256_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
+          ARGS=(
+            --all-topologies
+            -y
+            --log-file "${LOG_FILE}"
+            --rc-bin "${RC_BIN}"
+            --artifacts-dir "${TIER_ARTIFACTS_DIR}"
+          )
+          if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
+            ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
+          fi
+          if [ -n "${PACKAGE_SHA256}" ]; then
+            ARGS+=(--sha256 "${PACKAGE_SHA256}")
+          fi
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -126,15 +222,34 @@ jobs:
           fi
           ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
 
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-tier.log
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+      - name: Inject diagnostic case failure
+        if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }}
         run: |
           set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json"
+          test -s "${RESULT_FILE}"
+          TMP_FILE="$(mktemp "${TIER_ARTIFACTS_DIR}/cases/.forced.XXXXXX")"
+          jq '.status = "FAIL" | .case_rc = 97' "${RESULT_FILE}" > "${TMP_FILE}"
+          mv "${TMP_FILE}" "${RESULT_FILE}"
+
+      - name: Generate report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        env:
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          TRIGGER_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
+          REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md"
+          CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
           if [ -n "${PACKAGE_URL}" ]; then
             PACKAGE_SOURCE="${PACKAGE_URL}"
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -142,65 +257,31 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-tier-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          set +e
+          python3 auto-testing/rustfs_tier_report.py \
+            --results-dir "${TIER_ARTIFACTS_DIR}/cases" \
+            --provenance "${TIER_ARTIFACTS_DIR}/provenance.json" \
+            --output "${CASE_TABLE}"
+          CASE_GATE_RC=$?
+          set -e
+          printf '%s\n' "${CASE_GATE_RC}" > "${GATE_RC_FILE}"
+          if [ ! -s "${CASE_TABLE}" ]; then
+            {
+              echo "## Case Summary"
+              echo ""
+              echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
+            } > "${CASE_TABLE}"
+          fi
           {
             echo "# RustFS tier test report"
             echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${TRIGGER_NAME}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${TEST_OUTCOME}"
+            echo "- Structured Gate Exit: ${CASE_GATE_RC}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            cat "${CASE_TABLE}"
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -210,11 +291,11 @@ jobs:
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
           SUITE: tier
         run: |
           set -euo pipefail
@@ -236,115 +317,42 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: Verify required tier evidence
+        id: evidence_verify
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          failed=0
+          for name in \
+            rustfs-tier.log \
+            rustfs-tier-report.md \
+            rustfs-tier-cases.md \
+            rustfs-tier-gate.rc \
+            provenance.json; do
+            if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence is missing or empty: ${name}" >&2
+              failed=1
+            fi
+          done
+          for name in cases logs; do
+            if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence directory is missing: ${name}" >&2
+              failed=1
+            fi
+          done
+          if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then
+            echo "no atomic tier case result was produced" >&2
+            failed=1
           fi
+          [ "${failed}" -eq 0 ]
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-tier-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-tier.log
-            /tmp/rustfs-tier-report.md
-          if-no-files-found: warn
+          name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.TIER_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
@@ -367,6 +375,126 @@ jobs:
             '
           done
 
+      - name: Enforce tier suite result
+        id: gate
+        if: always()
+        env:
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
+        run: |
+          set -euo pipefail
+          failed=0
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${TEST_OUTCOME}" != "success" ]; then
+            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "structured gate result is unavailable because evidence initialization failed" >&2
+          elif [ ! -s "${GATE_RC_FILE}" ]; then
+            echo "structured gate result is missing" >&2
+            failed=1
+          else
+            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
+            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
+              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
+              failed=1
+            fi
+          fi
+          [ "${failed}" -eq 0 ]
+
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'tier'
+          SUITE_LABEL: 'Tier'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
+          GATE_OUTCOME: ${{ steps.gate.outcome }}
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
+          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
+            echo "- Evidence verification: ${VERIFY_OUTCOME}"
+            echo "- Final gate: ${GATE_OUTCOME}"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+              echo "(the run evidence directory was rejected; its contents were not read)"
+            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
+              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
+            elif [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
+      - name: "Continue functional chain (next: Storage engine)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Storage engine"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-storage' \
+            -F 'client_payload[from_suite]=tier'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-upgrade-test.yml

@@ -53,11 +53,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs first in the functional chain: upgrade compatibility gates the
-    # nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
+    types: [rustfs-chain-upgrade]
 
 permissions:
   contents: read
@@ -83,16 +81,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -288,155 +297,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 'upgrade', label: 'Upgrade' },
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('upgrade');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'upgrade'
+          SUITE_LABEL: 'Upgrade compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+          LOG_FILE: '/tmp/rustfs-upgrade.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -468,6 +386,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: S3 compatibility)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: S3 compatibility"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-s3' \
+            -F 'client_payload[from_suite]=upgrade'
+
       - name: Notify on failure
         if: failure()
         run: |

AGENTS.md

@@ -123,12 +123,13 @@ runtime/build output:
 - Use `make pre-commit` only when its repository-wide fast checks add confidence
   beyond the focused checks.
 
-### Broad or High-Risk Changes
+### Broad Cross-Module Changes
 
-After the required adversarial review, run `make pre-pr` when targeted coverage
-cannot bound the impact, including dependency/toolchain/build-matrix changes,
-unbounded cross-crate APIs, or locking, durability, erasure coding, replication,
-RPC, IAM/KMS/auth, cryptography, on-disk/on-wire, and S3-visible behavior.
+Do not run `make pre-pr` by default before opening a PR. Consider it only when
+the final diff is broad, spans multiple modules, and targeted checks cannot
+bound the impact. Decide dynamically from the affected boundaries and risks;
+otherwise use the scoped formatting, linting, compilation, and test checks
+above.
 
 `make pre-pr` includes `make pre-commit`; never run both for the same unchanged
 diff. Do not repeat a check already covered by a successful umbrella gate.

CLAUDE.md

@@ -15,7 +15,7 @@ cargo check -p <crate>               # fast type-check one crate
 cargo test -p <crate>                # test one crate
 cargo fmt --all                      # format (required before PR)
 make pre-commit                      # fast gate: fmt + arch checks + quick-check (NO clippy/tests)
-make pre-pr                          # full pre-PR gate: fmt + arch checks + clippy + tests
+make pre-pr                          # optional full gate for broad cross-module changes
 make build-docker BUILD_OS=ubuntu22.04
 ```
 

CONTRIBUTING.md

@@ -62,12 +62,20 @@ make test
 # Fast pre-commit gate — see below for exactly what it runs
 make pre-commit
 
-# Full pre-PR gate (pre-commit gates + clippy + tests)
+# Optional full gate for broad cross-module changes (pre-commit + clippy + tests)
 make pre-pr
 ```
 
 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.
 
+> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the
+> security-coverage and scheduled-validation self-tests in `make test`) and import
+> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through
+> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls
+> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a
+> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a
+> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`.
+
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
 > For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
@@ -88,14 +96,16 @@ make pre-pr
 8. `quick-check` — `cargo check --workspace --exclude e2e_test`
 
 **`make pre-commit` does NOT run clippy and does NOT run any tests.**
-A green `make pre-commit` is not enough to open a pull request.
+It does not replace the scoped Clippy and test checks applicable to a change.
 
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
 tests). Complete the applicable multi-role adversarial review described in
-`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
-updating a pull request. This is what CI enforces.
+`AGENTS.md` first. Do not run `make pre-pr` locally by default before opening or
+updating a pull request. Consider it only for a broad change that spans multiple
+modules and whose impact cannot be bounded by targeted checks; decide from the
+affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -114,8 +124,9 @@ Or manually:
 chmod +x .git/hooks/pre-commit
 ```
 
-With or without a hook, the expectation is the same: run `make pre-commit`
-before committing and `make pre-pr` before opening a pull request.
+With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
+applicable scoped checks, and reserve `make pre-pr` for broad cross-module
+changes whose impact cannot be bounded by those checks.
 
 ### 📝 Formatting Configuration
 
@@ -154,7 +165,9 @@ Example output when formatting fails:
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
-6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+6. **Run applicable scoped checks before opening/updating a PR**; consider
+   `make pre-pr` only for broad cross-module changes whose impact cannot be
+   bounded by targeted checks
 7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration

Cargo.lock

@@ -4210,7 +4210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5039,9 +5039,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284"
 
 [[package]]
 name = "hex"
@@ -5668,7 +5668,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6650,10 +6650,11 @@ dependencies = [
 
 [[package]]
 name = "mysql_async"
-version = "0.37.0"
+version = "0.37.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2"
+checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4"
 dependencies = [
+ "arc-swap",
  "bytes",
  "crossbeam-queue",
  "crossbeam-utils",
@@ -6986,9 +6987,9 @@ checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
 
 [[package]]
 name = "nvml-wrapper"
-version = "0.12.1"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
+checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
 dependencies = [
  "bitflags 2.13.1",
  "libloading",
@@ -7000,9 +7001,9 @@ dependencies = [
 
 [[package]]
 name = "nvml-wrapper-sys"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b"
+checksum = "5d2079f4c9b6d2170bfb71c6355734ead6c47da75c179847395c31f9f2f66ede"
 dependencies = [
  "libloading",
 ]
@@ -7013,7 +7014,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.21.7",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -8036,9 +8037,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppmd-rust"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
+checksum = "9e9219bcb9d7aca6b2f63c83cf100cf78bcd619ac46e6ecbd0dd90869a39345d"
 
 [[package]]
 name = "ppv-lite86"
@@ -8240,7 +8241,7 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -8260,7 +8261,7 @@ version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -8610,7 +8611,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -9461,6 +9462,7 @@ dependencies = [
  "rustfs-io-metrics",
  "rustfs-keystone",
  "rustfs-kms",
+ "rustfs-license",
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
@@ -9512,7 +9514,7 @@ dependencies = [
  "tokio-util",
  "tonic",
  "tower",
- "tower-http 0.7.0",
+ "tower-http 0.7.1",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -10063,6 +10065,13 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustfs-license"
+version = "1.0.0-rc.5"
+dependencies = [
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "rustfs-lifecycle"
 version = "1.0.0-rc.5"
@@ -10606,6 +10615,8 @@ dependencies = [
  "rustfs-s3select-api",
  "rustfs-test-utils",
  "s3s",
+ "serde",
+ "serde_json",
  "tokio",
  "tracing",
 ]
@@ -10943,7 +10954,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11016,7 +11027,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11073,7 +11084,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11104,6 +11115,7 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
+ "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
  "serde",
@@ -11127,28 +11139,45 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "s3s-rfc2047"
+version = "0.16.0-alpha.1"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
+dependencies = [
+ "base64-simd",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
+ "base64-simd",
+ "hmac 0.13.0",
  "jiff",
+ "sha1 0.11.0",
+ "smallvec",
  "thiserror 2.0.20",
 ]
 
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
 dependencies = [
  "arrayvec",
  "base64-simd",
  "hex-simd",
+ "hmac 0.13.0",
  "jiff",
  "nom 8.0.0",
  "serde",
+ "sha2 0.11.0",
  "smallvec",
+ "std-next",
  "thiserror 2.0.20",
+ "zeroize",
 ]
 
 [[package]]
@@ -12040,9 +12069,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "10.0.2"
+version = "11.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
+checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
 dependencies = [
  "async-trait",
  "chrono",
@@ -12233,10 +12262,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -12729,9 +12758,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
 dependencies = [
  "async-compression",
  "bitflags 2.13.1",
@@ -13355,7 +13384,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]

Cargo.toml

@@ -29,6 +29,7 @@ members = [
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
     "crates/keystone", # OpenStack Keystone integration
+    "crates/license", # License and entitlement provider contracts
     "crates/lifecycle", # Lifecycle rule evaluation contracts
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
@@ -71,7 +72,7 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.97.1"
+rust-version = "1.98.0"
 version = "1.0.0-rc.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
@@ -105,6 +106,7 @@ rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
 rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
 rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
+rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
 rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
 rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
 rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
@@ -142,7 +144,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
-mysql_async = { default-features = false, version = "0.37" }
+mysql_async = { default-features = false, version = "0.37.1" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
@@ -174,7 +176,7 @@ tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
 tower = { version = "0.5.3" }
-tower-http = { version = "0.7.0" }
+tower-http = { version = "0.7.1" }
 
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
@@ -232,7 +234,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
 astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
@@ -283,7 +285,7 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.16" }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.12.1"
+nvml-wrapper = "0.13.0"
 parking_lot = "0.12.5"
 path-absolutize = "4.0.1"
 percent-encoding = "2.3.2"
@@ -305,7 +307,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "28e9ebb23dd2fb7d667084f34121b4aa4807a5c6", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -355,7 +357,7 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "10.0.2" }
+suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"

Makefile

@@ -23,6 +23,12 @@ SHELL := $(shell which bash)
 .SHELLFLAGS = -eu -o pipefail -c
 
 DOCKER_CLI ?= docker
+# Python interpreter for the repository's helper scripts. They import tomllib
+# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go
+# through a resolver that picks a new-enough interpreter (or falls back to uv).
+# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via
+# RUSTFS_PYTHON_BIN=<command>.
+RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh
 IMAGE_NAME ?= rustfs:v1.0.0
 CONTAINER_NAME ?= rustfs-dev
 # Docker build configurations

crates/e2e_test/README.md

@@ -169,7 +169,7 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/common.rs

@@ -1469,31 +1469,18 @@ impl RustFSTestClusterEnvironment {
     ///   times out, or cluster service readiness times out.
     pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let binary_path = rustfs_binary_path();
+        self.start_with_binary(&binary_path).await
+    }
+
+    /// Start every cluster node with a specific RustFS binary.
+    ///
+    /// Upgrade compatibility tests use this to initialize a cluster with a
+    /// pinned previous release before replacing nodes with the workspace build.
+    pub async fn start_with_binary(&mut self, binary_path: &Path) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let volumes_arg = self.build_volumes_arg();
 
-        for (i, node) in self.nodes.iter_mut().enumerate() {
-            info!("Starting cluster node {} on {}", i, node.address);
-
-            let mut command = Command::new(&binary_path);
-            command
-                .env("RUSTFS_VOLUMES", &volumes_arg)
-                .env("RUSTFS_ADDRESS", &node.address)
-                .env("RUSTFS_ACCESS_KEY", &self.access_key)
-                .env("RUSTFS_SECRET_KEY", &self.secret_key)
-                .env("RUSTFS_CONSOLE_ENABLE", "false")
-                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
-
-            for (key, value) in &self.extra_env {
-                command.env(key, value);
-            }
-            for (key, value) in &self.node_extra_env[i] {
-                command.env(key, value);
-            }
-            capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
-
-            let process = command.current_dir(&node.data_dir).spawn()?;
-
-            node.process = Some(process);
+        for node_idx in 0..self.nodes.len() {
+            self.spawn_node(node_idx, binary_path, &volumes_arg)?;
         }
 
         for (i, node) in self.nodes.iter().enumerate() {
@@ -1509,20 +1496,46 @@ impl RustFSTestClusterEnvironment {
 
     /// Start one node process using the cluster's existing volume layout.
     pub async fn start_node(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        self.start_node_from_binary(node_idx, &binary_path).await
+    }
+
+    /// Start one stopped cluster node with a specific RustFS binary while
+    /// preserving the cluster's volume layout and that node's data directory.
+    pub async fn start_node_from_binary(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let volumes_arg = self.build_volumes_arg();
+        self.spawn_node(node_idx, binary_path, &volumes_arg)?;
+
+        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        self.wait_for_node_service_ready(node_idx).await?;
+        Ok(())
+    }
+
+    fn spawn_node(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+        volumes_arg: &str,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         self.ensure_node_index(node_idx)?;
         if self.nodes[node_idx].process.is_some() {
             return Err(format!("cluster node {node_idx} is already running").into());
         }
+        if !binary_path.is_file() {
+            return Err(format!("RustFS binary does not exist: {}", binary_path.display()).into());
+        }
 
-        let binary_path = rustfs_binary_path();
-        let volumes_arg = self.build_volumes_arg();
         let log_path = self.node_capture_log_paths[node_idx].clone();
         let node = &mut self.nodes[node_idx];
-        info!("Starting cluster node {} on {}", node_idx, node.address);
+        info!("Starting cluster node {} on {} with {}", node_idx, node.address, binary_path.display());
 
-        let mut command = Command::new(&binary_path);
+        let mut command = Command::new(binary_path);
         command
-            .env("RUSTFS_VOLUMES", &volumes_arg)
+            .env("RUSTFS_VOLUMES", volumes_arg)
             .env("RUSTFS_ADDRESS", &node.address)
             .env("RUSTFS_ACCESS_KEY", &self.access_key)
             .env("RUSTFS_SECRET_KEY", &self.secret_key)
@@ -1539,9 +1552,6 @@ impl RustFSTestClusterEnvironment {
 
         let process = command.current_dir(&node.data_dir).spawn()?;
         node.process = Some(process);
-
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
-        self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
 

crates/e2e_test/src/degraded_listing_availability_test.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression: an object legally committed at degraded write quorum must stay
+//! listable while a *different* drive is offline.
+//!
+//! On a 4-drive EC 2+2 set, a PUT made while one drive is down persists
+//! `xl.meta` on 3 of 4 drives (write quorum). If a different drive later goes
+//! offline before heal converges, a strict latest-listing quorum of 3 can only
+//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
+//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
+//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
+//! product fix relaxes the listing's required object quorum by the number of
+//! set drives the listing could not consult (see
+//! `latest_listing_required_object_quorum` in
+//! `crates/ecstore/src/store/list_objects.rs`).
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestClusterEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use bytes::Bytes;
+    use std::collections::HashSet;
+    use std::error::Error;
+    use std::time::{Duration, Instant};
+    use tracing::info;
+
+    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+    const BUCKET: &str = "degraded-listing-availability";
+    const OBJECT_COUNT: usize = 8;
+    /// Well under the observed heal-convergence gap (~50s in the CI incident),
+    /// so a listing that only completes after heal restores the missing copy
+    /// still fails this deadline on a regressed build.
+    const LISTING_DEADLINE: Duration = Duration::from_secs(25);
+    const GET_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+    const PUT_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+
+    fn object_key(idx: usize) -> String {
+        format!("degraded-object-{idx:02}")
+    }
+
+    async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
+        let mut keys = HashSet::new();
+        let mut continuation_token: Option<String> = None;
+        loop {
+            let response = client
+                .list_objects_v2()
+                .bucket(BUCKET)
+                .set_continuation_token(continuation_token.clone())
+                .send()
+                .await?;
+            keys.extend(
+                response
+                    .contents()
+                    .iter()
+                    .filter_map(|object| object.key().map(str::to_owned)),
+            );
+            match response.next_continuation_token() {
+                Some(token) => continuation_token = Some(token.to_owned()),
+                None => break,
+            }
+        }
+        Ok(keys)
+    }
+
+    /// 4-node single-drive cluster (EC 2+2, write quorum 3):
+    /// 1. Stop node 1 and PUT objects — each commits on nodes {0, 2, 3} only.
+    /// 2. Stop node 3 (a holder drive), then bring node 1 back before heal can
+    ///    recreate the missing copies there.
+    /// 3. Every object still satisfies read quorum (nodes 0 and 2), so GET
+    ///    must succeed AND ListObjectsV2 must report every key well before
+    ///    heal converges.
+    #[tokio::test]
+    async fn degraded_write_remains_listable_while_a_different_drive_is_offline() -> TestResult {
+        init_logging();
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+        // Listing availability must not depend on heal convergence: disable
+        // the background healers so the degraded objects keep their metadata
+        // on exactly 3 of 4 drives for the whole test.
+        cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
+        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
+        cluster.start().await?;
+        cluster.create_test_bucket(BUCKET).await?;
+        let client = cluster.create_s3_client(0)?;
+
+        info!("stopping node 1 so the uploads commit at degraded write quorum (3 of 4)");
+        cluster.stop_node(1)?;
+        // The first writes after a node drops can see transient 503s while the
+        // survivors notice the dead peer; retry briefly (overwrites of the same
+        // unversioned key are idempotent).
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let body = format!("degraded listing payload {idx}");
+            let deadline = Instant::now() + PUT_RETRY_DEADLINE;
+            loop {
+                let request = client
+                    .put_object()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .body(Bytes::from(body.clone()).into());
+                match request.send().await {
+                    Ok(_) => break,
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded PUT for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded PUT for {key} failed: {error}").into()),
+                }
+            }
+        }
+
+        info!("stopping node 3 (holds a copy) and restoring node 1 (holds none)");
+        cluster.stop_node(3)?;
+        cluster.start_node(1).await?;
+
+        // The first requests after a node drops can see transient 503s while
+        // the survivors notice the dead peer; retry briefly before asserting.
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let deadline = Instant::now() + GET_RETRY_DEADLINE;
+            let body = loop {
+                match client.get_object().bucket(BUCKET).key(&key).send().await {
+                    Ok(response) => break response.body.collect().await?.into_bytes(),
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded GET for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded object {key} failed read quorum GET: {error}").into()),
+                }
+            };
+            assert!(!body.is_empty(), "degraded object {key} should read back at read quorum");
+        }
+
+        let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
+        let deadline = Instant::now() + LISTING_DEADLINE;
+        let listed = loop {
+            let listed = match list_all_keys(&client).await {
+                Ok(keys) => keys,
+                Err(error) if Instant::now() < deadline => {
+                    info!("retrying degraded listing: {error}");
+                    tokio::time::sleep(Duration::from_millis(500)).await;
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            if expected.is_subset(&listed) {
+                break listed;
+            }
+            assert!(
+                Instant::now() < deadline,
+                "objects readable at read quorum stayed missing from ListObjectsV2 for {LISTING_DEADLINE:?}: \
+                 missing={:?} listed={listed:?}",
+                expected.difference(&listed).collect::<Vec<_>>(),
+            );
+            tokio::time::sleep(Duration::from_millis(500)).await;
+        };
+        info!(listed = listed.len(), "degraded objects are listable while node 3 is offline");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/lib.rs

@@ -348,6 +348,11 @@ mod delete_regression_test;
 #[cfg(test)]
 mod listing_regression_test;
 
+// Cluster regression: objects committed at degraded write quorum must stay
+// listable while a different drive is offline (CI run 33478999853).
+#[cfg(test)]
+mod degraded_listing_availability_test;
+
 // P1 regression: bucket statistics accuracy (rustfs#5615, #5008, #5116, #5055, #3898, #1012)
 #[cfg(test)]
 mod bucket_stats_regression_test;

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,14 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
 };
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::task::JoinSet;
+use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
@@ -28,6 +31,14 @@ const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
 const VERSIONED_BUCKET: &str = "upgrade-versioned-data";
+const MIXED_BUCKET: &str = "upgrade-mixed-version-data";
+const MIXED_NODE_COUNT: usize = 4;
+const MULTIPART_WORKERS: usize = 16;
+const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
+// Peers keep a restarted node's drive in Suspect/Returning for roughly
+// probe_interval (2s) x success_threshold (3) after it comes back; 30s
+// comfortably covers that window plus CI scheduling jitter.
+const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
@@ -103,6 +114,132 @@ async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<
     Ok(())
 }
 
+fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("mixed-upgrade-node-{node_idx}.log"));
+        cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
+    let mut tasks = JoinSet::new();
+    for worker in 0..MULTIPART_WORKERS {
+        let client = clients[worker % clients.len()].clone();
+        let phase = phase.to_string();
+        tasks.spawn(async move {
+            let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
+            for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
+                let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
+                let part = vec![u8::try_from(worker)?; 64 * 1024];
+                write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
+                keys.push(key);
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
+        });
+    }
+
+    let mut keys = Vec::with_capacity(MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER);
+    while let Some(result) = tasks.join_next().await {
+        keys.extend(result??);
+    }
+    Ok(keys)
+}
+
+/// Assert that `client` eventually lists exactly `expected` objects under
+/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
+///
+/// A single-snapshot assertion here is racy by construction: each phase both
+/// writes and lists within seconds of a node restart. While a peer still holds
+/// the restarted node's drive in Suspect/Returning, strict-quorum listing
+/// consults only the remaining three drives and drops any object that was
+/// itself legally written at write quorum (3/4 drives) during an earlier
+/// node's identical post-restart window — its xl.meta is then visible on only
+/// two of the three consulted drives, below the required object quorum of
+/// three. GET still succeeds for such objects; only the listing under-counts
+/// until drive health converges. A genuine upgrade data-loss regression still
+/// fails after the deadline.
+async fn wait_for_phase_listing(client: &Client, phase: &str, expected: usize, context: &str) -> TestResult {
+    let deadline = Instant::now() + LISTING_CONVERGENCE_TIMEOUT;
+    loop {
+        let listed = client
+            .list_objects_v2()
+            .bucket(MIXED_BUCKET)
+            .prefix(format!("{phase}/"))
+            .send()
+            .await?;
+        let count = listed.contents().len();
+        if count == expected {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!(
+                "{context}: listing under {phase}/ returned {count} of {expected} objects even after {}s of post-restart convergence",
+                LISTING_CONVERGENCE_TIMEOUT.as_secs()
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn exercise_mixed_cluster(
+    cluster: &RustFSTestClusterEnvironment,
+    phase: &str,
+    current_node: usize,
+    previous_node: usize,
+) -> TestResult {
+    let clients = cluster.create_all_clients()?;
+    let current_client = &clients[current_node];
+    let previous_client = &clients[previous_node];
+
+    let current_key = format!("{phase}/written-by-current");
+    let current_body = format!("{phase}: current RustFS build").into_bytes();
+    current_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&current_key)
+        .body(ByteStream::from(current_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(previous_client, MIXED_BUCKET, &current_key, None).await?.1, current_body);
+
+    let previous_key = format!("{phase}/written-by-previous");
+    let previous_body = format!("{phase}: previous RustFS release").into_bytes();
+    previous_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&previous_key)
+        .body(ByteStream::from(previous_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
+
+    let multipart_keys = write_multipart_load(&clients, phase).await?;
+    let expected_count = multipart_keys.len() + 2;
+    for (label, client) in [("current", current_client), ("previous", previous_client)] {
+        wait_for_phase_listing(
+            client,
+            phase,
+            expected_count,
+            &format!("the {label} RustFS version must stream the complete mixed-version listing"),
+        )
+        .await?;
+    }
+
+    let last_multipart_key = format!("{phase}/multipart/{:02}/{:02}", MULTIPART_WORKERS - 1, MULTIPART_UPLOADS_PER_WORKER - 1);
+    assert_eq!(
+        read_object(previous_client, MIXED_BUCKET, &last_multipart_key, None).await?.1,
+        vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -252,3 +389,43 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
 
     Ok(())
 }
+
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let current_binary = rustfs_binary_path();
+    let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
+    cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn");
+    configure_cluster_logs(&mut cluster)?;
+    cluster.start_with_binary(&previous_binary).await?;
+    cluster.create_test_bucket(MIXED_BUCKET).await?;
+
+    cluster.stop_node(0)?;
+    cluster.start_node_from_binary(0, &current_binary).await?;
+    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+
+    for node_idx in [1, 2] {
+        cluster.stop_node(node_idx)?;
+        cluster.start_node_from_binary(node_idx, &current_binary).await?;
+    }
+    exercise_mixed_cluster(&cluster, "one-previous-node", 0, 3).await?;
+
+    cluster.stop_node(3)?;
+    cluster.start_node_from_binary(3, &current_binary).await?;
+
+    for (node_idx, client) in cluster.create_all_clients()?.iter().enumerate() {
+        for phase in ["one-current-node", "one-previous-node"] {
+            wait_for_phase_listing(
+                client,
+                phase,
+                MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2,
+                &format!("node {node_idx}: the homogeneous current cluster must preserve every object"),
+            )
+            .await?;
+        }
+    }
+
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -75,6 +75,10 @@ pub mod bucket {
                 delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
                 inspect_transition_transaction_for_operator,
             };
+            #[cfg(feature = "test-util")]
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionTransactionRecoveryStats, recover_transition_transaction_records,
+            };
         }
 
         pub mod evaluator {
@@ -99,6 +103,17 @@ pub mod bucket {
             pub use crate::bucket::lifecycle::tier_delete_journal::{
                 persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
             };
+
+            #[cfg(feature = "test-util")]
+            pub mod test_util {
+                /// Model a single-node, all-v6 fleet after its capability probe has completed.
+                ///
+                /// Call this only once while constructing an isolated test store, before any
+                /// tier-delete journal permit or background worker can be active.
+                pub fn install_all_v6_fleet_capability_proof() {
+                    crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+                }
+            }
         }
 
         pub mod tier_last_day_stats {

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
     DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
 };
 use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
-use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
+use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
 use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
 use crate::bucket::object_lock::ObjectLockApi;
 use crate::bucket::versioning::VersioningApi as _;
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
 use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
 use crate::error::Error;
 use crate::error::StorageError;
-use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
+use crate::error::{
+    is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
+    is_network_or_host_down,
+};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
 use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
 use crate::services::tier::{
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
     }
 }
 
-async fn delete_free_version_remote_object(
+async fn acquire_free_version_tier_lease(
     oi: &ObjectInfo,
     tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
-) -> Result<(), std::io::Error> {
+) -> Result<(TierOperationLease, bool), std::io::Error> {
     let version_id_exact = validate_transition_remote_version(oi)?;
     let identity = tier_destination_id_from_metadata(&oi.user_defined)?
         .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
-    delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+    let lease =
+        TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
+            .await
+            .map_err(std::io::Error::other)?;
+    Ok((lease, version_id_exact))
+}
+
+async fn delete_free_version_remote_object_with_lease(
+    oi: &ObjectInfo,
+    lease: &TierOperationLease,
+    version_id_exact: bool,
+) -> Result<(), std::io::Error> {
+    delete_object_from_remote_tier_with_lease_idempotent(
         &oi.transitioned_object.name,
         &oi.transitioned_object.version_id,
-        &oi.transitioned_object.tier,
-        identity,
-        tier_config_mgr,
+        lease,
         version_id_exact,
     )
     .await?;
     Ok(())
 }
 
+fn free_version_physical_topology_generation(api: &ECStore) -> String {
+    let mut hasher = Sha256::new();
+    for pool in &api.pools {
+        hasher.update(pool.pool_idx.to_be_bytes());
+        hasher.update(pool.disk_set.len().to_be_bytes());
+        for set in &pool.disk_set {
+            hasher.update(set.set_index.to_be_bytes());
+        }
+    }
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
+    if candidate.transitioned_object.tier != expected.transitioned_object.tier
+        || candidate.transitioned_object.name != expected.transitioned_object.name
+    {
+        return Ok(false);
+    }
+    let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
+    let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
+    if candidate_identity != expected_identity {
+        return Ok(false);
+    }
+    if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+        || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+    {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version remote version state is unknown",
+        ));
+    }
+    Ok(candidate.transition_version_state == expected.transition_version_state
+        && candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
+}
+
+async fn scan_exact_free_version_targets(
+    api: &ECStore,
+    oi: &ObjectInfo,
+    local_object: &str,
+) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
+    let mut targets = Vec::new();
+    for pool in &api.pools {
+        for set in &pool.disk_set {
+            let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
+                Ok(Some(versions)) => versions,
+                Ok(None) => continue,
+                Err(err) if is_err_strict_volume_not_found(&err) => continue,
+                Err(err) => return Err(std::io::Error::other(err)),
+            };
+            for version in versions.versions.iter().chain(versions.free_versions.iter()) {
+                let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
+                if free_version_remote_tuple_matches(&candidate, oi)? {
+                    if candidate.transitioned_object.free_version {
+                        // Data movement can leave the same remote tuple in
+                        // several physical pools. Ordinary deletion assigns a
+                        // fresh local free-version UUID to each copy, but all
+                        // of those markers own the same idempotent remote
+                        // DELETE. Consume them together while holding every
+                        // physical object lock; treating their local UUIDs as
+                        // conflicting would strand cleanup forever.
+                        let mut actual = version.clone();
+                        actual.name = local_object.to_string();
+                        targets.push((Arc::clone(set), actual));
+                    } else {
+                        return Err(std::io::Error::new(
+                            std::io::ErrorKind::WouldBlock,
+                            "a live transitioned source still references the free-version remote tuple",
+                        ));
+                    }
+                }
+            }
+        }
+    }
+    Ok(targets)
+}
+
+fn free_version_cleanup_fences_current(
+    topology_generation: &str,
+    api: &ECStore,
+    bucket_guard: &rustfs_lock::NamespaceLockGuard,
+    object_guards: &[crate::store::ObjectLockDiagGuard],
+    lease: &TierOperationLease,
+    cancel: &CancellationToken,
+    deadline: tokio::time::Instant,
+) -> bool {
+    !cancel.is_cancelled()
+        && tokio::time::Instant::now() < deadline
+        && !bucket_guard.is_lock_lost()
+        && object_guards.iter().all(|guard| !guard.is_lock_lost())
+        && lease.is_current_generation()
+        && free_version_physical_topology_generation(api) == topology_generation
+}
+
+async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
+    const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
+
+    let topology_generation = free_version_physical_topology_generation(&api);
+    let bucket_guard = api
+        .acquire_bucket_lifecycle_read_lock(&oi.bucket)
+        .await
+        .map_err(std::io::Error::other)?;
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
+    let local_object = encode_dir_object(&oi.name);
+    let object_guards = api
+        .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
+        .await
+        .map_err(std::io::Error::other)?;
+    let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if targets.is_empty() {
+        return Ok(false);
+    }
+
+    let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence is invalid before remote delete",
+        ));
+    }
+    tokio::select! {
+        _ = cancel.cancelled() => {
+            return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
+        }
+        result = tokio::time::timeout_at(
+            deadline,
+            delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
+        ) => {
+            result
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
+        }
+    }
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        // Remote DELETE is idempotent, but a changed fence makes the local
+        // outcome ambiguous. Keep every marker for a fully fenced retry.
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence changed after remote delete",
+        ));
+    }
+
+    let mut first_error = None;
+    for (set, actual) in &targets {
+        let mut delete_request = FileInfo {
+            name: local_object.clone(),
+            version_id: actual.version_id,
+            ..Default::default()
+        };
+        delete_request.set_tier_free_version();
+        if let Err(err) = set
+            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
+            .await
+            && first_error.is_none()
+        {
+            first_error = Some(std::io::Error::other(err));
+        }
+    }
+    let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if !remaining.is_empty() {
+        return Err(first_error.unwrap_or_else(|| {
+            std::io::Error::new(
+                std::io::ErrorKind::WouldBlock,
+                "tier free-version cleanup remained on at least one physical set",
+            )
+        }));
+    }
+    if let Some(err) = first_error {
+        return Err(err);
+    }
+    Ok(true)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn delete_free_version_remote_object(
+    oi: &ObjectInfo,
+    tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
+) -> Result<(), std::io::Error> {
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
+}
+
 #[allow(
     dead_code,
     reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -618,8 +813,11 @@ where
     F: FnOnce() -> Fut,
     Fut: std::future::Future<Output = T>,
 {
-    delete_free_version_remote_object(oi, tier_config_mgr).await?;
-    Ok(delete_local().await)
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
+    let result = delete_local().await;
+    drop(lease);
+    Ok(result)
 }
 
 struct NewerNoncurrentTask {
@@ -690,6 +888,10 @@ impl ExpiryState {
         usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
     }
 
+    pub fn active_tasks(&self) -> usize {
+        usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
+    }
+
     fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
         let queued = wrkr.try_send(Some(task)).is_ok();
         if queued {
@@ -826,7 +1028,7 @@ impl ExpiryState {
     }
 
     pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let expiry_state = api.ctx.expiry_state();
         if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
             return;
         }
@@ -867,7 +1069,7 @@ impl ExpiryState {
         stats: Arc<ExpiryStats>,
         recovery_notify: Arc<Notify>,
     ) {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
             static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
             FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
         });
@@ -968,119 +1170,33 @@ impl ExpiryState {
                     else if v.as_any().is::<FreeVersionTask>() {
                         let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
                         let oi = v.0.clone();
-                        if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
-                            recovery_notify.notify_one();
-                            debug!(
-                                bucket = %oi.bucket,
-                                object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
-                                error = ?err,
-                                event = EVENT_LIFECYCLE_WORKER_STATE,
-                                component = LOG_COMPONENT_ECSTORE,
-                                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                reason = "remote_tier_delete_failed",
-                                "Lifecycle worker skipped remote tier delete"
-                            );
-                            continue;
-                        }
-
-                        let local_object = encode_dir_object(&oi.name);
-                        let mut fi = FileInfo {
-                            name: local_object.clone(),
-                            version_id: oi.version_id,
-                            ..Default::default()
-                        };
-                        // This removes an existing internal cleanup marker. Keeping
-                        // `deleted` false makes duplicate tasks return not-found
-                        // instead of creating an ordinary delete marker.
-                        fi.set_tier_free_version();
-
-                        let mut deleted_locally = false;
-                        for pool in &api.pools {
-                            let set = pool.get_disks_by_key(&local_object);
-                            let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
-                                Ok(lock) => lock,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        pool_index = pool.pool_idx,
-                                        set_index = set.set_index,
-                                        error = ?err,
-                                        reason = "local_free_version_lock_failed",
-                                        "Lifecycle worker failed to create local free-version cleanup lock"
-                                    );
-                                    continue;
-                                }
-                            };
-                            let _object_lock_guard =
-                                match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
-                                    Ok(guard) => guard,
-                                    Err(err) => {
-                                        recovery_notify.notify_one();
-                                        debug!(
-                                            event = EVENT_LIFECYCLE_WORKER_STATE,
-                                            component = LOG_COMPONENT_ECSTORE,
-                                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                            bucket = %oi.bucket,
-                                            object = %oi.name,
-                                            pool_index = pool.pool_idx,
-                                            set_index = set.set_index,
-                                            error = ?err,
-                                            reason = "local_free_version_lock_failed",
-                                            "Lifecycle worker failed to acquire local free-version cleanup lock"
-                                        );
-                                        continue;
-                                    }
-                                };
-                            match set
-                                .delete_object_version(&oi.bucket, &local_object, &fi, false)
-                                .await
-                            {
-                                Ok(()) => {
-                                    deleted_locally = true;
-                                    break;
-                                }
-                                Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        remote_object = %oi.transitioned_object.name,
-                                        remote_version_id = %oi.transitioned_object.version_id,
-                                        tier = %oi.transitioned_object.tier,
-                                        error = ?err,
-                                        reason = "local_free_version_delete_failed",
-                                        "Lifecycle worker failed local free-version cleanup"
-                                    );
-                                    break;
-                                }
-                            }
-                        }
-
-                        if !deleted_locally {
-                            debug!(
+                        match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
+                            Ok(true) => {}
+                            Ok(false) => debug!(
                                 event = EVENT_LIFECYCLE_WORKER_STATE,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
                                 bucket = %oi.bucket,
                                 object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
                                 reason = "local_free_version_missing",
-                                "Lifecycle worker could not find transitioned free version locally"
-                            );
+                                "Lifecycle worker found that the exact free-version was already absent"
+                            ),
+                            Err(err) => {
+                                recovery_notify.notify_one();
+                                debug!(
+                                    event = EVENT_LIFECYCLE_WORKER_STATE,
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                    bucket = %oi.bucket,
+                                    object = %oi.name,
+                                    remote_object = %oi.transitioned_object.name,
+                                    remote_version_id = %oi.transitioned_object.version_id,
+                                    tier = %oi.transitioned_object.tier,
+                                    error = ?err,
+                                    reason = "free_version_exact_cleanup_deferred",
+                                    "Lifecycle worker retained the exact free-version for a fenced retry"
+                                );
+                            }
                         }
                     }
                     else {
@@ -1152,8 +1268,8 @@ fn set_recovered_free_version_enqueue_observer(
     RecoveredFreeVersionEnqueueObserverGuard
 }
 
-pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
-    let expiry_state = runtime_sources::expiry_state_handle();
+pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
+    let expiry_state = api.ctx.expiry_state();
     let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
 
     #[cfg(test)]
@@ -2580,8 +2696,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
     }
 
     Some(tokio::spawn(async move {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
+        let expiry_state = api.ctx.expiry_state();
         run_tier_free_version_recovery_loop(
             cancel_token,
             expiry_state,
@@ -6229,9 +6345,18 @@ mod tests {
             rustfs_utils::crypto::hex(old_identity),
         );
         oi.user_defined = Arc::new(metadata.clone());
+        let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
         delete_free_version_remote_object_then(&oi, &manager, {
             let local_delete_calls = Arc::clone(&local_delete_calls);
+            let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
+            let manager = manager.clone();
             move || async move {
+                assert_eq!(
+                    crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+                    1,
+                    "the identity-bound tier lease must span the exact local marker delete"
+                );
+                lease_observed_during_local_delete.store(true, Ordering::Relaxed);
                 local_delete_calls.fetch_add(1, Ordering::Relaxed);
             }
         })
@@ -6239,6 +6364,12 @@ mod tests {
         .expect("matching destination identity should allow idempotent remote cleanup");
         assert_eq!(old_backend.remove_count().await, 1);
         assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
+        assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
+        assert_eq!(
+            crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+            0,
+            "the tier lease should be released after the local marker delete completes"
+        );
 
         let mut single_prefix_metadata = HashMap::new();
         single_prefix_metadata.insert(
@@ -6472,6 +6603,7 @@ mod tests {
         let state = ExpiryState::new();
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6480,6 +6612,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         let err = state
@@ -6620,6 +6753,7 @@ mod tests {
         let state = ExpiryState::new_with_unconsumed_worker_channel(1);
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6628,6 +6762,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         state
@@ -6759,7 +6894,7 @@ mod tests {
         };
 
         assert!(
-            super::enqueue_recovered_free_version(oi).await,
+            super::enqueue_recovered_free_version(&ecstore, oi).await,
             "the resized production worker queue should accept the task"
         );
         stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -6875,12 +7010,12 @@ mod tests {
             .await
             .expect("free-version task should reach the worker");
         tokio::time::timeout(StdDuration::from_secs(30), async {
-            while remote_backend.remove_count().await == 0 {
+            while stats.active_tasks() == 0 {
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("worker should complete remote cleanup before taking the local lock");
+        .expect("worker should mark the cleanup task active before the lock assertion");
         let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
             while stats.active_tasks() != 0 {
                 tokio::task::yield_now().await;
@@ -6889,7 +7024,12 @@ mod tests {
         .await;
         assert!(
             completed_while_locked.is_err(),
-            "local cleanup must wait while a competing object writer owns the namespace lock"
+            "the cleanup task must wait while a competing object writer owns the namespace lock"
+        );
+        assert_eq!(
+            remote_backend.remove_count().await,
+            0,
+            "the remote tuple must not be deleted before the all-physical namespace fence is acquired"
         );
         for disk_path in &disk_paths {
             assert!(
@@ -6900,6 +7040,13 @@ mod tests {
         }
 
         drop(object_lock_guard);
+        tokio::time::timeout(StdDuration::from_secs(30), async {
+            while remote_backend.remove_count().await == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("worker should delete the remote tuple after acquiring the released namespace fence");
         tx.send(None).await.expect("worker stop signal should be delivered");
         worker.await.expect("free-version worker should stop cleanly");
 
@@ -6995,6 +7142,7 @@ mod tests {
             .next()
             .expect("seeded free version should be recoverable");
         let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
+        let ordinary_marker_mod_time = OffsetDateTime::now_utc();
 
         for disk_path in &disk_paths {
             let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7017,7 +7165,7 @@ mod tests {
                     name: object.to_string(),
                     version_id: Some(stale_version_id),
                     deleted: true,
-                    mod_time: Some(OffsetDateTime::now_utc()),
+                    mod_time: Some(ordinary_marker_mod_time),
                     ..Default::default()
                 })
                 .expect("same-ID ordinary marker should replace the stale free version");
@@ -7031,6 +7179,13 @@ mod tests {
             .expect("same-ID ordinary marker metadata should be written");
         }
 
+        assert!(
+            !super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
+                .await
+                .expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
+            "the stale free-version task must not report local cleanup"
+        );
+
         let state = ExpiryState::new();
         let (stats, recovery_notify) = {
             let state = state.read().await;
@@ -11522,7 +11677,7 @@ mod tests {
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
+    async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11530,6 +11685,7 @@ mod tests {
             .expect("mock tier lease should be available")
             .backend_identity();
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "legacy-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -11538,25 +11694,28 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Unknown,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
+        crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
+            .await
+            .expect("legacy unknown journal should remain byte-compatible and persistable");
         let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
             .await
-            .expect_err("unknown journal state must fail before backend IO");
+            .expect_err("legacy unknown journal must be quarantined before backend IO");
 
-        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
         assert_eq!(backend.remove_count().await, 0);
     }
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_deletes_confirmed_exact_provider_token() {
+    async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
             .await
             .expect("mock tier lease should be available");
-        let identity = lease.backend_identity();
         backend
             .set_put_remote_version(Some("provider-version-token".to_string()))
             .await;
@@ -11570,34 +11729,30 @@ mod tests {
             .expect("confirmed remote candidate should be seeded");
         backend.set_remove_failure(true);
         backend.set_reject_non_empty_remote_versions(true);
-        let je = Jentry {
-            obj_name: "remote/object".to_string(),
-            version_id: "provider-version-token".to_string(),
-            tier_name: "WARM".to_string(),
-            backend_identity: Some(identity),
-            version_id_exact: true,
-            version_state: rustfs_filemeta::TransitionVersionState::Exact,
-            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
-            source: None,
-        };
-
-        crate::set_disk::cleanup_rejected_transition_upload_durably(
+        let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
             &lease,
-            &je.obj_name,
-            &je.version_id,
+            "remote/object",
+            "provider-version-token",
             true,
             Some(ecstore.clone()),
         )
         .await
-        .expect("failed immediate cleanup should remain durable in the journal");
-        assert!(backend.contains(&je.obj_name).await);
+        .expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
+        assert_eq!(err.kind(), std::io::ErrorKind::Other);
+        assert!(backend.contains("remote/object").await);
 
         backend.set_remove_failure(false);
-        crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
-            .await
-            .expect("identity-bound exact journal must retry confirmed candidate cleanup");
+        crate::set_disk::cleanup_rejected_transition_upload_durably(
+            &lease,
+            "remote/object",
+            "provider-version-token",
+            true,
+            Some(ecstore),
+        )
+        .await
+        .expect("the transaction retry must delete the same confirmed candidate");
 
-        assert!(!backend.contains(&je.obj_name).await);
+        assert!(!backend.contains("remote/object").await);
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(
             backend.remove_versions().await,
@@ -11760,11 +11915,14 @@ mod tests {
         };
         let mut recovery_rx = recovery_rx.lock().await;
         assert!(
-            super::enqueue_recovered_free_version(ObjectInfo {
-                bucket: "prefill".to_string(),
-                name: "prefill".to_string(),
-                ..Default::default()
-            })
+            super::enqueue_recovered_free_version(
+                &ecstore,
+                ObjectInfo {
+                    bucket: "prefill".to_string(),
+                    name: "prefill".to_string(),
+                    ..Default::default()
+                },
+            )
             .await,
             "the production recovery queue should accept its first task"
         );

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum DurableIlmRecordKind {
     TierDeleteJournal,
+    TierDeleteDispatchManifest,
     TransitionTransaction,
     ManualTransitionJob,
     ManualTransitionScope,
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
     max_record_size: 64 * 1024,
     kind: DurableIlmRecordKind::TierDeleteJournal,
 };
+pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-journal-v6",
+    prefix: "ilm/tier-delete-journal-v6/",
+    max_record_size: 64 * 1024,
+    kind: DurableIlmRecordKind::TierDeleteJournal,
+};
+pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-dispatch-manifest",
+    prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
+    max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
+    kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
+};
 pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
     name: "transition-transaction",
     prefix: "ilm/transition-transactions/records",
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
+    TIER_DELETE_JOURNAL_V6_NAMESPACE,
+    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
     TRANSITION_TRANSACTION_NAMESPACE,
     MANUAL_TRANSITION_JOB_NAMESPACE,
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         content_sha256: String,
         identity_sha256: String,
         committed: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        dispatch_identity_sha256: Option<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        state: Option<super::tier_sweeper::TierDeleteJournalState>,
+    },
+    TierDeleteDispatchManifest {
+        content_sha256: String,
+        identity_sha256: String,
+        state: tier_delete_journal::TierDeleteDispatchManifestState,
     },
     TransitionTransaction {
         content_sha256: String,
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
     pub(crate) fn content_sha256(&self) -> &str {
         match self {
             Self::TierDeleteJournal { content_sha256, .. }
+            | Self::TierDeleteDispatchManifest { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
     }
 
     pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
+        for checkpoint in [self, next] {
+            if let Self::TierDeleteJournal {
+                committed,
+                dispatch_identity_sha256,
+                state,
+                ..
+            } = checkpoint
+                && (state.is_some() != dispatch_identity_sha256.is_some()
+                    || state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
+            {
+                return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
+            }
+        }
         if self == next {
             if let Self::ManualTransitionJob {
                 progress,
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
         let valid = match (self, next) {
             (
                 Self::TierDeleteJournal {
+                    content_sha256: previous_content,
                     identity_sha256: previous_identity,
                     committed: previous_committed,
+                    dispatch_identity_sha256: previous_dispatch_identity,
+                    state: previous_state,
                     ..
                 },
                 Self::TierDeleteJournal {
+                    content_sha256: next_content,
                     identity_sha256: next_identity,
                     committed: next_committed,
+                    dispatch_identity_sha256: next_dispatch_identity,
+                    state: next_state,
                     ..
                 },
             ) => {
+                use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+                let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
+                    (Some(previous), Some(next)) => previous == next,
+                    (None, None) => true,
+                    // Old receipts did not record the v6 dispatch binding. A
+                    // byte-identical observation may adopt the stronger proof,
+                    // but an in-flight mutation must fail closed instead of
+                    // guessing which operation owned the journal.
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                let state_is_monotonic = match (previous_state, next_state) {
+                    (Some(previous), Some(next)) => {
+                        previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
+                    }
+                    (None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
+            }
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: previous_state,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: next_identity,
+                    state: next_state,
+                    ..
+                },
+            ) => {
+                use tier_delete_journal::TierDeleteDispatchManifestState::{
+                    Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
+                };
                 previous_identity == next_identity
-                    && (previous_committed == next_committed || (!previous_committed && *next_committed))
+                    && matches!(
+                        (previous_state, next_state),
+                        (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
+                    )
             }
             (
                 Self::TransitionTransaction {
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
             Err(Error::other("durable ILM record generation is not a monotonic successor"))
         }
     }
+
+    /// Whether `self` is an older generation of the same immutable record
+    /// that can reach `terminal` through one or more valid state transitions.
+    /// This is deliberately broader than `validate_successor`, which remains
+    /// adjacent-only for receipt advancement. Terminal cleanup uses this only
+    /// after the exact terminal ETag and terminal receipt were committed, to
+    /// purge older object versions exposed by that deletion.
+    pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
+        if self == terminal || self.validate_successor(terminal).is_ok() {
+            return true;
+        }
+        match (self, terminal) {
+            (
+                Self::TierDeleteJournal {
+                    identity_sha256: previous_identity,
+                    dispatch_identity_sha256: previous_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
+                    ..
+                },
+                Self::TierDeleteJournal {
+                    identity_sha256: terminal_identity,
+                    dispatch_identity_sha256: terminal_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
+                    ..
+                },
+            ) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: terminal_identity,
+                    state:
+                        tier_delete_journal::TierDeleteDispatchManifestState::Aborted
+                        | tier_delete_journal::TierDeleteDispatchManifestState::Completed,
+                    ..
+                },
+            ) => previous_identity == terminal_identity,
+            _ => false,
+        }
+    }
 }
 
 fn transition_state_distance(
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
             if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
                 return Err(Error::other("tier delete journal content does not match its path"));
             }
-            let operation_id = path
+            let legacy_operation_id = path
                 .strip_prefix(namespace.prefix)
                 .and_then(|suffix| suffix.strip_suffix(".json"))
                 .ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
+            // Legacy v1-v5 paths already expose a 64-hex operation id and
+            // must remain receipt-compatible. V6 uses an operation-scoped
+            // nested path, so derive a fixed, path-unique receipt id instead
+            // of embedding slashes in the receipt locator.
+            let operation_id = if entry.persisted_version == 6 {
+                hex_sha256(path.as_bytes(), ToOwned::to_owned)
+            } else {
+                legacy_operation_id.to_string()
+            };
             let identity_sha256 = checkpoint_hash(&(
                 &entry.obj_name,
                 &entry.version_id,
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 entry.version_state,
                 &entry.source,
             ))?;
+            let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
             (
                 "operation_id",
-                operation_id.to_string(),
+                operation_id,
                 DurableIlmRecordCheckpoint::TierDeleteJournal {
                     content_sha256,
                     identity_sha256,
                     committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
+                    dispatch_identity_sha256,
+                    state: (entry.persisted_version == 6).then_some(entry.state),
+                },
+            )
+        }
+        DurableIlmRecordKind::TierDeleteDispatchManifest => {
+            let (operation_id, identity_sha256, state) =
+                tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
+            (
+                "operation_id",
+                hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
+                DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
+                    content_sha256,
+                    identity_sha256,
+                    state,
                 },
             )
         }
@@ -956,6 +1108,87 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
+        use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
+
+        let operation_id = Uuid::new_v4();
+        let checkpoint = |state| {
+            let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
+            let namespace = classify_durable_ilm_record(&path)
+                .expect("dispatch manifest namespace should classify")
+                .expect("dispatch manifest should be durable");
+            assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
+            validate_durable_ilm_record(&path, &data)
+                .expect("dispatch manifest should validate")
+                .checkpoint
+        };
+
+        let preparing = checkpoint(Preparing);
+        let authorized = checkpoint(DispatchAuthorized);
+        let completed = checkpoint(Completed);
+        let aborting = checkpoint(Aborting);
+        let aborted = checkpoint(Aborted);
+
+        preparing
+            .validate_successor(&authorized)
+            .expect("Preparing may become DispatchAuthorized");
+        authorized
+            .validate_successor(&completed)
+            .expect("DispatchAuthorized may become Completed");
+        preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
+        aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
+        assert!(authorized.validate_successor(&aborting).is_err());
+        assert!(completed.validate_successor(&authorized).is_err());
+        assert!(aborted.validate_successor(&preparing).is_err());
+    }
+
+    #[test]
+    fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
+        use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+        let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: content.repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: state == Some(Committed),
+            dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
+            state,
+        };
+        let prepared = checkpoint("a", Some("d"), Some(Prepared));
+        let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
+        let committed = checkpoint("c", Some("d"), Some(Committed));
+        prepared
+            .validate_successor(&dispatched)
+            .expect("Prepared may advance to Dispatched");
+        dispatched
+            .validate_successor(&committed)
+            .expect("Dispatched may advance to Committed");
+        assert!(prepared.validate_successor(&committed).is_err());
+        assert!(dispatched.validate_successor(&prepared).is_err());
+
+        let rebound = checkpoint("b", Some("e"), Some(Dispatched));
+        assert!(dispatched.validate_successor(&rebound).is_err());
+
+        let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
+            "kind": "tier_delete_journal",
+            "content_sha256": "a".repeat(64),
+            "identity_sha256": "i".repeat(64),
+            "committed": false
+        }))
+        .expect("legacy tier-delete checkpoint should remain decodable");
+        legacy
+            .validate_successor(&prepared)
+            .expect("byte-identical legacy receipt may adopt the stronger v6 proof");
+        let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: "z".repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: false,
+            dispatch_identity_sha256: None,
+            state: None,
+        };
+        assert!(changed_legacy.validate_successor(&prepared).is_err());
+    }
+
     #[test]
     fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
         let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -34,6 +34,6 @@ pub mod tier_sweeper;
 pub mod transition_transaction;
 
 pub(crate) use durable_namespace::{
-    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
-    validate_durable_ilm_record,
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
+    classify_durable_ilm_record, validate_durable_ilm_record,
 };

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
         return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
     }
 
-    let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
+    let page =
+        list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
     let mut stats = FreeVersionRecoveryStats {
         scanned: 0,
         enqueued: 0,
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
             return Err(tier_free_version_recovery_cancelled());
         }
         retry_cursor.visit(&oi);
-        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
+        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
             let (bucket_marker, object_marker) = retry_cursor.retry_markers();
             stats.truncated = true;
             stats.next_bucket_marker = bucket_marker;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -255,6 +255,7 @@ impl ObjSweeper {
         }
         if del_tier {
             return Some(Jentry {
+                persisted_version: 0,
                 obj_name: self.remote_object.clone(),
                 version_id: self.transition_version_id.clone(),
                 tier_name: self.transition_tier.clone(),
@@ -266,6 +267,7 @@ impl ObjSweeper {
                 version_state: self.transition_version_state,
                 state: TierDeleteJournalState::Committed,
                 source: None,
+                dispatch: None,
             });
         }
         None
@@ -298,9 +300,19 @@ impl ObjSweeper {
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 pub(crate) enum TierDeleteJournalState {
     Prepared,
+    Dispatched,
     Committed,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct TierDeleteDispatchBinding {
+    pub(crate) operation_id: Uuid,
+    pub(crate) manifest_object: String,
+    pub(crate) journal_set_sha256: String,
+    pub(crate) topology_generation: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TierDeleteSourceIdentity {
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
 #[derive(Debug, Clone)]
 #[allow(unused_assignments)]
 pub struct Jentry {
+    /// On-disk format version when decoded. Newly constructed entries use 0;
+    /// the encoder chooses their format from the durable ownership fields.
+    /// Recovery uses this value to quarantine v1-v5 without rewriting them.
+    pub(crate) persisted_version: u8,
     pub(crate) obj_name: String,
     pub(crate) version_id: String,
     pub(crate) tier_name: String,
@@ -350,6 +366,23 @@ pub struct Jentry {
     pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
     pub(crate) state: TierDeleteJournalState,
     pub(crate) source: Option<TierDeleteSourceIdentity>,
+    pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
+}
+
+impl Jentry {
+    /// Whether this prepared transaction is eligible to become the sole
+    /// cleanup owner for its transitioned source. The caller may use this to
+    /// decide whether to persist it, but must not set `skip_free_version`
+    /// until persistence succeeds.
+    pub(crate) fn can_replace_tier_free_version(&self) -> bool {
+        self.state == TierDeleteJournalState::Prepared
+            && self.backend_identity.is_some()
+            && self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
+            && self
+                .source
+                .as_ref()
+                .is_some_and(TierDeleteSourceIdentity::has_stable_identity)
+    }
 }
 
 impl ExpiryOp for Jentry {
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
     }
 
     Some(Jentry {
+        persisted_version: 0,
         obj_name: transitioned.name.clone(),
         version_id: transitioned.version_id.clone(),
         tier_name: transitioned.tier.clone(),
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
         version_state: transition_version_state,
         state: TierDeleteJournalState::Committed,
         source: None,
+        dispatch: None,
     })
 }
 
@@ -673,17 +708,73 @@ mod test {
     use rustfs_s3_client::signer_error::invalid_utf8_header_error;
 
     use super::{
-        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
-        RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
-        delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
-        is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
-        should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
+        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
+        RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
+        delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
+        delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
+        is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
+        transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
     };
     use crate::storage_api_contracts::lifecycle::TransitionedObject;
     use rustfs_filemeta::TransitionVersionState;
     use std::io::{Error, ErrorKind};
     use std::time::{Duration, Instant};
 
+    fn stable_prepared_journal() -> Jentry {
+        Jentry {
+            persisted_version: 0,
+            obj_name: "remote/object".to_string(),
+            version_id: "remote-version".to_string(),
+            tier_name: "WARM".to_string(),
+            backend_identity: Some([7; 32]),
+            version_id_exact: true,
+            version_state: TransitionVersionState::Exact,
+            state: TierDeleteJournalState::Prepared,
+            source: Some(TierDeleteSourceIdentity {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: Some(uuid::Uuid::new_v4().to_string()),
+                versioned: true,
+                version_suspended: false,
+                data_dir: None,
+                etag: None,
+                mod_time: None,
+            }),
+            dispatch: None,
+        }
+    }
+
+    #[test]
+    fn only_stable_prepared_journal_can_replace_tier_free_version() {
+        let stable = stable_prepared_journal();
+        assert!(stable.can_replace_tier_free_version());
+
+        let mut committed = stable.clone();
+        committed.state = TierDeleteJournalState::Committed;
+        assert!(!committed.can_replace_tier_free_version());
+
+        let mut unbound = stable.clone();
+        unbound.backend_identity = None;
+        assert!(!unbound.can_replace_tier_free_version());
+
+        let mut unknown = stable.clone();
+        unknown.version_state = TransitionVersionState::Unknown;
+        assert!(!unknown.can_replace_tier_free_version());
+
+        let mut unstable = stable;
+        unstable.source = Some(TierDeleteSourceIdentity {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+            versioned: false,
+            version_suspended: false,
+            data_dir: None,
+            etag: Some("etag-only".to_string()),
+            mod_time: None,
+        });
+        assert!(!unstable.can_replace_tier_free_version());
+    }
+
     #[test]
     fn signer_header_error_detection_matches_utf8_failures() {
         let err = Error::new(

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1136,6 +1136,16 @@ pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Res
     bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
 }
 
+pub(crate) async fn has_authoritative_never_versioned_state_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<bool> {
+    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
+
+    bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
+}
+
 pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
     ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
     ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
     StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
-    tier_mutation_control_service_client::TierMutationControlServiceClient,
+    TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
+    node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
     pub applied: bool,
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("{message}")]
+struct TierMutationDefinitelyRejected {
+    message: String,
+}
+
+fn tier_mutation_definitely_rejected_error(message: String) -> Error {
+    Error::other(TierMutationDefinitelyRejected { message })
+}
+
+#[cfg(test)]
+pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
+    tier_mutation_definitely_rejected_error(message.to_string())
+}
+
+pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
+    matches!(
+        error,
+        Error::Io(io_error)
+            if io_error
+                .get_ref()
+                .is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
+    )
+}
+
 fn validate_tier_mutation_response_proof(
     version: u32,
     phase: TierMutationRpcPhase,
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
     canonical_payload: &[u8],
     response: &TierMutationControlResponse,
 ) -> Result<()> {
+    if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
+        return Err(Error::other("peer tier mutation response proof exceeds size limit"));
+    }
+    if response
+        .error_info
+        .as_ref()
+        .is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
+    {
+        return Err(Error::other("peer tier mutation error response exceeds size limit"));
+    }
     let canonical_response =
         rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
             version,
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
             state: response.state,
             applied: response.applied,
             error_info: response.error_info.as_deref(),
+            failure_class: response.failure_class,
         })
         .map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
     verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
         TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
         TierMutationRpcPhase::Abort => {
             if payload_len == 0 {
-                return Ok(());
+                return Err(Error::other("tier mutation abort payload is empty"));
             }
-            return Err(Error::other("tier mutation abort payload must be empty"));
+            rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
         }
         _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
     };
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
     }
 }
 
-fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
-    Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
+fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
+    let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
+    let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
+    // RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
+    if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && status.code() == tonic::Code::FailedPrecondition
+        && status.message().as_bytes() == legacy_rejection.as_bytes()
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    Error::other(message)
+}
+
+fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
+    let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
+    if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    // Missing/zero, unknown, and explicit Ambiguous are deliberately the same
+    // fail-closed result: the coordinator must include this peer in Abort.
+    Error::other(message)
 }
 
 impl PeerRestClient {
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
             .await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
-        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
+    pub async fn abort_tier_mutation(
+        &self,
+        mutation_id: Uuid,
+        canonical_prepare_payload: Bytes,
+    ) -> Result<PeerTierMutationOutcome> {
+        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
             .await
     }
 
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
                         client
                             .prepare_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Commit => {
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
                         client
                             .commit_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Abort => {
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
                         client
                             .abort_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
                 };
                 validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
                 if !response.success {
-                    return Err(Error::other(
-                        response
-                            .error_info
-                            .unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
-                    ));
+                    return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
+                }
+                if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+                    && response.failure_class != TierMutationFailureClass::Unspecified as i32
+                {
+                    return Err(Error::other("successful peer tier mutation response carried a failure class"));
                 }
                 let state = decode_tier_mutation_peer_state(response.state)?;
                 Ok(PeerTierMutationOutcome {
@@ -3332,6 +3394,7 @@ mod tests {
         state: i32,
         applied: bool,
         error_info: Option<&'a str>,
+        failure_class: i32,
     }
 
     fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3345,6 +3408,7 @@ mod tests {
                 state: input.state,
                 applied: input.applied,
                 error_info: input.error_info,
+                failure_class: input.failure_class,
             })
             .expect("small tier mutation response should encode");
         let response_proof =
@@ -3355,6 +3419,7 @@ mod tests {
             applied: input.applied,
             error_info: input.error_info.map(str::to_string),
             response_proof: response_proof.into(),
+            failure_class: input.failure_class,
         }
     }
 
@@ -3372,6 +3437,7 @@ mod tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
+            failure_class: TierMutationFailureClass::Unspecified as i32,
         });
         validate_tier_mutation_response_proof(
             rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3396,6 +3462,10 @@ mod tests {
                 applied: false,
                 ..response.clone()
             },
+            TierMutationControlResponse {
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+                ..response.clone()
+            },
         ] {
             let err = validate_tier_mutation_response_proof(
                 rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3419,6 +3489,44 @@ mod tests {
         assert!(err.to_string().contains("invalid tier mutation response proof"));
     }
 
+    #[test]
+    fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
+        let mutation_id = Uuid::new_v4();
+        let payload = b"tier-mutation-prepare";
+        let oversized_proof = TierMutationControlResponse {
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: None,
+            response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
+            failure_class: TierMutationFailureClass::Ambiguous as i32,
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_proof,
+        )
+        .expect_err("oversized proof must fail before cryptographic verification");
+        assert!(err.to_string().contains("response proof exceeds size limit"));
+
+        let oversized_error = TierMutationControlResponse {
+            response_proof: Bytes::new(),
+            error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
+            ..oversized_proof
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_error,
+        )
+        .expect_err("oversized error detail must fail before proof construction");
+        assert!(err.to_string().contains("error response exceeds size limit"));
+    }
+
     #[test]
     fn tier_mutation_peer_state_decode_fails_closed() {
         assert_eq!(
@@ -3463,8 +3571,17 @@ mod tests {
             )
             .is_err()
         );
-        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
-        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
+        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
+            .expect("max abort payload should fit");
+        assert!(
+            validate_tier_mutation_payload_len(
+                TierMutationRpcPhase::Abort,
+                rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -3479,7 +3596,7 @@ mod tests {
                 tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
                 tonic::Status::unavailable("peer tier mutation control unavailable"),
             ] {
-                let err = tier_mutation_control_status_error(phase, status);
+                let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
                 let rendered = err.to_string();
                 assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
                 assert!(
@@ -3493,6 +3610,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let exact = format!("unsupported tier mutation peer protocol version: {version}");
+        let rejected = tier_mutation_control_status_error(
+            TierMutationRpcPhase::Prepare,
+            version,
+            tonic::Status::failed_precondition(exact.clone()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for status in [
+            tonic::Status::failed_precondition(format!("{exact}.")),
+            tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
+            tonic::Status::invalid_argument(exact.clone()),
+            tonic::Status::unimplemented(exact),
+        ] {
+            let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "near-text, wrong-code, and Unimplemented failures must remain ambiguous"
+            );
+        }
+    }
+
+    #[test]
+    fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let rejected = tier_mutation_failed_response_error(
+            version,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("rejected".to_string()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for failure_class in [
+            TierMutationFailureClass::Unspecified as i32,
+            TierMutationFailureClass::Ambiguous as i32,
+            99,
+        ] {
+            let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
+            );
+        }
+
+        let v3_ignores_v4_class = tier_mutation_failed_response_error(
+            rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("legacy failure".to_string()),
+        );
+        assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
+    }
+
     #[tokio::test]
     async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
         let client = test_peer_client();

crates/ecstore/src/core/pools.rs

@@ -18,7 +18,8 @@ use crate::bucket::utils::is_meta_bucketname;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::bucket::{
     lifecycle::{
-        DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord,
+        DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
+        ValidatedDurableIlmRecord,
         bucket_lifecycle_audit::LcEventSrc,
         bucket_lifecycle_ops::{
             LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
@@ -26,6 +27,7 @@ use crate::bucket::{
         },
         classify_durable_ilm_record, get_expiry_configs,
         lifecycle::IlmAction,
+        tier_delete_journal::durable_ilm_v6_topology_generation,
         validate_durable_ilm_record,
     },
     metadata_sys,
@@ -49,6 +51,9 @@ use crate::error::{
 use crate::layout::endpoints::EndpointServerPools;
 use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions};
 use crate::runtime::sources as runtime_sources;
+use crate::services::notification_sys::{
+    acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
+};
 use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::{
@@ -2384,10 +2389,12 @@ struct DecommissionDurableIlmReceipt {
     id: String,
     checkpoint: DurableIlmRecordCheckpoint,
     terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    fleet_topology_generation: Option<String>,
 }
 
 impl DecommissionDurableIlmReceipt {
-    fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self {
+    fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
         Self {
             source_path: path.to_string(),
             namespace: record.namespace.to_string(),
@@ -2395,6 +2402,7 @@ impl DecommissionDurableIlmReceipt {
             id: record.id.clone(),
             checkpoint: record.checkpoint.clone(),
             terminal_checkpoint: None,
+            fleet_topology_generation,
         }
     }
 
@@ -2433,6 +2441,16 @@ impl DecommissionDurableIlmReceipt {
                 ))
             })?;
         }
+        if self
+            .fleet_topology_generation
+            .as_deref()
+            .is_some_and(|generation| !is_sha256_checksum(generation))
+        {
+            return Err(Error::other_with_context(
+                "receipt fleet topology generation is invalid",
+                format!("source path `{}` {}", self.source_path, self.context()),
+            ));
+        }
         Ok(())
     }
 
@@ -2541,6 +2559,21 @@ fn merge_decommission_durable_ilm_receipts(
         id: existing.id.clone(),
         checkpoint,
         terminal_checkpoint,
+        fleet_topology_generation: match (
+            existing.fleet_topology_generation.as_ref(),
+            incoming.fleet_topology_generation.as_ref(),
+        ) {
+            (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
+                Some(existing_generation.clone())
+            }
+            (None, None) => None,
+            _ => {
+                return Err(Error::other_with_context(
+                    "durable ILM receipt fleet topology conflict",
+                    format!("source path `{}` {}", existing.source_path, existing.context()),
+                ));
+            }
+        },
     };
     merged.validate()?;
     Ok(merged)
@@ -7050,6 +7083,15 @@ pub(crate) struct DecommissionCapacityOwner {
     pub(crate) mutation_id: Option<uuid::Uuid>,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct DecommissionDurableIlmCheckpointTarget {
+    pub(crate) source_pool_index: usize,
+    pub(crate) target_pool_index: usize,
+    pub(crate) capacity_owner: DecommissionCapacityOwner,
+    pub(crate) already_committed: bool,
+    pub(crate) target_etag: Option<String>,
+}
+
 impl DecommissionCapacityOwner {
     pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
         opts.src_pool_idx = self.source_pool_index;
@@ -7369,51 +7411,18 @@ impl DecommissionPoolCapacityInfo {
             physical_used,
         }
     }
-
-    #[cfg(test)]
-    fn from_logical(pool_index: usize, space: PoolSpaceInfo) -> Self {
-        Self {
-            pool_index,
-            space,
-            layout: DecommissionErasureLayout { data: 1, parity: 0 },
-            physical_free: space.free,
-            physical_total: space.total,
-            physical_used: space.used,
-        }
-    }
 }
 
-#[cfg(test)]
-type DecommissionSpaceInfoOverrides = std::sync::Mutex<HashMap<uuid::Uuid, Vec<(usize, PoolSpaceInfo)>>>;
-
 #[cfg(test)]
 type DecommissionCapacityInfoOverrides =
     std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
 
-#[cfg(test)]
-static DECOMMISSION_SPACE_INFO_OVERRIDES: std::sync::OnceLock<DecommissionSpaceInfoOverrides> = std::sync::OnceLock::new();
-
 #[cfg(test)]
 static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
 
-#[cfg(test)]
-pub(crate) fn set_decommission_space_info_override_for_test(store_id: uuid::Uuid, space_infos: Vec<(usize, PoolSpaceInfo)>) {
-    DECOMMISSION_SPACE_INFO_OVERRIDES
-        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
-        .lock()
-        .expect("decommission space info override should not be poisoned")
-        .insert(store_id, space_infos);
-}
-
-#[cfg(test)]
-fn take_decommission_space_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<(usize, PoolSpaceInfo)>> {
-    DECOMMISSION_SPACE_INFO_OVERRIDES
-        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
-        .lock()
-        .expect("decommission space info override should not be poisoned")
-        .remove(&store_id)
-}
-
+/// Queues capacity snapshots consumed in order by `get_decommission_all_pool_capacity_infos`;
+/// the final snapshot is retained and replayed for every subsequent sample, so tests never
+/// fall back to the host's real disk statistics once an override is installed.
 #[cfg(test)]
 pub(crate) fn set_decommission_capacity_info_overrides_for_test(
     store_id: uuid::Uuid,
@@ -7432,11 +7441,15 @@ fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Op
         .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
         .lock()
         .expect("decommission capacity info override should not be poisoned");
-    let snapshot = overrides.get_mut(&store_id)?.pop_front();
-    if overrides.get(&store_id).is_some_and(std::collections::VecDeque::is_empty) {
-        overrides.remove(&store_id);
+    let queue = overrides.get_mut(&store_id)?;
+    if queue.len() > 1 {
+        queue.pop_front()
+    } else {
+        // The final snapshot is replayed forever: extra sampling points added to
+        // the decommission paths must keep observing injected capacity instead of
+        // silently falling back to the host's real statfs numbers (see #6989).
+        queue.front().cloned()
     }
-    snapshot
 }
 
 #[cfg(test)]
@@ -7493,6 +7506,7 @@ impl DecommissionCapacityLockOrderBarrier {
         Self { state }
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) async fn wait_until_owner_paused(&self) {
         tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
             .await
@@ -7505,6 +7519,7 @@ impl DecommissionCapacityLockOrderBarrier {
             .expect("external mutation should release capacity before waiting for the object namespace");
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
         tokio::time::timeout(
             std::time::Duration::from_secs(30),
@@ -7514,6 +7529,7 @@ impl DecommissionCapacityLockOrderBarrier {
         .expect("external object mutation should acquire its no-active capacity probe");
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
         tokio::time::timeout(
             std::time::Duration::from_secs(30),
@@ -7538,24 +7554,29 @@ impl DecommissionCapacityLockOrderBarrier {
         .expect("external heal should attempt the target namespace lock after capacity admission");
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) fn release_owner(&self) {
         self.state.owner_release.notify_one();
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) fn pause_external_object_commit_phase(&self) {
         self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) fn release_external_object_commit_phase(&self) {
         self.state.external_object_commit_phase_release.notify_one();
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) fn pause_external_object_capacity_probe(&self) {
         self.state
             .external_object_capacity_probe_paused
             .store(true, Ordering::Release);
     }
 
+    #[cfg(feature = "test-util")]
     pub(crate) fn release_external_object_capacity_probe(&self) {
         self.state.external_object_capacity_probe_release.notify_one();
     }
@@ -8054,6 +8075,19 @@ impl ECStore {
         write_state: &mut PoolMetaWriteState,
         operation: &str,
     ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
+        self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
+            .await
+    }
+
+    async fn acquire_pool_meta_write_guard_with_lock_error<F>(
+        &self,
+        write_state: &mut PoolMetaWriteState,
+        operation: &str,
+        map_lock_error: F,
+    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
+    where
+        F: FnOnce(rustfs_lock::LockError) -> Error,
+    {
         write_state.ensure_write_safe(operation)?;
         load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
         let pool = self.pools.first().cloned().ok_or_else(|| {
@@ -8067,7 +8101,7 @@ impl ECStore {
         let pool_meta_guard = pool_meta_lock
             .get_write_lock(get_lock_acquire_timeout())
             .await
-            .map_err(activation_pool_meta_lock_error)?;
+            .map_err(map_lock_error)?;
         let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
         write_state.observe_replicas(selection.replica_state);
         write_state.ensure_write_safe(operation)?;
@@ -8123,6 +8157,27 @@ impl ECStore {
         Ok((pool_meta_guard, has_active_source))
     }
 
+    /// Fence healing of the pool metadata object itself without recursively
+    /// acquiring its namespace lock through the ordinary capacity probe. The
+    /// caller must retain the returned write guard through every admitted
+    /// repair; admission results preserve the input target order.
+    pub(crate) async fn acquire_pool_meta_object_heal_fence(
+        &self,
+        target_pool_indices: &[usize],
+    ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
+        let mut save_guard = self.pool_meta_save_gate.lock().await;
+        let (pool_meta_guard, snapshot) = self
+            .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
+            .await?;
+        let admissions = target_pool_indices
+            .iter()
+            .copied()
+            .map(|target_pool_index| ensure_external_decommission_target_admission(&snapshot, target_pool_index, "heal"))
+            .collect();
+        drop(save_guard);
+        Ok((pool_meta_guard, admissions))
+    }
+
     pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
         &self,
     ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
@@ -8317,6 +8372,122 @@ impl ECStore {
             .await
     }
 
+    /// Finish the capacity transaction for an identity-preserving temporary
+    /// replacement whose target bytes are already durably present. This is
+    /// the crash-recovery half of `run_decommission_capacity_temporary_mutation`:
+    /// it never writes the target again and only resolves a pending intent
+    /// owned by the exact deterministic mutation id.
+    pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
+        &self,
+        owner: DecommissionCapacityOwner,
+        target_pool_index: usize,
+        expected_data_bytes: usize,
+    ) -> Result<()> {
+        let mut save_guard = self.pool_meta_save_gate.lock().await;
+        let (pool_meta_guard, mut snapshot) = self
+            .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
+            .await?;
+        let source_pool_index = owner.source_pool_index;
+        let mutation_id = owner
+            .mutation_id
+            .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
+        let (target_layout, pending_physical_bytes, pending_mutation_id, already_reconciled) = {
+            let reservation = snapshot
+                .pools
+                .get(source_pool_index)
+                .and_then(|pool| pool.decommission.as_ref())
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .filter(|reservation| reservation.admits_cleanup_owner(owner))
+                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
+            let target = reservation
+                .targets
+                .iter()
+                .find(|target| target.pool_index == target_pool_index)
+                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
+            (
+                target.layout,
+                target.pending_physical_bytes,
+                target.pending_mutation_id,
+                target
+                    .temporary_mutations
+                    .iter()
+                    .any(|mutation| mutation.mutation_id == mutation_id),
+            )
+        };
+        if pending_physical_bytes == 0 {
+            // Either the successful attempt already saved its progress, or a
+            // byte-neutral replacement had no inflight delta to record.
+            ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
+            return Ok(());
+        }
+        if pending_mutation_id != Some(mutation_id) {
+            return Err(decommission_capacity_blocked_error(
+                "equivalent temporary target pending intent belongs to another mutation",
+            ));
+        }
+        if already_reconciled {
+            return Err(decommission_capacity_blocked_error(
+                "equivalent temporary target has both pending and reconciled state",
+            ));
+        }
+        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
+        if pending_physical_bytes < expected_target_physical_bytes {
+            return Err(decommission_capacity_blocked_error(
+                "equivalent temporary target pending capacity is smaller than the committed checkpoint",
+            ));
+        }
+        resolve_decommission_target_pending(
+            &mut snapshot,
+            source_pool_index,
+            target_pool_index,
+            expected_target_physical_bytes,
+            mutation_id,
+        )?;
+        // The replacement is byte-non-growing and its exact bytes were read
+        // before this call, so no new physical delta is inferred on replay.
+        // A prior successful progress save would have taken the idempotent
+        // pending==0 return above.
+        record_decommission_target_inflight(
+            &mut snapshot,
+            source_pool_index,
+            target_pool_index,
+            0,
+            mutation_id,
+            OffsetDateTime::now_utc(),
+        )?;
+        let outcome = snapshot
+            .save_no_lock_armed(
+                self.pools.clone(),
+                &mut save_guard,
+                pool_meta_guard.lock_lost_signal(),
+                &[source_pool_index],
+            )
+            .await?;
+        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
+        let persisted_info = outcome
+            .committed
+            .pools
+            .get(source_pool_index)
+            .and_then(|pool| pool.decommission.as_ref())
+            .cloned()
+            .ok_or_else(|| decommission_metadata_not_initialized_error("publish equivalent temporary target reconciliation"))?;
+        {
+            let mut pool_meta = self.pool_meta.write().await;
+            let pool_count = pool_meta.pools.len();
+            pool_meta.version = pool_meta.version.max(outcome.committed.version);
+            let info = pool_meta
+                .pools
+                .get_mut(source_pool_index)
+                .and_then(|pool| pool.decommission.as_mut())
+                .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
+            info.capacity_reservation = persisted_info.capacity_reservation;
+            info.capacity_blocked_reason = persisted_info.capacity_blocked_reason;
+        }
+        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
+        outcome.disarm();
+        Ok(())
+    }
+
     pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
         &self,
         target_pool_index: usize,
@@ -9156,14 +9327,6 @@ impl ECStore {
             return Ok(capacity_infos);
         }
 
-        #[cfg(test)]
-        if let Some(space_infos) = take_decommission_space_info_override_for_test(self.id) {
-            return Ok(space_infos
-                .into_iter()
-                .map(|(pool_index, space)| DecommissionPoolCapacityInfo::from_logical(pool_index, space))
-                .collect());
-        }
-
         let mut capacity_infos = Vec::with_capacity(self.pools.len());
         for idx in 0..self.pools.len() {
             capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?);
@@ -12857,6 +13020,7 @@ impl ECStore {
         let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
         let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
         let mut proof = None::<DecommissionDurableIlmReceipt>;
+        let mut nonterminal_receipt_found = false;
         for pool_idx in 0..self.pools.len() {
             if pool_idx == source_pool_idx {
                 continue;
@@ -12900,23 +13064,35 @@ impl ECStore {
                     receipt.context()
                 )));
             }
-            source_record
-                .checkpoint
-                .validate_successor(&receipt.checkpoint)
-                .map_err(|err| {
-                    Error::other(format!(
-                        "terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}",
-                        source_record.context()
-                    ))
-                })?;
-            if receipt.terminal_checkpoint.is_some() {
+            if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
+                if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
+                    return Err(Error::other_with_context(
+                        "terminal durable ILM decommission receipt does not cover source",
+                        format!("path `{path}` {}", source_record.context()),
+                    ));
+                }
                 proof = Some(match proof {
                     Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
                     None => receipt,
                 });
+            } else {
+                source_record
+                    .checkpoint
+                    .validate_successor(&receipt.checkpoint)
+                    .map_err(|err| {
+                        Error::other_with_context(
+                            "durable ILM decommission receipt does not cover source",
+                            format!("path `{path}` {}: {err}", source_record.context()),
+                        )
+                    })?;
+                nonterminal_receipt_found = true;
             }
         }
-        Ok(proof)
+        // A terminal receipt on one target must not hide another target copy
+        // whose receipt was installed later and has not reached terminal yet.
+        // Returning no proof makes recovery advance every outstanding copy
+        // before source cleanup can treat the operation as complete.
+        if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
     }
 
     async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
@@ -13060,6 +13236,7 @@ impl ECStore {
         pool_idx: usize,
         receipt_path: &str,
         record: &ValidatedDurableIlmRecord,
+        fleet_topology_generation: Option<&str>,
         terminal: bool,
     ) -> Result<bool> {
         let stage = if terminal { "terminal" } else { "progress" };
@@ -13095,6 +13272,12 @@ impl ECStore {
                 ))
             })?;
             Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
+            if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
+                return Err(Error::other_with_context(
+                    "durable ILM decommission receipt fleet topology mismatch",
+                    format!("path `{}` {}", receipt.source_path, receipt.context()),
+                ));
+            }
             receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
                 Error::other(format!(
                     "{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
@@ -13207,6 +13390,7 @@ impl ECStore {
         let stage = if terminal { "terminal" } else { "progress" };
         let record = validate_durable_ilm_record(path, data)
             .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
+        let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
         let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
         let mut terminal_target_pool_indices = Vec::new();
         for (source_pool_idx, run_token) in active_runs {
@@ -13215,7 +13399,13 @@ impl ECStore {
             for pool_idx in 0..self.pools.len() {
                 if pool_idx != source_pool_idx {
                     let found = self
-                        .advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal)
+                        .advance_durable_ilm_decommission_receipt(
+                            pool_idx,
+                            &receipt_path,
+                            &record,
+                            fleet_topology_generation.as_deref(),
+                            terminal,
+                        )
                         .await?;
                     receipt_found |= found;
                     if terminal
@@ -13237,6 +13427,240 @@ impl ECStore {
         Ok(Some(terminal_target_pool_indices))
     }
 
+    /// Resolve the exact receipt-bearing target copies on which a v6 dispatch
+    /// manifest may advance while a decommission reservation is active.
+    ///
+    /// This is intentionally not a general capacity bypass. The target copy
+    /// must still be covered by the active source reservation and its durable
+    /// receipt, the ETag must be the caller's exact CAS generation, and the
+    /// replacement must be a byte-non-growing adjacent manifest checkpoint.
+    pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
+        &self,
+        path: &str,
+        next_data: &[u8],
+        expected_etag: &str,
+    ) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
+        let active_runs = {
+            let pool_meta = self.pool_meta.read().await;
+            let mut active_runs = Vec::new();
+            for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
+                let Some(info) = pool
+                    .decommission
+                    .as_ref()
+                    .filter(|info| info.has_decommission_state() && !info.complete)
+                else {
+                    continue;
+                };
+                let Some(start_time) = info.start_time else {
+                    continue;
+                };
+                let reservation = info.capacity_reservation.clone().ok_or_else(|| {
+                    decommission_capacity_blocked_error(format!(
+                        "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
+                    ))
+                })?;
+                if !reservation.active() {
+                    return Err(decommission_capacity_blocked_error(format!(
+                        "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
+                    )));
+                }
+                active_runs.push((
+                    source_pool_index,
+                    decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
+                    reservation,
+                ));
+            }
+            active_runs
+        };
+        if active_runs.is_empty() {
+            return Ok(None);
+        }
+
+        let next_record = validate_durable_ilm_record(path, next_data)?;
+        if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
+            return Ok(None);
+        }
+        let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
+
+        let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
+        let mut missing_source_receipts = 0usize;
+        let mut stale_target_etag_mismatch = false;
+        for (source_pool_index, run_token, reservation) in active_runs {
+            let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
+            let mut source_receipt_found = false;
+            for allocation in &reservation.targets {
+                let receipt_data = match read_config_limited_preserve_empty(
+                    self.pools[allocation.pool_index].clone(),
+                    &receipt_path,
+                    DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
+                )
+                .await
+                {
+                    Ok(data) => data,
+                    Err(err)
+                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
+                            || is_err_object_not_found(&err)
+                            || is_err_version_not_found(&err) =>
+                    {
+                        continue;
+                    }
+                    Err(err) => return Err(err),
+                };
+                let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
+                let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
+                Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
+                if receipt.source_path != path
+                    || receipt.namespace != next_record.namespace
+                    || receipt.id_kind != next_record.id_kind
+                    || receipt.id != next_record.id
+                {
+                    return Err(Error::other_with_context(
+                        "durable ILM checkpoint receipt identity does not authorize source",
+                        format!("path `{path}` {}", next_record.context()),
+                    ));
+                }
+                if receipt.fleet_topology_generation != next_fleet_topology_generation {
+                    return Err(Error::other_with_context(
+                        "durable ILM checkpoint receipt fleet topology does not authorize source",
+                        format!("path `{path}` {}", next_record.context()),
+                    ));
+                }
+                receipt
+                    .checkpoint
+                    .validate_successor(&next_record.checkpoint)
+                    .map_err(|err| {
+                        Error::other_with_context(
+                            "durable ILM checkpoint receipt is not a predecessor of the requested generation",
+                            format!(
+                                "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
+                                allocation.pool_index,
+                                next_record.context(),
+                                receipt.checkpoint,
+                                next_record.checkpoint
+                            ),
+                        )
+                    })?;
+
+                let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
+                    self.pools[allocation.pool_index].clone(),
+                    path,
+                    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
+                )
+                .await?;
+                let target_record = validate_durable_ilm_record(path, &target_data)?;
+                if target_record.namespace != next_record.namespace
+                    || target_record.id_kind != next_record.id_kind
+                    || target_record.id != next_record.id
+                {
+                    return Err(Error::other_with_context(
+                        "durable ILM checkpoint target identity does not match source",
+                        format!("path `{path}` {}", next_record.context()),
+                    ));
+                }
+                let already_committed = target_data.as_slice() == next_data;
+                let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
+                    Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
+                })?;
+                if already_committed {
+                    receipt
+                        .checkpoint
+                        .validate_successor(&target_record.checkpoint)
+                        .map_err(|err| {
+                            Error::other_with_context(
+                                "durable ILM checkpoint receipt is not a predecessor of the committed target generation",
+                                format!("path `{path}` {}: {err}", next_record.context()),
+                            )
+                        })?;
+                } else {
+                    if target_record.checkpoint != receipt.checkpoint {
+                        return Err(Error::other_with_context(
+                            "durable ILM checkpoint target is not the receipt generation",
+                            format!("path `{path}` {}", next_record.context()),
+                        ));
+                    }
+                    target_record
+                        .checkpoint
+                        .validate_successor(&next_record.checkpoint)
+                        .map_err(|err| {
+                            Error::other_with_context(
+                                "durable ILM checkpoint target is not a predecessor of the requested generation",
+                                format!("path `{path}` {}: {err}", next_record.context()),
+                            )
+                        })?;
+                    if next_data.len() > target_data.len() {
+                        return Err(decommission_capacity_blocked_error(format!(
+                            "durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
+                            target_data.len(),
+                            next_data.len()
+                        )));
+                    }
+                    stale_target_etag_mismatch |= target_etag != expected_etag;
+                }
+
+                source_receipt_found = true;
+                if let Some(existing) = targets
+                    .iter_mut()
+                    .find(|target| target.target_pool_index == allocation.pool_index)
+                {
+                    if existing.already_committed != already_committed {
+                        return Err(Error::other_with_context(
+                            "durable ILM checkpoint target state changed during authorization",
+                            format!("path `{path}`"),
+                        ));
+                    }
+                    continue;
+                }
+                let base_owner = DecommissionCapacityOwner {
+                    source_pool_index,
+                    operation_id: reservation.operation_id,
+                    generation: reservation.generation,
+                    owner_nonce: reservation.owner_nonce,
+                    mutation_id: None,
+                };
+                let mutation_id = decommission_capacity_mutation_id(
+                    base_owner,
+                    RUSTFS_META_BUCKET,
+                    path,
+                    Some(next_record.checkpoint.content_sha256()),
+                    false,
+                    None,
+                );
+                targets.push(DecommissionDurableIlmCheckpointTarget {
+                    source_pool_index,
+                    target_pool_index: allocation.pool_index,
+                    capacity_owner: base_owner.with_mutation_id(mutation_id),
+                    already_committed,
+                    target_etag: Some(target_etag),
+                });
+            }
+            if !source_receipt_found {
+                missing_source_receipts = missing_source_receipts.saturating_add(1);
+            }
+        }
+
+        if missing_source_receipts > 0 {
+            return Err(decommission_capacity_blocked_error(format!(
+                "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
+            )));
+        }
+        if targets.is_empty() {
+            return Err(decommission_capacity_blocked_error(format!(
+                "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
+            )));
+        }
+        if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
+            return Err(Error::PreconditionFailed);
+        }
+        // After a partial multi-target commit, an aggregate read may return
+        // the ETag of the already-advanced target while another authorized
+        // target still has the predecessor ETag. The exact committed bytes
+        // plus every target's receipt/checkpoint proof authorize repairing
+        // that predecessor with its own target-local CAS. Without an exact
+        // committed target, retain the caller-ETag requirement above.
+        targets.sort_unstable_by_key(|target| target.target_pool_index);
+        Ok(Some(targets))
+    }
+
     pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
         self.advance_durable_ilm_decommission_receipts(path, data, false)
             .await
@@ -13258,6 +13682,66 @@ impl ECStore {
         self.advance_durable_ilm_decommission_receipts(path, data, true).await
     }
 
+    /// Return true only when `data` is the exact copy still owned by an active
+    /// decommission source and a target-side terminal receipt authorizes that
+    /// source's later verified cleanup. Lifecycle recovery may then regard the
+    /// logical record as terminal without deleting the source checkpoint.
+    pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
+        let namespace = classify_durable_ilm_record(path)?
+            .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
+        let source_record = validate_durable_ilm_record(path, data)?;
+        let active_runs = {
+            let pool_meta = self.pool_meta.read().await;
+            pool_meta
+                .pools
+                .iter()
+                .enumerate()
+                .filter_map(|(source_pool_index, pool)| {
+                    pool.decommission
+                        .as_ref()
+                        .filter(|info| info.has_decommission_state() && !info.complete)
+                        .and_then(|info| info.start_time)
+                        .map(|start_time| {
+                            (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
+                        })
+                })
+                .collect::<Vec<_>>()
+        };
+        let mut covered_active_source = false;
+        for (source_pool_index, run_token) in active_runs {
+            let source_data =
+                match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
+                    .await
+                {
+                    Ok(source_data) => source_data,
+                    Err(err)
+                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
+                            || is_err_object_not_found(&err)
+                            || is_err_version_not_found(&err) =>
+                    {
+                        continue;
+                    }
+                    Err(err) => return Err(err),
+                };
+            if source_data.as_slice() != data {
+                continue;
+            }
+            if self
+                .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
+                .await?
+                .is_some()
+            {
+                covered_active_source = true;
+            } else {
+                // Every active source that still stores this exact generation
+                // needs complete terminal receipt coverage. One covered source
+                // cannot authorize lifecycle recovery to skip another.
+                return Ok(false);
+            }
+        }
+        Ok(covered_active_source)
+    }
+
     async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
         for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
             match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
@@ -13326,12 +13810,20 @@ impl ECStore {
             .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
         let source_record = validate_durable_ilm_record(path, &source)
             .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
+        let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
         let target = self
             .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
             .await?;
         let manifest_receipt = if let Some((target_pool_idx, target)) = target {
             let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
-            let receipt = DecommissionDurableIlmReceipt::new(path, &target_record);
+            let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
+            if target_fleet_topology_generation != source_fleet_topology_generation {
+                return Err(Error::other_with_context(
+                    "target durable ILM fleet topology generation differs from source",
+                    format!("path `{path}` {}", source_record.context()),
+                ));
+            }
+            let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
             self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
                 .await?;
             receipt
@@ -13345,9 +13837,34 @@ impl ECStore {
                     ))
                 })?
         };
+        if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
+            return Err(Error::other_with_context(
+                "terminal durable ILM receipt fleet topology generation does not cover source",
+                format!("path `{path}` {}", source_record.context()),
+            ));
+        }
+        let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
+            let proof = acquire_tier_delete_journal_fleet_proof()
+                .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
+            if tier_delete_journal_topology_generation(&proof) != expected_generation
+                || !tier_delete_journal_fleet_proof_matches(&proof)
+            {
+                return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
+            }
+            Some(proof)
+        } else {
+            None
+        };
         self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
             .await?;
 
+        if fleet_proof
+            .as_ref()
+            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
+        {
+            return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
+        }
+
         let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
             source_set,
             RUSTFS_META_BUCKET,
@@ -13364,7 +13881,16 @@ impl ECStore {
                 source_record.context()
             ))
         });
-        resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path)
+        let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
+        if fleet_proof
+            .as_ref()
+            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
+        {
+            return Err(Error::other(
+                "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
+            ));
+        }
+        cleanup_result
     }
 
     #[cfg(all(test, feature = "test-util"))]
@@ -13400,7 +13926,26 @@ impl ECStore {
         record: &ValidatedDurableIlmRecord,
         terminal: bool,
     ) -> Result<String> {
-        let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record);
+        let fleet_topology_generation = match read_config_limited_preserve_empty(
+            self.pools[target_pool_idx].clone(),
+            source_path,
+            classify_durable_ilm_record(source_path)?
+                .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
+                .max_record_size,
+        )
+        .await
+        {
+            Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
+            Err(err)
+                if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
+                    || is_err_object_not_found(&err)
+                    || is_err_version_not_found(&err) =>
+            {
+                None
+            }
+            Err(err) => return Err(err),
+        };
+        let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
         if terminal {
             receipt.terminal_checkpoint = Some(record.checkpoint.clone());
         }
@@ -17371,11 +17916,15 @@ mod pools_tests {
             content_sha256: "b".repeat(64),
             identity_sha256: "c".repeat(64),
             committed: false,
+            dispatch_identity_sha256: None,
+            state: None,
         };
         let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
             content_sha256: "d".repeat(64),
             identity_sha256: "c".repeat(64),
             committed: true,
+            dispatch_identity_sha256: None,
+            state: None,
         };
         let incoming = DecommissionDurableIlmReceipt {
             source_path,
@@ -17384,6 +17933,7 @@ mod pools_tests {
             id: operation_id,
             checkpoint: checkpoint.clone(),
             terminal_checkpoint: None,
+            fleet_topology_generation: None,
         };
         let existing = DecommissionDurableIlmReceipt {
             terminal_checkpoint: Some(terminal_checkpoint.clone()),
@@ -17394,7 +17944,24 @@ mod pools_tests {
             .expect("retry receipt must merge with a terminal receipt");
 
         assert_eq!(merged.checkpoint, checkpoint);
-        assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint));
+        assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
+
+        let topology_bound = DecommissionDurableIlmReceipt {
+            fleet_topology_generation: Some("e".repeat(64)),
+            ..incoming
+        };
+        let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
+            .expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
+            .to_string();
+        assert!(mixed_error.contains("fleet topology conflict"));
+
+        let topology_existing = DecommissionDurableIlmReceipt {
+            terminal_checkpoint: Some(terminal_checkpoint),
+            ..topology_bound.clone()
+        };
+        let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
+            .expect("receipts bound to the same fleet topology should merge");
+        assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
     }
 
     #[test]
@@ -17419,7 +17986,7 @@ mod pools_tests {
         assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
         let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
         let expected_checkpoint = record.checkpoint.clone();
-        let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record);
+        let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
         receipt.terminal_checkpoint = Some(record.checkpoint);
 
         let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");

crates/ecstore/src/core/sets.rs

@@ -37,8 +37,9 @@ use crate::{
     runtime::instance::{InstanceContext, bootstrap_ctx},
     runtime::sources as runtime_sources,
     set_disk::{PreparedGetObjectMetadata, SetDisks},
-    store::init_format::{
-        check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
+    store::{
+        RemoteTuplePublicationFence,
+        init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
     },
 };
 use futures::{
@@ -625,6 +626,19 @@ impl Sets {
             .put_object_with_old_current_size(bucket, object, data, opts)
             .await
     }
+
+    pub(crate) async fn put_object_with_old_current_size_for_data_movement(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut PutObjReader,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
+        self.get_disks_by_key(object)
+            .put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
+            .await
+    }
 }
 
 #[async_trait::async_trait]

crates/ecstore/src/data_movement/mod.rs

@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
+use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
     flag.store(false, Ordering::Relaxed);
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 struct DataMovementMultipartAbortBarrierState {
     bucket: String,
     object: String,
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
     release: tokio::sync::Notify,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) struct DataMovementMultipartAbortBarrier {
     state: Arc<DataMovementMultipartAbortBarrierState>,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
     std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
 > = std::sync::OnceLock::new();
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl DataMovementMultipartAbortBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(DataMovementMultipartAbortBarrierState {
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl Drop for DataMovementMultipartAbortBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
     let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
     capacity_owner: Option<DecommissionCapacityOwner>,
 ) -> Result<()> {
     let source = rd.object_info.clone();
-    let _mutation_fence = store
+    let mutation_fence = store
         .acquire_decommission_object_mutation_fence(&bucket, &source.name)
         .await?;
     let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
         op_label,
         None,
         capacity_owner,
-        Some(&_mutation_fence),
+        Some(mutation_fence),
     )
     .await
 }
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
     op_label: &str,
     lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     capacity_owner: Option<DecommissionCapacityOwner>,
-    mutation_fence: Option<&ObjectLockDiagGuard>,
+    mutation_fence: Option<DecommissionFixedReadAnchor>,
 ) -> Result<()> {
+    let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
     let capacity_owner = capacity_owner.map(|owner| {
         let version_id = object_info.version_id.map(|version_id| version_id.to_string());
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
         });
         owner.with_mutation_id(mutation_id)
     });
+    // Capture the exact source/tier identity before any client-paced read, but
+    // defer both the tier lease and source/target write locks to the final
+    // publication. Decommission already owns main's fixed-domain mutation
+    // fence, so reacquiring that domain as a write lock would self-deadlock.
+    let remote_tuple_publication_fence = store
+        .acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
+        .await?;
     let has_part_checksums = object_info
         .parts
         .iter()
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
             }
             let mut cleanup_opts =
                 data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut cleanup_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut cleanup_opts);
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
             }
         }
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
+            .handle_new_multipart_upload_with_pool_idx(
+                &bucket,
+                &object_info.name,
+                &new_multipart_opts,
+                mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
+            )
             .await
         {
             Ok(res) => res,
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut complete_multipart_opts);
             }
+            let remote_tuple_publication_fence = match mutation_fence.take() {
+                Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+                None => remote_tuple_publication_fence,
+            };
             if let Err(err) = store
                 .clone()
-                .complete_multipart_upload_for_data_movement(
-                    (target_pool_idx, mutation_fence),
+                .complete_multipart_upload_for_data_movement_with_publication_fence(
+                    target_pool_idx,
                     &bucket,
                     &object_info.name,
                     &res.upload_id,
                     parts,
                     &complete_multipart_opts,
+                    remote_tuple_publication_fence,
                 )
                 .await
             {
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
             let mut abort_opts =
                 data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut abort_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut abort_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
 
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
-                #[cfg(test)]
+                #[cfg(all(test, feature = "test-util"))]
                 pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
                 let mut abort_opts =
                     data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-                if let Some(fence) = mutation_fence {
-                    fence.add_namespace_lock_fence(&mut abort_opts);
+                if let Some(anchor) = mutation_fence.as_ref() {
+                    anchor.guard().add_namespace_lock_fence(&mut abort_opts);
                 }
                 if let Some(fence) = multipart_mutation_fence.as_ref() {
                     fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
     if let Some(signal) = lock_lost_signal {
         put_opts.add_namespace_lock_lost_signal(signal);
     }
+    let remote_tuple_publication_fence = match mutation_fence.take() {
+        Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+        None => remote_tuple_publication_fence,
+    };
     let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
+        .put_object_for_data_movement_with_publication_fence(
+            &bucket,
+            &object_info.name,
+            &mut data,
+            &put_opts,
+            remote_tuple_publication_fence,
+        )
         .await
         .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
     if let Err(err) = put_result {

crates/ecstore/src/disk/disk_store.rs

@@ -2435,6 +2435,25 @@ mod tests {
         assert_eq!(window.acc_time, 18_000);
     }
 
+    #[test]
+    fn timed_action_slot_snapshot_skips_writer_owned_slot() {
+        let slot = TimedActionSlot::default();
+        slot.unix_sec.store(70, Ordering::Relaxed);
+        slot.count.store(2, Ordering::Relaxed);
+        slot.acc_time.store(18_000, Ordering::Relaxed);
+        slot.version.store(2, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
+
+        assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
+        slot.unix_sec.store(71, Ordering::Relaxed);
+        slot.count.store(1, Ordering::Relaxed);
+        slot.acc_time.store(11_000, Ordering::Relaxed);
+        assert_eq!(slot.snapshot(), None);
+
+        slot.version.store(4, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
+    }
+
     #[test]
     fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
         let metrics = DiskHealthMetricEpoch::default();

crates/ecstore/src/disk/local.rs

@@ -7092,6 +7092,9 @@ impl LocalDisk {
                                 .await?
                         {
                             meta.name.push_str(SLASH_SEPARATOR);
+                            // Conservative listings verify physical prefixes. Never-versioned
+                            // buckets use the bounded fast path and reclaim residue after an
+                            // exact recursive listing proves that prefix empty.
                             if opts.recursive
                                 || opts.incl_deleted
                                 || opts.skip_hidden_prefix_check
@@ -17619,7 +17622,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_scan_dir_nonrecursive_visible_prefix_probe_cost() {
+    async fn test_scan_dir_nonrecursive_fast_path_preserves_probe_bound() {
         use rustfs_filemeta::MetacacheReader;
         use tempfile::tempdir;
 
@@ -17651,6 +17654,10 @@ mod test {
             expected_names.push(format!("{prefix}/"));
         }
 
+        fs::create_dir_all(bucket_dir.join("stale/nested/residue"))
+            .await
+            .expect("stale backing directory should be created");
+
         async fn scan_prefixes(disk: &LocalDisk, bucket: &str, skip_hidden_prefix_check: bool) -> (Vec<String>, usize) {
             let probe_count = Arc::new(AtomicUsize::new(0));
             let (reader, mut writer) = tokio::io::duplex(64 * 1024);
@@ -17693,8 +17700,11 @@ mod test {
         let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
 
         assert_eq!(conservative_names, expected_names);
-        assert_eq!(fast_path_names, expected_names);
-        assert_eq!(conservative_probes, PREFIX_COUNT * 3);
+        let mut expected_fast_path_names = expected_names.clone();
+        expected_fast_path_names.push("stale/".to_owned());
+        assert_eq!(fast_path_names, expected_fast_path_names);
+        let expected_probes = PREFIX_COUNT * 3 + 3;
+        assert_eq!(conservative_probes, expected_probes);
         assert_eq!(fast_path_probes, 0);
     }
 

crates/ecstore/src/disk/os.rs

@@ -298,6 +298,39 @@ pub(crate) mod windows_rename_test_hooks {
     }
 }
 
+/// Test-only hooks into the destination-parent walk of rename preparation.
+///
+/// The prune race lives between two syscalls inside
+/// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
+/// point reproduces it deterministically. Hooks are keyed by the absolute path
+/// of the component just opened and queued per path: a retrying preparation
+/// visits the same component again, so a test models a pruner that keeps
+/// walking upward by queueing one hook per visit.
+#[cfg(all(test, unix))]
+pub(crate) mod prepare_rename_test_hooks {
+    use super::*;
+
+    type Hook = Box<dyn FnOnce() + Send>;
+
+    static AFTER_COMPONENT_OPENED: LazyLock<Mutex<HashMap<PathBuf, VecDeque<Hook>>>> =
+        LazyLock::new(|| Mutex::new(HashMap::new()));
+
+    pub(crate) fn queue_after_component_opened(path: &Path, hook: impl FnOnce() + Send + 'static) {
+        AFTER_COMPONENT_OPENED
+            .lock()
+            .entry(path.to_path_buf())
+            .or_default()
+            .push_back(Box::new(hook));
+    }
+
+    pub(crate) fn run_after_component_opened(path: &Path) {
+        let hook = AFTER_COMPONENT_OPENED.lock().get_mut(path).and_then(VecDeque::pop_front);
+        if let Some(hook) = hook {
+            hook();
+        }
+    }
+}
+
 /// Fsync a directory so recently created or renamed entries survive power loss.
 /// No-op on non-Unix platforms where directories cannot be opened for syncing.
 pub fn fsync_dir_std(dir: impl AsRef<Path>) -> io::Result<()> {
@@ -1905,8 +1938,8 @@ pub(crate) async fn rename_all_with_prepared_source(
         let base_dir = base_dir.clone();
         move || {
             validate_prepared_rename_source(&prepared_source, &src_file_path)?;
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2008,8 +2041,8 @@ async fn reliable_rename_inner_with_lease(
         let dst_file_path = dst_file_path.clone();
         let base_dir = base_dir.clone();
         move || {
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2233,12 +2266,13 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
     let mut attempt = 0;
     loop {
         match prepare_rename(src_file_path, dst_file_path, base_dir, publication_root) {
-            Ok(preparation) => return Ok((preparation, attempt)),
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Ok(preparation) => return Ok(preparation),
+            Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                 attempt += 1;
             }
             Err(err) => return Err(err),
@@ -2252,23 +2286,26 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
     let source_parent = src_file_path
         .parent()
         .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
     let destination_parent = dst_file_path.parent();
-    let mut attempt = 0;
-    let prepare_destination_parent = |attempt: &mut usize| -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
+    // The destination walk and the source open below keep separate counters:
+    // exhausting one must not deny the other its own retry.
+    let prepare_destination_parent = || -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+        let mut attempt = 0;
         loop {
             let result = destination_parent
                 .map(|parent| mkdir_all_below_existing_base_std(parent, base_dir, publication_root))
                 .transpose();
             match result {
                 Ok(parent_guard) => break Ok(parent_guard),
-                Err(err) if should_retry_rename(&err, *attempt) => {
+                Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                     #[cfg(test)]
                     windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                    *attempt += 1;
+                    attempt += 1;
                 }
                 Err(err) => break Err(err),
             }
@@ -2281,7 +2318,7 @@ fn prepare_rename_with_retry(
         None => false,
     };
     let (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity) = if same_parent {
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         let source_parent_guard = parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?
@@ -2293,16 +2330,17 @@ fn prepare_rename_with_retry(
         let source_parent_guard = lock_windows_directory_tree(source_parent, destination_parent, publication_root)?;
         let (source_identity_anchor, expected_source_identity) =
             open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity)
     };
+    let mut source_attempt = 0;
     let source = loop {
         match open_windows_rename_source(src_file_path, &source_parent_guard) {
             Ok(source) => break source,
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Err(err) if should_retry_rename(&err, source_attempt) => {
                 #[cfg(test)]
                 windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                attempt += 1;
+                source_attempt += 1;
             }
             Err(err) => return Err(err),
         }
@@ -2315,14 +2353,11 @@ fn prepare_rename_with_retry(
     }
     drop(source_identity_anchor);
 
-    Ok((
-        RenamePreparation {
-            parent_guard,
-            _source_parent_guard: source_parent_guard,
-            source,
-        },
-        attempt,
-    ))
+    Ok(RenamePreparation {
+        parent_guard,
+        _source_parent_guard: source_parent_guard,
+        source,
+    })
 }
 
 #[cfg(not(windows))]
@@ -2339,24 +2374,22 @@ fn prepare_rename(
     Ok(RenamePreparation { parent_guard })
 }
 
-fn rename_prepared(
-    _src_file_path: &Path,
-    dst_file_path: &Path,
-    preparation: &RenamePreparation,
-    attempt: usize,
-) -> io::Result<()> {
+/// Publish a prepared rename. The retry budget starts fresh here: preparation
+/// keeps its own counter, so a chain rebuilt after a concurrent prune must not
+/// cost the rename its one retry.
+fn rename_prepared(_src_file_path: &Path, dst_file_path: &Path, preparation: &RenamePreparation) -> io::Result<()> {
     #[cfg(windows)]
     {
         let parent_guard = preparation
             .parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?;
-        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, attempt)
+        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, 0)
     }
 
     #[cfg(not(windows))]
     {
-        let mut attempt = attempt;
+        let mut attempt = 0;
         loop {
             let rename_result = rename_into_existing_parent(_src_file_path, dst_file_path, preparation.parent_guard.as_ref());
             match rename_result {
@@ -3756,6 +3789,8 @@ pub(crate) fn mkdir_all_below_existing_base_std(
         let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
         let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
 
+        #[cfg(test)]
+        let mut walked_path = base_dir.to_path_buf();
         for component in relative.components() {
             let Component::Normal(component) = component else {
                 continue;
@@ -3769,6 +3804,11 @@ pub(crate) fn mkdir_all_below_existing_base_std(
                 Err(err) => return Err(err.into()),
             }
             parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
+            #[cfg(test)]
+            {
+                walked_path.push(component);
+                prepare_rename_test_hooks::run_after_component_opened(&walked_path);
+            }
         }
 
         Ok(parents)
@@ -3866,11 +3906,51 @@ fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base
 /// cleanup renames (e.g. `move_to_trash` on an already-removed tmp path) a
 /// pointless second syscall. This predicate is shared by the `rename_data`
 /// commit path via `rename_all`, so any relaxation here must keep genuine
-/// transient errors retryable.
+/// transient errors retryable. The *preparation* phase deliberately uses
+/// [`should_retry_prepare`] instead — see there for why `NotFound` is
+/// recoverable while the destination parent chain is still being built.
 fn should_retry_rename(err: &io::Error, attempt: usize) -> bool {
     attempt == 0 && err.kind() != io::ErrorKind::NotFound
 }
 
+/// How many times rename preparation may retry a `NotFound`.
+///
+/// A pruning walk (`LocalDisk::delete_file`) removes empty ancestors
+/// monotonically upward and stops at the volume root, so it can invalidate
+/// each component *below* the base at most once. One attempt per such
+/// component therefore outlasts a pruning walk, and concurrent walks only
+/// steal an attempt by making that same upward progress. A destination whose
+/// parent *is* the base gets a budget of zero, keeping `NotFound` immediately
+/// terminal for speculative cleanup renames.
+fn prepare_prune_budget(dst_file_path: &Path, base_dir: &Path) -> usize {
+    dst_file_path
+        .parent()
+        .and_then(|parent| parent.strip_prefix(base_dir).ok())
+        .map(|relative| relative.components().count())
+        .unwrap_or(0)
+}
+
+/// Whether a failed rename *preparation* attempt (building the destination
+/// parent chain) should be retried.
+///
+/// Unlike [`should_retry_rename`], `NotFound` is recoverable here: a concurrent
+/// delete prunes now-empty parent directories, so it can unlink an intermediate
+/// destination component between this walk opening a directory and creating the
+/// next child inside it, which a handle-relative `mkdirat`/`openat` reports as
+/// `NotFound`. Each retry rebuilds the whole chain from the base directory,
+/// which no walk below it can remove; `prune_budget` bounds how far a pruner
+/// can push the walk back. A genuinely missing base directory fails identically
+/// on every attempt — the base is only ever opened, never created — so the
+/// missing-base contract holds at the cost of a few extra syscalls on an
+/// already-failing path.
+fn should_retry_prepare(err: &io::Error, attempt: usize, prune_budget: usize) -> bool {
+    if err.kind() == io::ErrorKind::NotFound {
+        attempt < prune_budget
+    } else {
+        attempt == 0
+    }
+}
+
 pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
     let mut i = 0;
 
@@ -4401,6 +4481,141 @@ mod tests {
         assert!(!should_retry_rename(&denied, 1));
     }
 
+    #[test]
+    fn prepare_retry_budget_covers_every_prunable_component() {
+        // A pruner can invalidate each component below the base once, so the
+        // budget must match the chain depth, not a fixed count.
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "pruned");
+        assert!(should_retry_prepare(&not_found, 0, 2));
+        assert!(should_retry_prepare(&not_found, 1, 2));
+        assert!(!should_retry_prepare(&not_found, 2, 2));
+    }
+
+    #[test]
+    fn prepare_retry_keeps_other_errors_at_a_single_retry() {
+        // Only a prune produces a recoverable NotFound; everything else keeps
+        // the historical single retry so persistent failures stay cheap.
+        let denied = io::Error::new(io::ErrorKind::PermissionDenied, "denied");
+        assert!(should_retry_prepare(&denied, 0, 3));
+        assert!(!should_retry_prepare(&denied, 1, 3));
+    }
+
+    #[test]
+    fn prepare_retry_budget_is_zero_when_the_parent_is_the_base() {
+        // Speculative cleanup renames (move_to_trash) land directly in their
+        // base, so a NotFound there is a missing base: terminal, not a prune.
+        let base = Path::new("/vol");
+        assert_eq!(prepare_prune_budget(Path::new("/vol/entry"), base), 0);
+        assert_eq!(prepare_prune_budget(Path::new("/vol/data-movement/sha/id/xl.meta"), base), 3);
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "missing base");
+        assert!(!should_retry_prepare(&not_found, 0, 0));
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_concurrent_empty_parent_prune() {
+        // A multipart staging cleanup prunes the momentarily empty shared
+        // `data-movement/` prefix while a concurrent upload publishes its
+        // xl.meta below that same prefix. The writer's walk holds an fd to the
+        // pruned component, so its next handle-relative mkdirat fails
+        // NotFound; preparation must rebuild the chain and still publish.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let pruned = shared.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            // The cleanup chain's empty-parent prune lands after the writer
+            // opened the shared component but before it creates its child.
+            std::fs::remove_dir(&pruned).expect("prune the empty shared prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a concurrently pruned intermediate directory must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_a_prune_walking_up_every_shared_component() {
+        // The multipart data-movement chain has TWO shared components below the
+        // volume (`data-movement/` and the per-object `<sha>/`), so one cleanup
+        // walk pruning upward can invalidate the writer twice: once at <sha>,
+        // then again at data-movement while the writer rebuilds. A budget that
+        // covers only a single component would still break write quorum here.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let movement = base.join("data-movement");
+        let sha = movement.join("sha");
+        std::fs::create_dir_all(&sha).expect("create shared chain");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = sha.join("upload-id").join("xl.meta");
+
+        // First visit of `data-movement` is the writer's initial walk, which the
+        // pruner has not reached yet; it prunes on the writer's rebuild.
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, || {});
+        let pruned_sha = sha.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&sha, move || {
+            std::fs::remove_dir(&pruned_sha).expect("prune the empty per-object prefix");
+        });
+        let pruned_movement = movement.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, move || {
+            std::fs::remove_dir(&pruned_movement).expect("prune the empty data-movement prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a prune walking up the whole shared chain must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_rejects_a_symlink_swapped_in_between_prepare_attempts() {
+        // The retry must not become a traversal window: replacing the pruned
+        // component with a symlink out of the volume before the rebuilt walk
+        // reopens it must fail closed, exactly as a symlink staged before the
+        // first attempt does.
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        std::fs::create_dir_all(&outside).expect("create outside target");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let swapped = shared.clone();
+        let outside_target = outside.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            std::fs::remove_dir(&swapped).expect("prune the shared prefix");
+            symlink(&outside_target, &swapped).expect("replace the pruned component with a symlink");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect_err("a symlink swapped in between attempts must not be followed");
+
+        assert!(src.exists(), "rejected publish must preserve the staged source");
+        assert!(
+            !outside.join("sha").exists(),
+            "the rebuilt walk must not create or publish through the replacement symlink"
+        );
+    }
+
     #[test]
     fn is_dir_not_empty_error_recognizes_directory_not_empty_kind() {
         let err = io::Error::from(io::ErrorKind::DirectoryNotEmpty);

crates/ecstore/src/error/conversion_roundtrip_tests.rs

@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
     assert_eq!(count, 3);
     assert_eq!(err, Some(DiskError::other("can not get client")));
 }
+
+#[test]
+fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
+    let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
+    let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
+
+    assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
+    let StorageError::Io(io_error) = first else {
+        panic!("stable context must remain an io error");
+    };
+    assert_eq!(io_error.to_string(), "tier mutation intent changed");
+    let context = io_error.get_ref().expect("stable context must remain downcastable");
+    assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
+}

crates/ecstore/src/error/mod.rs

@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;
 
+/// Keeps high-cardinality diagnostic detail in the error source while making
+/// the rendered `io::Error` stable for quorum aggregation.
+#[derive(Debug)]
+struct StableIoContextError {
+    message: &'static str,
+    source: Box<dyn std::error::Error + Send + Sync>,
+}
+
+impl std::fmt::Display for StableIoContextError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(self.message)
+    }
+}
+
+impl std::error::Error for StableIoContextError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(self.source.as_ref())
+    }
+}
+
+pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
+where
+    E: Into<Box<dyn std::error::Error + Send + Sync>>,
+{
+    std::io::Error::other(StableIoContextError {
+        message,
+        source: source.into(),
+    })
+}
+
 /// Storage layer error type covering disk, volume, bucket, object, multipart,
 /// erasure-coding, and operational error conditions.
 ///
@@ -264,6 +294,13 @@ impl StorageError {
         StorageError::Io(std::io::Error::other(error))
     }
 
+    pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        StorageError::Io(stable_io_error(message, source))
+    }
+
     pub fn is_not_found(&self) -> bool {
         matches!(
             self,

crates/ecstore/src/object_api/types.rs

@@ -301,36 +301,23 @@ pub enum LifecycleDeleteAllPhase {
 #[doc(hidden)]
 #[derive(Default)]
 pub struct LifecycleDeleteAllJournalState {
-    prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
     mutation_started: bool,
 }
 
 impl Debug for LifecycleDeleteAllJournalState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LifecycleDeleteAllJournalState")
-            .field("prepared_count", &self.prepared.len())
             .field("mutation_started", &self.mutation_started)
             .finish()
     }
 }
 
 impl LifecycleDeleteAllJournalState {
-    pub(crate) fn contains(&self, name: &str) -> bool {
-        self.prepared.contains_key(name)
-    }
-
-    pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
-        self.prepared.insert(name, entry);
-    }
-
-    pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
-        self.prepared.values().cloned().collect()
-    }
-
     pub(crate) fn mark_mutation_started(&mut self) {
         self.mutation_started = true;
     }
 
+    #[cfg(test)]
     pub(crate) fn mutation_started(&self) -> bool {
         self.mutation_started
     }
@@ -706,6 +693,12 @@ pub struct ObjectOptions {
     pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
     #[doc(hidden)]
     pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
+    /// Whole-operation authorization created only by consuming a validated
+    /// v6 dispatch-manifest permit. Clones share the authorization, not the
+    /// one-shot permit itself.
+    #[doc(hidden)]
+    pub tier_delete_dispatch_authorization:
+        Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
     /// RustFS-only compare-and-set condition checked under the object write lock.
     pub expected_current_version_id: Option<String>,
     /// Persisted bucket incarnation observed before authorization.
@@ -847,6 +840,7 @@ impl std::fmt::Debug for ObjectOptions {
             .field("version_id", &self.version_id.is_some())
             .field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
             .field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
+            .field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
             .field("expected_current_version_id", &self.expected_current_version_id.is_some())
             .field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
             .field("no_lock", &self.no_lock)
@@ -972,7 +966,7 @@ impl ObjectOptions {
     }
 
     #[cfg(test)]
-    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+    pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
         self.namespace_lock_fence
             .get_or_insert_with(NamespaceLockFence::new)
             .extend(fence);

crates/ecstore/src/services/notification_sys.rs

@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
+use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
-use std::sync::{Arc, Mutex, OnceLock};
+use std::sync::{
+    Arc, Mutex, OnceLock,
+    atomic::{AtomicBool, AtomicUsize, Ordering},
+};
 use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
 const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
+const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
+type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
+
+fn cross_pool_fence_policy_results(
+    peer_epochs: BTreeMap<String, Uuid>,
+    minimum_version: u32,
+) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
+    let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
+        Ok(peer_epochs.clone())
+    } else {
+        Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
+    };
+    (Ok(peer_epochs), journal_result)
+}
 
 #[derive(Clone, Debug)]
 pub struct ScannerPublicationLeaseGrant {
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
     topology_fingerprint: String,
     peer_epochs: Arc<BTreeMap<String, Uuid>>,
     expires_at: Instant,
+    generation: Arc<FleetCapabilityProofGeneration>,
 }
 
 impl FleetCapabilityProof {
+    fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
+        Self {
+            topology_fingerprint,
+            peer_epochs,
+            expires_at,
+            generation: FleetCapabilityProofGeneration::fresh(),
+        }
+    }
+
     fn token(&self) -> FleetCapabilityProofToken {
         FleetCapabilityProofToken {
             topology_fingerprint: self.topology_fingerprint.clone(),
             peer_epochs: self.peer_epochs.clone(),
         }
     }
+
+    #[cfg(any(test, feature = "test-util"))]
+    fn with_fresh_generation(&self) -> Self {
+        Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
+    }
+}
+
+/// Admission generation for effects that must not straddle a fleet-proof
+/// replacement. Revocation is deliberately non-blocking: it closes admission
+/// immediately, while the proof slot withholds the successor generation until
+/// every admitted operation has drained.
+#[derive(Default)]
+struct FleetCapabilityProofGeneration {
+    accepting: AtomicBool,
+    active: AtomicUsize,
+}
+
+impl FleetCapabilityProofGeneration {
+    fn fresh() -> Arc<Self> {
+        Arc::new(Self {
+            accepting: AtomicBool::new(true),
+            active: AtomicUsize::new(0),
+        })
+    }
+
+    fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
+        if !self.accepting.load(Ordering::Acquire) {
+            return None;
+        }
+        self.active.fetch_add(1, Ordering::AcqRel);
+        if self.accepting.load(Ordering::Acquire) {
+            Some(FleetCapabilityProofPermit {
+                generation: Arc::clone(self),
+            })
+        } else {
+            self.release();
+            None
+        }
+    }
+
+    fn revoke(&self) {
+        self.accepting.store(false, Ordering::Release);
+    }
+
+    fn is_accepting(&self) -> bool {
+        self.accepting.load(Ordering::Acquire)
+    }
+
+    fn is_drained(&self) -> bool {
+        self.active.load(Ordering::Acquire) == 0
+    }
+
+    fn release(&self) {
+        let previous = self.active.fetch_sub(1, Ordering::AcqRel);
+        debug_assert!(previous > 0, "fleet capability permit count underflow");
+    }
+}
+
+struct FleetCapabilityProofPermit {
+    generation: Arc<FleetCapabilityProofGeneration>,
+}
+
+impl Drop for FleetCapabilityProofPermit {
+    fn drop(&mut self) {
+        self.generation.release();
+    }
 }
 
 #[derive(Clone, PartialEq, Eq)]
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
 #[derive(Default)]
 struct FleetCapabilityProofState {
     proof: Option<FleetCapabilityProof>,
+    draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
     topology_conflict: bool,
 }
 
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
 #[derive(Clone, PartialEq, Eq)]
 pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
 
+/// A point-in-time proof that every current storage member implements the v6
+/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
+/// one acquisition authorizes one manifest construction attempt.
+pub(crate) struct TierDeleteJournalFleetProofToken {
+    token: FleetCapabilityProofToken,
+    _permit: FleetCapabilityProofPermit,
+}
+
 static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
+static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
     REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
-fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
-    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
+    TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
+}
+
+fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
+    if state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        state.draining_generation = None;
+    }
+}
+
+fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    revoke_fleet_capability_proof_state(&mut state);
+}
+
+fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.topology_conflict = true;
+    revoke_fleet_capability_proof_state(&mut state);
 }
 
 fn publish_fleet_capability_probe_result(
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
     match result {
         Ok(peer_epochs) => {
             let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-            let peer_epochs = state
+            if let Some(current) = state
                 .proof
-                .as_ref()
+                .as_mut()
                 .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
-                .map(|proof| Arc::clone(&proof.peer_epochs))
-                .unwrap_or_else(|| Arc::new(peer_epochs));
-            state.proof = Some(FleetCapabilityProof {
-                topology_fingerprint: topology_fingerprint.to_string(),
-                peer_epochs,
-                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
-            });
+            {
+                current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
+                return None;
+            }
+
+            if let Some(previous) = state.proof.take() {
+                previous.generation.revoke();
+                if !previous.generation.is_drained() {
+                    state.draining_generation = Some(previous.generation);
+                }
+            }
+            if state
+                .draining_generation
+                .as_ref()
+                .is_some_and(|generation| generation.is_drained())
+            {
+                state.draining_generation = None;
+            }
+            if state.draining_generation.is_some() {
+                return Some(Error::other(
+                    "fleet capability proof successor waits for the previous generation to drain",
+                ));
+            }
+            state.proof = Some(FleetCapabilityProof::new(
+                topology_fingerprint.to_string(),
+                Arc::new(peer_epochs),
+                observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            ));
             None
         }
         Err(err) => {
-            replace_fleet_capability_proof(slot, None);
+            revoke_fleet_capability_proof(slot);
             Some(err)
         }
     }
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
     fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
 }
 
-#[cfg(test)]
+pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_tier_delete_journal_fleet_proof_from(
+    state: &FleetCapabilityProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<TierDeleteJournalFleetProofToken> {
+    let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
+    let permit = state.proof.as_ref()?.generation.try_acquire()?;
+    Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
+}
+
+pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn tier_delete_journal_fleet_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &TierDeleteJournalFleetProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof._permit.generation.is_accepting()
+        && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
+        && state
+            .proof
+            .as_ref()
+            .is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
+}
+
+pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
+    stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
+        || state
+            .draining_generation
+            .as_ref()
+            .is_some_and(|generation| !generation.is_drained())
+}
+
+fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
+    hasher.update(topology_fingerprint.as_bytes());
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
         .get()
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let now = Instant::now();
+    let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
+        state.proof.clone()
+    } else {
+        Some(FleetCapabilityProof::new(
+            topology,
+            Arc::new(BTreeMap::new()),
+            now + Duration::from_secs(60 * 60),
+        ))
+    };
     state.topology_conflict = false;
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: topology,
-        peer_epochs: Arc::new(BTreeMap::new()),
-        expires_at: Instant::now() + Duration::from_secs(60 * 60),
-    });
+    state.proof = proof.clone();
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    debug_assert!(
+        journal_state
+            .proof
+            .as_ref()
+            .is_none_or(|current| current.generation.is_drained())
+    );
+    journal_state.topology_conflict = false;
+    journal_state.draining_generation = None;
+    journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
 }
 
 #[cfg(test)]
 pub(crate) struct CrossPoolFenceFleetProofGuard {
     previous_proof: Option<FleetCapabilityProof>,
     previous_topology_conflict: bool,
+    previous_journal_proof: Option<FleetCapabilityProof>,
+    previous_journal_topology_conflict: bool,
 }
 
 #[cfg(test)]
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
         let mut state = cross_pool_fence_fleet_proof_slot()
             .write()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
-        state.proof = self.previous_proof.take();
+        state.proof = self
+            .previous_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        state.draining_generation = None;
         state.topology_conflict = self.previous_topology_conflict;
+        drop(state);
+        let mut journal_state = tier_delete_journal_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        journal_state.proof = self
+            .previous_journal_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        journal_state.draining_generation = None;
+        journal_state.topology_conflict = self.previous_journal_topology_conflict;
     }
 }
 
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
     let guard = CrossPoolFenceFleetProofGuard {
         previous_proof: state.proof.clone(),
         previous_topology_conflict: state.topology_conflict,
+        previous_journal_proof: journal_state.proof.clone(),
+        previous_journal_topology_conflict: journal_state.topology_conflict,
     };
-    state.proof = None;
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
     state.topology_conflict = true;
+    if let Some(proof) = journal_state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            journal_state.draining_generation = Some(proof.generation);
+        }
+    }
+    journal_state.topology_conflict = true;
     guard
 }
 
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
     let Some(current) = state.proof.as_ref() else {
         return false;
     };
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: current.topology_fingerprint.clone(),
-        peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
-        expires_at: current.expires_at,
-    });
+    let proof = FleetCapabilityProof::new(
+        current.topology_fingerprint.clone(),
+        Arc::new(current.peer_epochs.as_ref().clone()),
+        current.expires_at,
+    );
+    state.proof = Some(proof.clone());
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    journal_state.topology_conflict = false;
+    if let Some(previous) = journal_state.proof.take() {
+        previous.generation.revoke();
+        if !previous.generation.is_drained() {
+            journal_state.draining_generation = Some(previous.generation);
+        }
+    }
+    if journal_state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        journal_state.draining_generation = None;
+    }
+    if journal_state.draining_generation.is_none() {
+        journal_state.proof = Some(proof.with_fresh_generation());
+    }
     true
 }
 
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
         return false;
     };
     let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
-    if state.topology_conflict {
-        return false;
-    }
-    state.proof.as_ref().is_some_and(|current| {
-        current.topology_fingerprint == *expected_topology
-            && current.topology_fingerprint == proof.topology_fingerprint
-            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
-            && Instant::now() < current.expires_at
-    })
+    fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn fleet_capability_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &FleetCapabilityProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    !state.topology_conflict
+        && state.proof.as_ref().is_some_and(|current| {
+            current.topology_fingerprint == expected_topology
+                && current.topology_fingerprint == proof.topology_fingerprint
+                && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+                && now < current.expires_at
+        })
 }
 
 fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
 #[cfg(test)]
 impl Drop for RemoteVersionStateFleetProofGuard {
     fn drop(&mut self) {
-        replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
+        revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
     }
 }
 
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
 pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
     if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
         if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
-            for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
-                let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-                state.topology_conflict = true;
-                state.proof = None;
+            for slot in [
+                remote_version_state_fleet_proof_slot(),
+                cross_pool_fence_fleet_proof_slot(),
+                tier_delete_journal_fleet_proof_slot(),
+            ] {
+                mark_fleet_capability_topology_conflict(slot);
             }
         }
         return;
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 }
                 None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
             };
-            let fence_result = match get_global_notification_sys() {
+            let fence_probe = match get_global_notification_sys() {
                 Some(notification_sys) => timeout(
                     REMOTE_VERSION_STATE_PROBE_TIMEOUT,
                     notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
                 None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
             };
+            let (fence_result, journal_result) = match fence_probe {
+                Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
+                Err(err) => {
+                    let message = err.to_string();
+                    (Err(Error::other(message.clone())), Err(Error::other(message)))
+                }
+            };
             let topology_conflict = remote_version_state_fleet_proof_slot()
                 .read()
                 .unwrap_or_else(std::sync::PoisonError::into_inner)
                 .topology_conflict;
             if topology_conflict {
-                replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
-                replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
+                revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
+                revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
+                revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                     event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_NOTIFICATION,
-                    capability = "cross_pool_fence_v2",
+                    capability = "cross_pool_fence",
+                    state = "failed_closed",
+                    error = %err,
+                    "notification capability probe"
+                );
+            }
+            if !topology_conflict
+                && let Some(err) = publish_fleet_capability_probe_result(
+                    tier_delete_journal_fleet_proof_slot(),
+                    &topology_fingerprint,
+                    journal_result,
+                    Instant::now(),
+                )
+            {
+                debug!(
+                    event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    capability = "tier_delete_journal_v6_policy",
                     state = "failed_closed",
                     error = %err,
                     "notification capability probe"
@@ -483,7 +811,7 @@ impl NotificationSys {
         Ok(peer_epochs)
     }
 
-    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
         if self.peer_clients.len() != self.peer_topology_hosts.len() {
             return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
         }
@@ -494,14 +822,21 @@ impl NotificationSys {
             client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
         });
         let mut peer_epochs = BTreeMap::new();
+        let mut minimum_version = u32::MAX;
         for result in join_all(probes).await {
             let (peer, version, epoch) = result?;
             if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
                 return Err(Error::other("cross-pool fence capability version is unsupported"));
             }
+            minimum_version = minimum_version.min(version);
             insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
         }
-        Ok(peer_epochs)
+        // A single-node deployment has no remote member to lower the local
+        // policy version advertised by this binary.
+        if minimum_version == u32::MAX {
+            minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
+        }
+        Ok((peer_epochs, minimum_version))
     }
 }
 
@@ -1827,12 +2162,13 @@ impl NotificationSys {
         join_all(futures).await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
+    pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter().cloned() {
+            let payload = canonical_prepare_payload.clone();
             futures.push(async move {
                 if let Some(client) = client {
-                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
+                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
                 } else {
                     unreachable_notification_peer_err()
                 }
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
+        let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
+        let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
+        assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
+        assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
+
+        let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
+        assert!(generic_v3.is_ok());
+        assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
         let now = Instant::now();
         let mut peer_epochs = BTreeMap::new();
         peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(peer_epochs),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
         assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
@@ -2495,11 +2839,7 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_accepts_single_node_membership() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::new()),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
     }
@@ -2507,21 +2847,97 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new(
+            "topology-a".to_string(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
         let captured = proof.token();
-        let restarted = FleetCapabilityProof {
-            topology_fingerprint: proof.topology_fingerprint.clone(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: proof.expires_at,
-        };
+        let restarted = FleetCapabilityProof::new(
+            proof.topology_fingerprint.clone(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            proof.expires_at,
+        );
 
         assert!(captured != restarted.token());
     }
 
+    #[test]
+    fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
+        let topology = "topology-a";
+        let now = Instant::now();
+        let node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let node_b_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let restarted_node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+
+        let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
+            .map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
+        assert_eq!(generations[0], generations[1]);
+        assert_eq!(generations[0], generations[2]);
+        assert_ne!(
+            generations[0],
+            stable_tier_delete_journal_topology_generation("topology-b"),
+            "a real topology change must produce a different durable generation"
+        );
+    }
+
+    #[test]
+    fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+        let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
+
+        let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
+                .is_none()
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        let fresh = state
+            .proof
+            .as_ref()
+            .expect("restart probe should publish a fresh proof")
+            .token();
+
+        assert!(!fleet_capability_proof_matches_at(
+            &state,
+            &original,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert!(fleet_capability_proof_matches_at(
+            &state,
+            &fresh,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert_eq!(
+            original_generation,
+            stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
         let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
@@ -2561,15 +2977,106 @@ mod tests {
         assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
     }
 
+    #[test]
+    fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(
+                tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+                "a freshly admitted journal proof must remain current"
+            );
+            assert!(
+                acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
+                    .is_none(),
+                "TTL expiry must stop new admission"
+            );
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(
+                    &state,
+                    &admitted,
+                    "topology-a",
+                    now + REMOTE_VERSION_STATE_PROOF_TTL,
+                ),
+                "TTL expiry must also stop an admitted proof at its next durable fence"
+            );
+            assert!(!admitted._permit.generation.is_drained());
+        }
+
+        let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let blocked = publish_fleet_capability_probe_result(
+            &slot,
+            "topology-a",
+            Ok(restarted_peers.clone()),
+            now + Duration::from_millis(1),
+        )
+        .expect("a successor proof must wait for the admitted generation");
+        assert!(blocked.to_string().contains("previous generation to drain"));
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
+            assert!(state.draining_generation.is_some());
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
+                "a restarted peer must revoke an admitted proof before its next durable fence"
+            );
+        }
+
+        drop(admitted);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
+                .is_none(),
+            "the successor may publish after the in-flight operation releases its permit"
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.proof.is_some());
+        assert!(state.draining_generation.is_none());
+    }
+
+    #[test]
+    fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+
+        mark_fleet_capability_topology_conflict(&slot);
+
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.topology_conflict);
+        assert!(state.proof.is_none());
+        assert!(state.draining_generation.is_some());
+        assert!(!admitted._permit.generation.is_accepting());
+        assert!(
+            !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+            "topology conflict must revoke an already admitted journal proof"
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
         let now = Instant::now();
         let mut state = FleetCapabilityProofState {
-            proof: Some(FleetCapabilityProof {
-                topology_fingerprint: "topology-a".to_string(),
-                peer_epochs: Arc::new(BTreeMap::new()),
-                expires_at: now + Duration::from_secs(1),
-            }),
+            proof: Some(FleetCapabilityProof::new(
+                "topology-a".to_string(),
+                Arc::new(BTreeMap::new()),
+                now + Duration::from_secs(1),
+            )),
+            draining_generation: None,
             topology_conflict: false,
         };
         assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
@@ -3279,7 +3786,7 @@ mod tests {
         assert_eq!(commit.len(), 1);
         assert!(commit[0].err.is_some());
 
-        let abort = sys.abort_tier_mutation(mutation_id).await;
+        let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
         assert_eq!(abort.len(), 1);
         assert!(abort[0].err.is_some());
     }

crates/ecstore/src/services/rebalance/control.rs

@@ -1329,8 +1329,9 @@ mod tests {
     use super::*;
     use crate::config::com::delete_config;
     use crate::core::pools::{
-        POOL_META_NAME, PoolActivationDurableSaveBarrier, PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState,
-        persist_pool_meta_identity_for_startup,
+        DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
+        PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
+        set_decommission_capacity_info_overrides_for_test,
     };
     use crate::object_api::NamespaceLockFence;
     use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
@@ -1751,26 +1752,15 @@ mod tests {
         ];
         set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
         set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
-        crate::core::pools::set_decommission_space_info_override_for_test(
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        let capacity_snapshot = vec![
+            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
+            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
+        ];
+        // Decommission start samples capacity before and inside its durable activation fence.
+        set_decommission_capacity_info_overrides_for_test(
             decommission_store.id,
-            vec![
-                (
-                    0,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 0,
-                        total: 100,
-                        used: 100,
-                    },
-                ),
-                (
-                    1,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 200,
-                        total: 200,
-                        used: 0,
-                    },
-                ),
-            ],
+            vec![capacity_snapshot.clone(), capacity_snapshot],
         );
         let (first_object, competing_object, competing_kind) = match paused_kind {
             PoolActivationStartKind::Rebalance => {

crates/ecstore/src/services/rebalance/entry.rs

@@ -356,7 +356,7 @@ impl ECStore {
             };
             run_guard.ensure_held("rebalance version migration")?;
             let result = migrate_entry_version(
-                &RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
+                &RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
                 bucket.clone(),
                 pool_index,
                 version,
@@ -1478,6 +1478,7 @@ mod tests {
         let (_temp_dirs, store, _unused_store) =
             crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
         prepare_rebalance_test_volumes(store.as_ref()).await;
+        crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
         let source_set = store.pools[0].get_disks_by_key(object);
         let target_set = store.pools[1].get_disks_by_key(object);
         let version_id = uuid::Uuid::new_v4();
@@ -1520,14 +1521,17 @@ mod tests {
         let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
         let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
         let barrier = TieredMetadataCommitBarrier::install(bucket, object);
-        let task = spawn_real_rebalance_entry(
+        let mut task = spawn_real_rebalance_entry(
             Arc::clone(&store),
             Arc::clone(&source_set),
             entry,
             REBALANCE_ID,
             Arc::new(RebalanceBucketConfigs::default()),
         );
-        barrier.wait_until_paused().await;
+        tokio::select! {
+            _ = barrier.wait_until_paused() => {}
+            result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
+        }
         run_signal_fence.mark_lost();
         barrier.release();
         drop(barrier);

crates/ecstore/src/services/rebalance/migration.rs

@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
 
 pub(crate) struct RebalanceMigrationBackend<'a> {
     source: &'a SetDisks,
-    store: &'a ECStore,
+    store: std::sync::Arc<ECStore>,
     lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 }
 
 impl<'a> RebalanceMigrationBackend<'a> {
     pub(crate) fn new(
         source: &'a SetDisks,
-        store: &'a ECStore,
+        store: std::sync::Arc<ECStore>,
         lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     ) -> Self {
         Self {

crates/ecstore/src/services/rebalance/mod.rs

@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
     test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
     rebalance_meta: Option<RebalanceMeta>,
 ) -> (

crates/ecstore/src/services/tier/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub mod test_util;
 pub mod tier;
 pub mod tier_admin;

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
 use serde::{Deserialize, Serialize};
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
 pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
 pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
+pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
 const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
 pub(crate) type TierMutationDigest = [u8; 32];
 
+static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
+    LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
+
+/// Serializes every local phase and recovery action for one mutation id while
+/// retaining bounded parallelism for unrelated mutations.
+pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
+    TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
+        .lock()
+        .await
+}
+
+fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
+    let raw = mutation_id.as_u128();
+    let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
+    // MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
+    // or relying on RandomState, whose seed differs between processes.
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
+    mixed ^= mixed >> 33;
+    (mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
+}
+
 pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
 
 #[derive(Debug, thiserror::Error)]
@@ -265,6 +290,30 @@ impl TierMutationIntent {
             && self.expires_at_unix_nanos == other.expires_at_unix_nanos
     }
 
+    /// Reconstruct the exact Prepared record that originally produced this
+    /// intent. Abort RPCs are identity-bound to that payload; serializing an
+    /// Aborted terminal record would both violate the wire contract and use a
+    /// different revision if a missing peer has to persist a tombstone.
+    pub(crate) fn original_prepared(&self) -> Result<Self> {
+        if self.state == TierMutationIntentState::Prepared {
+            self.validate()?;
+            return Ok(self.clone());
+        }
+        let mut prepared = self.clone();
+        prepared.revision =
+            prepared
+                .revision
+                .checked_sub(1)
+                .filter(|revision| *revision != 0)
+                .ok_or(TierMutationIntentError::Corrupt(
+                    "terminal intent cannot reconstruct its prepared revision",
+                ))?;
+        prepared.state = TierMutationIntentState::Prepared;
+        prepared.committed_config_etag = None;
+        prepared.validate()?;
+        Ok(prepared)
+    }
+
     pub(crate) fn encode(&self) -> Result<Vec<u8>> {
         self.validate()?;
         let intent_bytes = serde_json::to_vec(self)?;
@@ -439,6 +488,16 @@ where
     load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
+pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+) -> EcstoreResult<(TierMutationIntent, String)>
+where
+    S: EcstoreObjectIO,
+{
+    load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
 async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
     api: Arc<S>,
     prefix: &str,
@@ -507,6 +566,7 @@ where
     .await
 }
 
+#[cfg(test)]
 pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -514,13 +574,36 @@ where
     delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
-pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
+pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
 {
-    delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+    delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
+        .await
 }
 
+pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    delete_tier_mutation_intent_record_if_current_with_prefix(
+        api,
+        TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
+        mutation_id,
+        current_etag,
+    )
+    .await
+}
+
+#[cfg(test)]
 async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -533,6 +616,40 @@ where
     }
 }
 
+async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    if current_etag.trim().is_empty() {
+        return Err(Error::other("tier mutation intent current ETag is empty"));
+    }
+    let object =
+        tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
+    match api
+        .delete_object(
+            RUSTFS_META_BUCKET,
+            &object,
+            ObjectOptions {
+                http_preconditions: Some(HTTPPreconditions {
+                    if_match: Some(current_etag.to_string()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
+        Err(err) => Err(err),
+    }
+}
+
 pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
     api: Arc<S>,
     mutation_id: Uuid,
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::time::Duration;
 
     const OLD_IDENTITY: TierDestinationId = [1; 32];
     const NEW_IDENTITY: TierDestinationId = [2; 32];
@@ -736,6 +854,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn mutation_mutex_uses_exactly_64_stable_shards() {
+        assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
+        assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
+
+        let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
+        let shard = tier_mutation_mutex_shard_index(mutation_id);
+        assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
+        assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_serializes_the_same_id() {
+        let mutation_id = Uuid::new_v4();
+        let first = acquire_tier_mutation_mutex(mutation_id).await;
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            started_tx.send(()).expect("test receiver should remain alive");
+            let _second = acquire_tier_mutation_mutex(mutation_id).await;
+            acquired_tx.send(()).expect("test receiver should remain alive");
+        });
+        started_rx.await.expect("waiter should start");
+        assert!(
+            tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
+                .await
+                .is_err(),
+            "the same mutation id must not enter concurrently"
+        );
+
+        drop(first);
+        tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
+            .await
+            .expect("waiter should acquire after release")
+            .expect("waiter should report acquisition");
+        waiter.await.expect("waiter task should finish");
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_allows_different_shards_to_progress() {
+        let first_id = Uuid::new_v4();
+        let first_shard = tier_mutation_mutex_shard_index(first_id);
+        let second_id = (0..1024)
+            .map(|_| Uuid::new_v4())
+            .find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
+            .expect("a distinct shard should be easy to find");
+        let first = acquire_tier_mutation_mutex(first_id).await;
+
+        let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
+            .await
+            .expect("a different shard must not wait for the first mutation");
+        drop(first);
+    }
+
     #[test]
     fn intent_round_trip_preserves_committed_state() {
         let mut intent = prepared_intent();
@@ -873,6 +1046,37 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn terminal_intent_reconstructs_original_prepared_abort_payload() {
+        for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
+            let original = prepared_intent();
+            let mut intent = original.clone();
+            let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
+            intent
+                .advance(terminal, committed_etag)
+                .expect("terminal transition should succeed");
+
+            let reconstructed = intent
+                .original_prepared()
+                .expect("terminal record should recover prepared payload");
+            assert_eq!(reconstructed, original);
+            assert!(intent.same_identity_as(&reconstructed));
+        }
+    }
+
+    #[test]
+    fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
+        let mut corrupt = prepared_intent();
+        corrupt.state = TierMutationIntentState::Aborted;
+
+        assert!(matches!(
+            corrupt.original_prepared(),
+            Err(TierMutationIntentError::Corrupt(
+                "terminal intent cannot reconstruct its prepared revision"
+            ))
+        ));
+    }
+
     #[test]
     fn intent_validation_rejects_placeholder_identity() {
         let mut intent = prepared_intent();

crates/ecstore/src/services/tier/tier_mutation_peer.rs

@@ -15,12 +15,13 @@
 use std::sync::Arc;
 
 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
+use time::OffsetDateTime;
 use uuid::Uuid;
 
 use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
 use super::tier_mutation_intent::{
-    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
-    load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
+    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
+    advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
 };
 use crate::error::{Error, StorageError};
 use crate::store::ECStore;
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
     CommitProofMismatch,
     #[error("tier mutation peer abort proof does not match the persisted tier configuration")]
     AbortProofMismatch,
+    #[error("tier mutation peer prepared intent has expired")]
+    ExpiredIntent,
     #[error("tier mutation peer runtime error: {0}")]
     Runtime(#[source] AdminError),
     #[error("tier mutation peer store error: {0}")]
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
     validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
+    let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
     match phase {
         TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
         TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
@@ -103,43 +107,57 @@ async fn handle_prepare(
     }
     let tier_config_mgr = api.tier_config_mgr();
 
-    match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
-        Ok(()) => {
-            TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
-                .await
-                .map_err(TierMutationPeerError::Runtime)?;
-            Ok(TierMutationPeerOutcome {
-                state: TierMutationPeerState::Prepared,
-                applied: true,
-            })
-        }
-        Err(Error::PreconditionFailed) => {
-            let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
-            if !existing.same_identity_as(&intent) {
-                return Err(TierMutationPeerError::ConflictingIntent);
+    for _ in 0..3 {
+        let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&intent) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                (existing, false)
             }
-            match existing.state {
-                TierMutationIntentState::Prepared => {
-                    TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
+            Err(Error::ConfigNotFound) => {
+                let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
+                if intent.expires_at_unix_nanos <= now {
+                    return Err(TierMutationPeerError::ExpiredIntent);
                 }
-                TierMutationIntentState::Committed => {
-                    TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
-                }
-                TierMutationIntentState::Aborted => {
-                    TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
+                    Ok(()) => (intent.clone(), true),
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
                 }
             }
-            Ok(TierMutationPeerOutcome {
-                state: peer_state_from_intent(existing.state),
-                applied: false,
-            })
+            Err(err) => return Err(err.into()),
+        };
+
+        match stored.state {
+            TierMutationIntentState::Prepared => {
+                TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Committed => {
+                TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Aborted => {
+                TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+            }
         }
-        Err(err) => Err(err.into()),
+        return Ok(TierMutationPeerOutcome {
+            state: peer_state_from_intent(stored.state),
+            applied,
+        });
     }
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation prepare raced repeatedly with another decision",
+    )))
 }
 
 async fn handle_commit(
@@ -201,26 +219,106 @@ async fn handle_abort(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
-    if !canonical_payload.is_empty() {
-        return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
+    let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+    if prepared.state != TierMutationIntentState::Prepared {
+        return Err(TierMutationPeerError::InvalidPayload(
+            "abort payload must carry the original prepared intent".to_string(),
+        ));
     }
-    let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
-    if existing.state == TierMutationIntentState::Prepared
-        && !tier_config_abort_matches(api.clone(), &existing)
-            .await
-            .map_err(Error::other)?
-    {
-        return Err(TierMutationPeerError::AbortProofMismatch);
+    let mut tombstone = prepared.clone();
+    tombstone
+        .advance(TierMutationIntentState::Aborted, None)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+
+    for _ in 0..3 {
+        match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&prepared) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                match existing.state {
+                    TierMutationIntentState::Committed => {
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Committed,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Aborted => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Prepared => {}
+                }
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                let advanced = advance_tier_mutation_intent_record_idempotent(
+                    api.clone(),
+                    mutation_id,
+                    TierMutationIntentState::Aborted,
+                    None,
+                )
+                .await;
+                let (intent, applied) = match advanced {
+                    Ok(result) => result,
+                    Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+                        Ok(current)
+                            if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
+                        {
+                            (current, false)
+                        }
+                        _ => return Err(err.into()),
+                    },
+                };
+                if intent.state == TierMutationIntentState::Aborted {
+                    TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                    TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                        .await
+                        .map_err(TierMutationPeerError::Runtime)?;
+                }
+                return Ok(TierMutationPeerOutcome {
+                    state: peer_state_from_intent(intent.state),
+                    applied,
+                });
+            }
+            Err(Error::ConfigNotFound) => {
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
+                    Ok(()) => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: true,
+                        });
+                    }
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
+                }
+            }
+            Err(err) => return Err(err.into()),
+        }
     }
-    let (intent, applied) =
-        advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
-    if intent.state == TierMutationIntentState::Aborted {
-        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
-    }
-    Ok(TierMutationPeerOutcome {
-        state: peer_state_from_intent(intent.state),
-        applied,
-    })
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation abort raced repeatedly with prepare",
+    )))
 }
 
 fn validate_peer_request_envelope(
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<()> {
-    if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
+    if !matches!(
+        protocol_version,
+        rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
+    ) {
         return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
     }
     if mutation_id.is_nil() {
@@ -276,6 +377,8 @@ mod tests {
     #[test]
     fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
         let mutation_id = Uuid::new_v4();
+        validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
+            .expect("v3 must remain accepted during the v4 rollout");
         assert!(matches!(
             validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
             Err(TierMutationPeerError::UnsupportedProtocolVersion(_))

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -41,22 +41,22 @@ use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
 #[cfg(test)]
 use super::super::get_metadata_slowtail_fault_delay;
 use super::super::{
-    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
-    EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion, GetCodecStreamingFallbackReason,
-    GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource,
-    LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions,
-    ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq,
-    ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError,
-    UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
-    capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason,
-    current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info,
-    inline_erasure_shard_file_offset, inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found,
-    is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
-    is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
-    is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
-    merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
-    reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
-    try_read_inline_data_shards_direct, warn,
+    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+    EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion,
+    GetCodecStreamingFallbackReason, GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult,
+    HealChannelPriority, HealRequestSource, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext,
+    OBJECT_OP_IGNORED_ERRS, ObjectOptions, ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET,
+    RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks,
+    SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs,
+    can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
+    collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
+    file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
+    inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
+    is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
+    is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
+    issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
+    object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
+    send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
 };
 #[cfg(test)]
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
@@ -3733,16 +3733,34 @@ fn dangling_delete_grace() -> time::Duration {
 /// Result of scanning one disk's copy of a directory prefix while deciding
 /// whether an orphan (metadata-less) directory tree can be safely purged.
 enum OrphanDirScan {
-    /// The subtree holds at least one regular file (object metadata or data), so
-    /// it is a real object and must not be purged.
+    /// The subtree holds object metadata or uncommitted data, so it must not be
+    /// purged.
     HasData,
-    /// The prefix exists on this disk and contains only nested empty directories.
-    /// Carries every directory path in pre-order (parents before children).
-    Empty(Vec<String>),
+    /// The prefix contains only empty directories and/or UUID data directories
+    /// carrying a committed delete marker.
+    Purgeable {
+        empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    },
     /// The prefix does not exist on this disk.
     Missing,
 }
 
+fn is_safe_orphan_dir_entry(entry: &str) -> bool {
+    let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
+    !component.is_empty()
+        && component != "."
+        && component != ".."
+        && !component.contains(SLASH_SEPARATOR)
+        && !component.contains('\\')
+}
+
+fn is_committed_delete_marker(entry: &str) -> bool {
+    entry
+        .strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
+        .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil()))
+}
+
 /// Outcome of a *post-quorum* `rename_data` commit, classifying whether the
 /// committed replicas converged so the caller can decide heal admission
 /// WITHOUT conflating "a version signature exists" with "this write needs
@@ -6125,52 +6143,151 @@ impl SetDisks {
     }
 
     /// Scan a single disk's copy of `prefix` and decide whether it is an orphan
-    /// (metadata-less) directory subtree. Walks the tree iteratively and returns
-    /// [`OrphanDirScan::HasData`] as soon as any regular file is found.
+    /// directory subtree. Only empty directories and UUID data directories with
+    /// valid committed delete markers are purgeable; every child is still scanned.
     async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
         let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
         let mut stack = vec![root.clone()];
         // Pre-order list of directories (a parent always precedes its descendants),
         // so reversing it yields a safe children-first removal order.
         let mut dirs: Vec<String> = Vec::new();
+        let mut committed_files: Vec<String> = Vec::new();
         let mut existed = false;
 
         while let Some(dir) = stack.pop() {
             let entries = match disk.list_dir("", bucket, &dir, 0).await {
                 Ok(entries) => entries,
-                Err(_) => {
-                    // The root missing (or never existing) means there is nothing to
-                    // purge on this disk. A nested directory vanishing mid-scan is a
-                    // benign race, so skip it and keep walking.
+                Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {
                     if dir == root {
                         return OrphanDirScan::Missing;
                     }
+                    // A nested directory vanishing mid-scan is a benign race.
                     continue;
                 }
+                // Classification must fail closed: committed residue is safe to
+                // remove only after every reachable child was inspected.
+                Err(_) => return OrphanDirScan::HasData,
             };
 
             existed = true;
-            dirs.push(dir.clone());
+            let mut child_dirs = Vec::new();
+            let mut files = Vec::new();
 
             for entry in entries {
+                if !is_safe_orphan_dir_entry(&entry) {
+                    return OrphanDirScan::HasData;
+                }
                 match entry.strip_suffix(SLASH_SEPARATOR) {
-                    // `read_dir` marks directories with a trailing slash; anything else
-                    // is a regular file, which means real object data lives here.
-                    Some(child) => stack.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
-                    None => return OrphanDirScan::HasData,
+                    Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
+                    None => files.push(entry),
                 }
             }
+
+            if !files.is_empty() {
+                let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
+                let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
+                let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
+
+                if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
+                    return OrphanDirScan::HasData;
+                }
+
+                committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
+                dirs.push(dir);
+                stack.extend(child_dirs);
+                continue;
+            }
+
+            dirs.push(dir);
+            stack.extend(child_dirs);
         }
 
         if existed {
-            OrphanDirScan::Empty(dirs)
+            OrphanDirScan::Purgeable {
+                empty_dirs: dirs,
+                committed_files,
+            }
         } else {
             OrphanDirScan::Missing
         }
     }
 
+    async fn delete_purgeable_orphan_entries(
+        disk: &DiskStore,
+        bucket: &str,
+        object: &str,
+        mut empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    ) {
+        // Keep every committed marker until all ordinary residue files are gone.
+        // If any delete fails, a later request can still recognize and retry the
+        // committed cleanup instead of stranding an unmarked partial residue.
+        for delete_markers in [false, true] {
+            for file in &committed_files {
+                let is_marker = file.rsplit(SLASH_SEPARATOR).next().is_some_and(is_committed_delete_marker);
+                if is_marker != delete_markers {
+                    continue;
+                }
+                if let Err(err) = disk
+                    .delete(
+                        bucket,
+                        file,
+                        DeleteOptions {
+                            recursive: false,
+                            immediate: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                {
+                    debug!(
+                        event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_SET_DISK,
+                        bucket,
+                        object,
+                        path = file,
+                        error = ?err,
+                        "Orphan prefix purge skipped"
+                    );
+                    return;
+                }
+            }
+        }
+        empty_dirs.reverse();
+        for dir in empty_dirs {
+            if let Err(err) = disk
+                .delete(
+                    bucket,
+                    &dir,
+                    DeleteOptions {
+                        recursive: false,
+                        immediate: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                // Best effort: a sibling removal may have already cleared a shared
+                // parent, or a concurrent writer repopulated the directory. Neither
+                // is fatal to purging the orphan tree.
+                debug!(
+                    event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object,
+                    path = dir,
+                    error = ?err,
+                    "Orphan prefix purge skipped"
+                );
+            }
+        }
+    }
+
     /// Purge an orphan directory prefix — a trailing-slash key that exists on disk
-    /// as an empty directory tree with no object metadata on any disk of this set.
+    /// as empty directories or committed delete residue, with no object metadata
+    /// or uncommitted data on any disk of this set.
     /// Such prefixes are listable (see `scan_dir`) yet are not real objects, so the
     /// normal delete path returns NotFound and leaves them stranded (issue #4189).
     ///
@@ -6187,15 +6304,18 @@ impl SetDisks {
         // Phase 1: classify every online disk. Refuse to purge if ANY disk holds
         // object data under the prefix, so a degraded/healable object is never
         // destroyed.
-        let mut per_disk_dirs: Vec<(usize, Vec<String>)> = Vec::new();
+        let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
         let mut existed = false;
         for (i, disk) in disks.iter().enumerate() {
             let Some(disk) = disk else { continue };
             match Self::scan_orphan_dir(disk, bucket, object).await {
                 OrphanDirScan::HasData => return Ok(false),
-                OrphanDirScan::Empty(dirs) => {
+                OrphanDirScan::Purgeable {
+                    empty_dirs,
+                    committed_files,
+                } => {
                     existed = true;
-                    per_disk_dirs.push((i, dirs));
+                    per_disk_dirs.push((i, empty_dirs, committed_files));
                 }
                 OrphanDirScan::Missing => {}
             }
@@ -6205,32 +6325,14 @@ impl SetDisks {
             return Ok(false);
         }
 
-        // Phase 2: remove the empty directories children-first on each disk. A
-        // non-recursive delete performs an empty-only `rmdir`, so a directory that
-        // concurrently gained an object fails with DirectoryNotEmpty and is skipped —
-        // a racing PutObject is never clobbered.
-        for (i, mut dirs) in per_disk_dirs {
+        // Phase 2: remove only the files classified as committed residue, then
+        // remove directories children-first. Every directory delete is
+        // non-recursive, so a directory that concurrently gained an object fails
+        // with DirectoryNotEmpty and is skipped — a racing PutObject is never
+        // clobbered.
+        for (i, empty_dirs, committed_files) in per_disk_dirs {
             let Some(disk) = disks[i].as_ref() else { continue };
-            dirs.reverse();
-            for dir in dirs {
-                if let Err(err) = disk
-                    .delete(
-                        bucket,
-                        &dir,
-                        DeleteOptions {
-                            recursive: false,
-                            immediate: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                {
-                    // Best effort: a sibling removal may have already cleared a shared
-                    // parent, or a concurrent writer repopulated the directory. Neither
-                    // is fatal to purging the orphan tree.
-                    debug!(bucket, object, dir, error = ?err, "purge_orphan_dir_object: skipped non-empty/absent directory");
-                }
-            }
+            Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
         }
 
         Ok(true)
@@ -7049,6 +7151,16 @@ mod tests {
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
 
+    #[test]
+    fn orphan_dir_entries_must_be_single_relative_components() {
+        for entry in ["part.1", "child/", "delete-data.00000000-0000-0000-0000-000000000001"] {
+            assert!(is_safe_orphan_dir_entry(entry), "{entry:?} should be accepted");
+        }
+        for entry in ["", "/", ".", "..", "../", "child//", "a/b", r"a\b", "./"] {
+            assert!(!is_safe_orphan_dir_entry(entry), "{entry:?} should be rejected");
+        }
+    }
+
     #[test]
     #[serial_test::serial(codec_streaming_env)]
     fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() {
@@ -7227,6 +7339,96 @@ mod tests {
         (dir, disk)
     }
 
+    #[tokio::test]
+    async fn orphan_cleanup_preserves_object_published_after_scan() {
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let transaction = Uuid::new_v4();
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+
+        let nested_object = residue.join("nested");
+        tokio::fs::create_dir_all(&nested_object)
+            .await
+            .expect("concurrent object directory should be created");
+        tokio::fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"new metadata")
+            .await
+            .expect("concurrent object metadata should be written");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        assert!(
+            nested_object.join(STORAGE_FORMAT_FILE).exists(),
+            "an object published after classification must survive cleanup"
+        );
+        assert!(!residue.join("part.1").exists(), "classified stale data should be reclaimed");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn orphan_cleanup_keeps_commit_marker_when_residue_delete_fails() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4());
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(&marker_name), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
+            .await
+            .expect("residue directory should become read-only");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        let part_remains = residue.join("part.1").exists();
+        let marker_remains = residue.join(marker_name).exists();
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o755))
+            .await
+            .expect("residue directory permissions should be restored");
+        assert!(part_remains, "the injected residue delete failure should retain the part");
+        assert!(marker_remains, "the commit marker must remain so a later cleanup can retry");
+    }
+
     async fn io_primitives_test_set(disks: Vec<Option<DiskStore>>, default_parity_count: usize) -> Arc<SetDisks> {
         let set_drive_count = disks.len();
         SetDisks::new(

crates/ecstore/src/set_disk/mod.rs

@@ -329,6 +329,7 @@ const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
 const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
 const EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED: &str = "set_disk_rename_tail_drain_failed";
 const EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY: &str = "set_disk_put_object_stage_summary";
+const EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED: &str = "set_disk_orphan_purge_skipped";
 const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
 const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
 const DEFAULT_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 64 * 1024 * 1024;
@@ -858,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
 
 mod core;
 #[cfg(test)]
-pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
+pub(crate) use core::io_primitives::disk_call_counters;
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
 mod ctx;
 mod metadata;
 mod ops;
@@ -871,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
 #[cfg(test)]
 pub(crate) use ops::object::DeleteObjectCommitBarrier;
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
@@ -1638,10 +1641,13 @@ mod prepared_get_object_metadata_tests {
         .await;
     }
 
-    #[tokio::test]
+    #[test]
     #[serial_test::serial(body_cache_hook)]
-    async fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
-        let (_dirs, set_disks) = make_local_set_disks(6, 2).await;
+    fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("current-thread runtime should build");
         let bucket = "inline-read-plan-unequal";
         let object = object_with_initial_data_shards_for_geometry(bucket, "inline-object", 6, 2);
         let payload = b"inline quorum payload".repeat(256);
@@ -1650,55 +1656,65 @@ mod prepared_get_object_metadata_tests {
             ..Default::default()
         };
 
-        set_disks
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("bucket should be created");
-        let mut put_reader = PutObjReader::from_vec(payload.clone());
-        set_disks
-            .put_object(bucket, &object, &mut put_reader, &opts)
-            .await
-            .expect("inline object should be written");
-
-        let read_once = |enabled: bool| {
-            let set_disks = Arc::clone(&set_disks);
-            let bucket = bucket.to_string();
-            let object = object.clone();
-            let payload = payload.clone();
-            let opts = opts.clone();
-            async move {
-                temp_env::async_with_vars(
-                    [
-                        (
-                            "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
-                            Some(if enabled { "true" } else { "false" }),
-                        ),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
-                    ],
-                    async {
-                        let calls = disk_call_counters::observe(&object);
-                        let mut reader = set_disks
-                            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &opts)
-                            .await
-                            .expect("inline GET reader should open");
-                        let mut restored = Vec::new();
-                        reader
-                            .stream
-                            .read_to_end(&mut restored)
-                            .await
-                            .expect("inline GET body should stream");
-                        assert_eq!(restored, payload);
-                        calls.total(disk_call_counters::KIND_READ_VERSION)
-                    },
-                )
+        let (_dirs, set_disks) = runtime.block_on(async {
+            let (dirs, set_disks) = make_local_set_disks(6, 2).await;
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
                 .await
-            }
-        };
+                .expect("bucket should be created");
+            let mut put_reader = PutObjReader::from_vec(payload.clone());
+            set_disks
+                .put_object(bucket, &object, &mut put_reader, &opts)
+                .await
+                .expect("inline object should be written");
+            (dirs, set_disks)
+        });
 
-        let gate_off_calls = read_once(false).await;
-        let gate_on_calls = read_once(true).await;
-        assert_eq!(gate_on_calls, gate_off_calls, "inline gate must not add reserve fanout");
+        // disk_call_counters counts tasks that started running, so an
+        // early-stop abort races the single-pending inline hedge into a ±1
+        // count per read. The fanout lifecycle histogram records the
+        // scheduling decision itself and stays deterministic under load.
+        let recorder = CapturingRecorder::default();
+        let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
+        rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            runtime.block_on(async {
+                for enabled in [false, true] {
+                    temp_env::async_with_vars(
+                        [
+                            (
+                                "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
+                                Some(if enabled { "true" } else { "false" }),
+                            ),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
+                        ],
+                        async {
+                            let mut reader = set_disks
+                                .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
+                                .await
+                                .expect("inline GET reader should open");
+                            let mut restored = Vec::new();
+                            reader
+                                .stream
+                                .read_to_end(&mut restored)
+                                .await
+                                .expect("inline GET body should stream");
+                            assert_eq!(restored, payload);
+                        },
+                    )
+                    .await;
+                }
+            })
+        });
+        rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
+
+        let scheduled = recorder.histogram_values(
+            "rustfs_io_get_object_metadata_fanout_scheduled",
+            &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
+        );
+        assert_eq!(scheduled.len(), 2, "each GET should run exactly one metadata fanout");
+        assert_eq!(scheduled[1], scheduled[0], "inline gate must not add reserve fanout");
     }
 
     #[test]
@@ -8775,6 +8791,111 @@ mod tests {
         assert!(root.join("bucket").exists(), "bucket volume should remain");
     }
 
+    #[tokio::test]
+    async fn purge_orphan_dir_object_removes_committed_delete_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let data_dir = Uuid::new_v4();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("nested")
+            .join("object")
+            .join(data_dir.to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed delete residue should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("purge should succeed");
+
+        assert!(purged, "committed delete residue should be purgeable");
+        assert!(!root.join("bucket").join("pfx").exists(), "prefix directory should be gone");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_uncommitted_data_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("uncommitted data residue should be created");
+        fs::write(residue.join("part.1"), b"possibly live")
+            .await
+            .expect("data part should be written");
+        fs::write(residue.join("delete-data.not-a-uuid"), [])
+            .await
+            .expect("malformed marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "data without a valid committed marker must be preserved");
+        assert!(residue.join("part.1").exists(), "possibly live data must remain");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_nested_object_below_committed_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+        let nested_object = residue.join("nested");
+        fs::create_dir_all(&nested_object)
+            .await
+            .expect("nested object directory should be created");
+        fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"meta")
+            .await
+            .expect("nested object metadata should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "nested object metadata must veto committed-residue cleanup");
+        assert!(nested_object.join(STORAGE_FORMAT_FILE).exists(), "nested object metadata must remain");
+        assert!(residue.join("part.1").exists(), "committed residue must remain when cleanup is vetoed");
+    }
+
     // issue #4189: a prefix that still anchors a real object must be left intact.
     #[tokio::test]
     async fn purge_orphan_dir_object_preserves_prefix_with_object() {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -2812,9 +2812,20 @@ mod heal_result_report_tests {
             }
 
             let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-            set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("source object should be written");
+            // This fixture reads and removes physical shards immediately after
+            // PUT. A lock-owning PUT may quorum-ack before its rename tail
+            // drains, so keep the setup on the full-fanout commit path.
+            set.put_object(
+                &bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("source object should be written");
             let source = disks[2]
                 .read_version("", &bucket, object, "", &ReadOptions::default())
                 .await
@@ -3230,9 +3241,20 @@ mod heal_result_report_tests {
         }
 
         let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-        set.put_object(bucket, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("source object should be written");
+        // The target-evidence readback below asserts per-disk state right
+        // after PUT. A lock-owning PUT may quorum-ack before its rename tail
+        // drains, so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source object should be written");
         let source = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
             .await
@@ -3279,6 +3301,9 @@ mod heal_result_report_tests {
         .await
         .expect("versioned bucket should be created");
 
+        // The per-version target-evidence readback below asserts per-disk
+        // state right after PUT. A lock-owning PUT may quorum-ack before its
+        // rename tail drains, so keep the setup on the full-fanout commit path.
         let mut old_reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
         let old_info = set
             .put_object(
@@ -3287,6 +3312,7 @@ mod heal_result_report_tests {
                 &mut old_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -3304,6 +3330,7 @@ mod heal_result_report_tests {
                 &mut latest_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -4325,9 +4352,20 @@ mod heal_result_report_tests {
 
         const PAYLOAD_SIZE: usize = 1024 * 1024;
         let mut initial_reader = PutObjReader::from_vec(vec![0x11; PAYLOAD_SIZE]);
-        set.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
-            .await
-            .expect("initial object should be written");
+        // This fixture reads and removes physical shards immediately after
+        // PUT. A lock-owning PUT may quorum-ack before its rename tail drains,
+        // so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut initial_reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("initial object should be written");
 
         let current = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
@@ -4415,11 +4453,12 @@ mod heal_result_report_tests {
                 // Give the heal something to rebuild on alternating rounds: remove a
                 // shard of the current data dir right before the race.
                 if round % 2 == 1 {
-                    let current = disks[2]
-                        .read_version("", bucket, object, "", &ReadOptions::default())
-                        .await
-                        .expect("current metadata should be readable");
-                    if let Some(data_dir) = current.data_dir {
+                    // The previous round's lock-owning PUT may still be
+                    // draining its rename tail on this disk; shard damage is
+                    // best-effort here, so skip injection when it lags.
+                    if let Ok(current) = disks[2].read_version("", bucket, object, "", &ReadOptions::default()).await
+                        && let Some(data_dir) = current.data_dir
+                    {
                         let shard = temp_dirs[3]
                             .path()
                             .join(bucket)

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
 }
 
 #[cfg(any(test, feature = "test-util"))]
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum MultipartCommitPause {
     NewUploadBeforeLockLost,
     PutPartBeforeLockAcquire,
@@ -5459,19 +5459,42 @@ mod tests {
             disk.make_volume(bucket).await.expect("bucket volume should be created");
         }
         let mut initial_reader = PutObjReader::from_vec(b"old multipart body".to_vec());
+        // A lock-owning PUT may quorum-ack before its rename tail drains, and
+        // cache priming refuses to publish while a straggler disk still reads
+        // as an error; keep the setup on the full-fanout commit path.
         set_disks
-            .put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
+            .put_object(
+                bucket,
+                object,
+                &mut initial_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
             .await
             .expect("initial object should be written");
-        set_disks
-            .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
-            .await
-            .expect("initial metadata should resolve");
-        let generation = set_disks
-            .get_object_metadata_cache_generation(bucket, object)
-            .expect("metadata cache generation should be active");
-        let retired_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
-        assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_some());
+        // The publish is also bounded by the cache TTL, so re-prime until the
+        // current generation is observably cached instead of asserting on a
+        // single read that a loaded host can stall past expiry.
+        let retired_key = tokio::time::timeout(std::time::Duration::from_secs(30), async {
+            loop {
+                set_disks
+                    .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+                    .await
+                    .expect("initial metadata should resolve");
+                let generation = set_disks
+                    .get_object_metadata_cache_generation(bucket, object)
+                    .expect("metadata cache generation should be active");
+                let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+                if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
+                    return key;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("metadata priming should publish the current generation");
 
         let upload = set_disks
             .new_multipart_upload(bucket, object, &ObjectOptions::default())

crates/ecstore/src/store/bucket.rs

@@ -29,6 +29,25 @@ use std::future::Future;
 
 const DELETED_BUCKETS_PREFIX: &str = ".deleted";
 const SCANNER_BUCKET_LIST_SET_CONCURRENCY: usize = 4;
+const EVENT_BUCKET_DELETE_BLOCKED: &str = "bucket_delete_blocked";
+
+fn record_bucket_delete_blocker(bucket: &str, kind: BucketDeleteBlockerKind, residue: &BucketMetadataLessResidue) {
+    metrics::counter!("rustfs_bucket_delete_blockers_total", "kind" => kind.as_str()).increment(1);
+    debug!(
+        event = EVENT_BUCKET_DELETE_BLOCKED,
+        component = "ecstore",
+        subsystem = "bucket",
+        bucket,
+        blocker = kind.as_str(),
+        files = residue.files,
+        uuid_data_dirs = residue.uuid_data_dirs,
+        entries_scanned = residue.entries_scanned,
+        diagnostic_bytes_read = residue.diagnostic_bytes_read,
+        diagnostic_truncated = residue.diagnostic_truncated,
+        sample = residue.sample.as_deref().unwrap_or("<none>"),
+        "Bucket deletion was blocked by durable local state"
+    );
+}
 
 fn scanner_bucket_list_set_concurrency(set_count: usize) -> usize {
     set_count.clamp(1, SCANNER_BUCKET_LIST_SET_CONCURRENCY)
@@ -156,6 +175,7 @@ where
 async fn bucket_delete_local_blocker(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+    budget: &mut BucketDeleteDiagnosticBudget,
 ) -> Result<Option<StorageError>> {
     let local_disks = runtime_sources::local_disks_in(ctx).await;
     let mut residue = BucketMetadataLessResidue::default();
@@ -164,18 +184,30 @@ async fn bucket_delete_local_blocker(
         let Some(bucket_path) = disk.get_bucket_path_for_io_if_local(bucket) else {
             continue;
         };
-        let scan = scan_metadata_less_residue(&bucket_path?).await?;
+        let scan = scan_metadata_less_residue_with_budget(&bucket_path?, budget).await?;
         if scan.xlmeta_found {
+            record_bucket_delete_blocker(bucket, scan.xlmeta_blocker.unwrap_or(BucketDeleteBlockerKind::UnknownXlMeta), &scan);
             return Ok(Some(StorageError::BucketNotEmpty(bucket.to_string())));
         }
         residue.files = residue.files.saturating_add(scan.files);
         residue.uuid_data_dirs = residue.uuid_data_dirs.saturating_add(scan.uuid_data_dirs);
+        residue.entries_scanned = residue.entries_scanned.saturating_add(scan.entries_scanned);
+        residue.diagnostic_bytes_read = residue.diagnostic_bytes_read.saturating_add(scan.diagnostic_bytes_read);
         if residue.sample.is_none() {
-            residue.sample = scan.sample;
+            residue.sample = scan.sample.clone();
+        }
+        if scan.diagnostic_truncated {
+            residue.diagnostic_truncated = true;
+            record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::DiagnosticBudgetExceeded, &residue);
+            return Ok(Some(StorageError::BucketNotEmptyWithDetails {
+                bucket: bucket.to_string(),
+                details: residue.describe(),
+            }));
         }
     }
 
     if residue.has_residue_without_xlmeta() {
+        record_bucket_delete_blocker(bucket, BucketDeleteBlockerKind::OrphanDirectory, &residue);
         return Ok(Some(StorageError::BucketNotEmptyWithDetails {
             bucket: bucket.to_string(),
             details: residue.describe(),
@@ -783,12 +815,14 @@ impl ECStore {
                 }
             }
         };
+        let mut diagnostic_budget = None;
 
         if bucket_exists {
             validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
 
             if !opts.force {
-                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await? {
+                let budget = diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new);
+                if let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket, budget).await? {
                     return Err(blocker);
                 }
                 delete_opts.force_if_empty = true;
@@ -827,7 +861,12 @@ impl ECStore {
         {
             if delete_opts.force_if_empty
                 && matches!(&err, StorageError::BucketNotEmpty(_))
-                && let Some(blocker) = bucket_delete_local_blocker(&self.ctx, bucket).await?
+                && let Some(blocker) = bucket_delete_local_blocker(
+                    &self.ctx,
+                    bucket,
+                    diagnostic_budget.get_or_insert_with(BucketDeleteDiagnosticBudget::new),
+                )
+                .await?
             {
                 return Err(blocker);
             }
@@ -856,15 +895,17 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::{
-        SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes,
-        bucket_deleted_marker_prefix, bucket_deleted_marker_volume, run_bucket_usage_cleanup, run_physical_bucket_deletion,
-        scan_metadata_less_residue, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
+        BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind,
+        BucketDeleteDiagnosticBudget, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation,
+        bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, bucket_deleted_marker_volume,
+        run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue,
+        scan_metadata_less_residue_with_budget, scanner_bucket_list_set_concurrency, should_override_created_from_metadata,
         validate_table_bucket_delete_allowed,
     };
     use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::bucket::metadata_sys;
     use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
-    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
     use crate::error::StorageError;
     use crate::object_api::{ObjectOptions, PutObjReader};
     use crate::runtime::instance::InstanceContext;
@@ -879,6 +920,7 @@ mod tests {
         layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
     };
     use rustfs_data_usage::{BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageInfo};
+    use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE};
     use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
     use serial_test::serial;
     use std::path::{Path, PathBuf};
@@ -892,6 +934,88 @@ mod tests {
 
     static BUCKET_DELETE_TEST_ENV: OnceCell<(Vec<PathBuf>, Arc<ECStore>)> = OnceCell::const_new();
 
+    #[tokio::test(start_paused = true)]
+    async fn bucket_delete_diagnostic_budget_starts_with_first_scan_io_and_latches_once() {
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
+        tokio::time::advance(Duration::from_secs(10)).await;
+
+        let first_polled = Arc::new(AtomicBool::new(false));
+        let first_polled_for_io = first_polled.clone();
+        let first = budget
+            .run_io(async move {
+                first_polled_for_io.store(true, Ordering::SeqCst);
+                Ok::<_, std::io::Error>(7_u8)
+            })
+            .await
+            .expect("the first diagnostic IO should succeed");
+        assert_eq!(first, Some(7));
+        assert!(first_polled.load(Ordering::SeqCst));
+
+        tokio::time::advance(Duration::from_millis(101)).await;
+        let expired_polled = Arc::new(AtomicBool::new(false));
+        let expired_polled_for_io = expired_polled.clone();
+        let expired = budget
+            .run_io(async move {
+                expired_polled_for_io.store(true, Ordering::SeqCst);
+                Ok::<_, std::io::Error>(9_u8)
+            })
+            .await
+            .expect("an expired diagnostic budget should not become an IO error");
+        assert_eq!(expired, None);
+        assert!(
+            !expired_polled.load(Ordering::SeqCst),
+            "the deadline must remain latched after the first scan IO"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bucket_delete_diagnostic_budget_times_out_its_first_pending_io() {
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::from_millis(100));
+        let io_polled = Arc::new(AtomicBool::new(false));
+        let io_polled_for_future = io_polled.clone();
+
+        let result = budget
+            .run_io(std::future::poll_fn(move |_cx| {
+                io_polled_for_future.store(true, Ordering::SeqCst);
+                std::task::Poll::<std::io::Result<()>>::Pending
+            }))
+            .await
+            .expect("a diagnostic timeout should fail closed without an IO error");
+
+        assert_eq!(result, None);
+        assert!(
+            io_polled.load(Ordering::SeqCst),
+            "the first diagnostic IO must be polled before its timeout"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delayed_metadata_less_scans_still_detect_orphans_and_xlmeta() {
+        let root = tempfile::tempdir().expect("temporary delayed-scan roots should be created");
+        let orphan_root = root.path().join("orphan-root");
+        let xlmeta_root = root.path().join("xlmeta-root");
+        std::fs::create_dir_all(&orphan_root).expect("orphan root should be created");
+        std::fs::create_dir_all(&xlmeta_root).expect("xlmeta root should be created");
+        std::fs::write(orphan_root.join("orphan-part"), b"orphan").expect("orphan fixture should be written");
+        std::fs::write(xlmeta_root.join(STORAGE_FORMAT_FILE), b"invalid-xlmeta").expect("xl.meta fixture should be written");
+
+        let mut orphan_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
+        let mut xlmeta_budget = BucketDeleteDiagnosticBudget::with_limits(16, Duration::from_secs(5));
+        tokio::time::advance(Duration::from_secs(60)).await;
+
+        let orphan = scan_metadata_less_residue_with_budget(&orphan_root, &mut orphan_budget)
+            .await
+            .expect("delayed orphan scan should complete");
+        assert!(orphan.has_residue_without_xlmeta());
+        assert!(!orphan.diagnostic_truncated);
+
+        let xlmeta = scan_metadata_less_residue_with_budget(&xlmeta_root, &mut xlmeta_budget)
+            .await
+            .expect("delayed xl.meta scan should complete");
+        assert!(xlmeta.xlmeta_found);
+        assert!(!xlmeta.diagnostic_truncated);
+    }
+
     #[tokio::test(start_paused = true)]
     async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
         let ttl = Duration::from_millis(20);
@@ -1313,6 +1437,175 @@ mod tests {
         assert!(sample.ends_with("/part.1"));
     }
 
+    #[tokio::test]
+    async fn metadata_less_residue_scan_shares_one_entry_budget_across_roots() {
+        let root = tempfile::tempdir().expect("temporary diagnostic roots should be created");
+        let first_root = root.path().join("disk-a");
+        let second_root = root.path().join("disk-b");
+        tokio::fs::create_dir_all(&first_root)
+            .await
+            .expect("first diagnostic root should be created");
+        tokio::fs::create_dir_all(&second_root)
+            .await
+            .expect("second diagnostic root should be created");
+        for index in 0..3 {
+            std::fs::write(first_root.join(format!("first-{index}")), b"").expect("first-root fixture should be written");
+            std::fs::write(second_root.join(format!("second-{index}")), b"").expect("second-root fixture should be written");
+        }
+
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(4, Duration::from_secs(5));
+        let first = scan_metadata_less_residue_with_budget(&first_root, &mut budget)
+            .await
+            .expect("first root should fit the shared budget");
+        assert!(!first.diagnostic_truncated);
+        let second = scan_metadata_less_residue_with_budget(&second_root, &mut budget)
+            .await
+            .expect("second root should stop at the remaining shared budget");
+        assert!(second.diagnostic_truncated);
+        assert!(first.entries_scanned + second.entries_scanned <= 4);
+    }
+
+    #[tokio::test]
+    async fn metadata_less_residue_scan_honors_an_expired_request_deadline() {
+        let root = tempfile::tempdir().expect("temporary diagnostic root should be created");
+        std::fs::write(root.path().join("orphan"), b"").expect("deadline fixture should be written");
+        let mut budget = BucketDeleteDiagnosticBudget::with_limits(8, Duration::ZERO);
+
+        let scan = scan_metadata_less_residue_with_budget(root.path(), &mut budget)
+            .await
+            .expect("an expired diagnostic budget should fail closed without an IO error");
+
+        assert!(scan.diagnostic_truncated);
+        assert_eq!(scan.entries_scanned, 0);
+        assert_eq!(scan.diagnostic_bytes_read, 0);
+    }
+
+    #[tokio::test]
+    async fn metadata_less_residue_scan_stops_at_diagnostic_budget() {
+        let root = tempfile::tempdir().expect("temporary bucket root should be created");
+        let bucket_path = root.path().join("bucket");
+        tokio::fs::create_dir_all(&bucket_path)
+            .await
+            .expect("budget fixture directory should be created");
+        for index in 0..(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES + 32) {
+            std::fs::write(bucket_path.join(format!("orphan-{index:05}")), b"").expect("budget fixture file should be created");
+        }
+
+        let residue = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("budgeted residue scan should fail closed without an IO error");
+        assert!(residue.diagnostic_truncated);
+        assert!(residue.has_residue_without_xlmeta());
+        assert!(!residue.xlmeta_found);
+        assert!(residue.entries_scanned <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
+        assert!(residue.files <= BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES);
+        assert_eq!(residue.diagnostic_bytes_read, 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_residue_scan_distinguishes_visible_and_tier_free_xlmeta() {
+        let root = tempfile::tempdir().expect("temporary bucket root should be created");
+        let bucket_path = root.path().join("bucket");
+        let visible_path = bucket_path.join("visible").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(visible_path.parent().expect("visible xl.meta should have a parent"))
+            .await
+            .expect("visible object directory should be created");
+        let mut visible = FileMeta::new();
+        visible
+            .add_version(FileInfo {
+                version_id: Some(Uuid::new_v4()),
+                mod_time: Some(OffsetDateTime::now_utc()),
+                ..Default::default()
+            })
+            .expect("visible version should encode");
+        tokio::fs::write(&visible_path, visible.marshal_msg().expect("visible xl.meta should marshal"))
+            .await
+            .expect("visible xl.meta should be written");
+
+        let visible_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("visible xl.meta scan should succeed");
+        assert_eq!(visible_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::VisibleVersion));
+
+        tokio::fs::remove_dir_all(bucket_path.join("visible"))
+            .await
+            .expect("visible fixture should be removed");
+        let free_path = bucket_path.join("free").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(free_path.parent().expect("free xl.meta should have a parent"))
+            .await
+            .expect("free-version object directory should be created");
+        let source_version_id = Uuid::new_v4();
+        let mut free = FileMeta::new();
+        free.add_version(FileInfo {
+            version_id: Some(source_version_id),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_version_id: Some(Uuid::new_v4()),
+            transition_tier: "WARM".to_string(),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        })
+        .expect("transitioned source should encode");
+        let mut delete = FileInfo {
+            version_id: Some(source_version_id),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        delete.set_tier_free_version_id(&Uuid::new_v4().to_string());
+        free.delete_version(&delete)
+            .expect("transitioned source delete should create a free-version");
+        tokio::fs::write(&free_path, free.marshal_msg().expect("free-version xl.meta should marshal"))
+            .await
+            .expect("free-version xl.meta should be written");
+
+        let free_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("free-version xl.meta scan should succeed");
+        assert_eq!(free_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::TierFreeVersion));
+
+        tokio::fs::remove_dir_all(bucket_path.join("free"))
+            .await
+            .expect("free-version fixture should be removed");
+
+        let exact_limit_path = bucket_path.join("exact-limit").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(exact_limit_path.parent().expect("exact-limit xl.meta should have a parent"))
+            .await
+            .expect("exact-limit object directory should be created");
+        let exact_limit = tokio::fs::File::create(&exact_limit_path)
+            .await
+            .expect("exact-limit xl.meta should be created");
+        exact_limit
+            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES)
+            .await
+            .expect("exact-limit xl.meta should be extended without allocating its contents");
+        let exact_limit_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("exact-limit xl.meta scan should remain fail closed");
+        assert_eq!(exact_limit_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
+        assert_eq!(exact_limit_scan.diagnostic_bytes_read, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
+        tokio::fs::remove_dir_all(bucket_path.join("exact-limit"))
+            .await
+            .expect("exact-limit fixture should be removed");
+
+        let oversized_path = bucket_path.join("oversized").join(STORAGE_FORMAT_FILE);
+        tokio::fs::create_dir_all(oversized_path.parent().expect("oversized xl.meta should have a parent"))
+            .await
+            .expect("oversized object directory should be created");
+        let oversized = tokio::fs::File::create(&oversized_path)
+            .await
+            .expect("oversized xl.meta should be created");
+        oversized
+            .set_len(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES + 1)
+            .await
+            .expect("oversized xl.meta should be extended without allocating its contents");
+        let oversized_scan = scan_metadata_less_residue(&bucket_path)
+            .await
+            .expect("oversized xl.meta scan should remain fail closed");
+        assert_eq!(oversized_scan.xlmeta_blocker, Some(BucketDeleteBlockerKind::UnknownXlMeta));
+        assert_eq!(oversized_scan.diagnostic_bytes_read, 0);
+        assert!(oversized_scan.diagnostic_bytes_read <= BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES);
+    }
+
     #[tokio::test]
     #[serial]
     async fn scanner_bucket_listing_unions_every_erasure_set() {

crates/ecstore/src/store/heal.rs

@@ -36,6 +36,10 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
+    bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
+}
+
 #[derive(Debug, Clone, Copy)]
 enum HealFormatPoolSkip {
     Completed,
@@ -492,7 +496,7 @@ impl ECStore {
         #[cfg(test)]
         let store_id = self.id;
 
-        let mut futures = Vec::with_capacity(pools.len());
+        let mut heal_pools = Vec::with_capacity(pools.len());
         for pool in pools.iter() {
             let suspended_complete = {
                 let pool_meta = self.pool_meta.read().await;
@@ -520,19 +524,58 @@ impl ECStore {
                 }
                 continue;
             }
-            let pool_idx = pool.pool_idx;
-            let pool = Arc::clone(pool);
-            let pool_object = object.clone();
-            let opts = *opts;
-            futures.push(
-                self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move {
-                    #[cfg(test)]
-                    crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
-                    pool.heal_object(bucket, &pool_object, version_id, &opts).await
-                }),
-            );
+            heal_pools.push(Arc::clone(pool));
         }
-        let results = join_all(futures).await;
+        let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
+            let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
+            match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
+                Ok((pool_meta_guard, admissions)) => {
+                    let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
+                    let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
+                        let pool = Arc::clone(pool);
+                        let pool_object = object.clone();
+                        let fixed_set = fixed_set.clone();
+                        let mut opts = *opts;
+                        async move {
+                            admission?;
+                            let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
+                            let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
+                            opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
+                            #[cfg(test)]
+                            if !opts.no_lock {
+                                crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
+                            }
+                            #[cfg(test)]
+                            crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                            pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                        }
+                    });
+                    let results = join_all(futures).await;
+                    drop(pool_meta_guard);
+                    results
+                }
+                Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
+            }
+        } else {
+            let mut futures = Vec::with_capacity(heal_pools.len());
+            for pool in heal_pools {
+                let pool_idx = pool.pool_idx;
+                let pool_object = object.clone();
+                let opts = *opts;
+                futures.push(self.run_external_decommission_capacity_heal(
+                    pool_idx,
+                    bucket,
+                    &object,
+                    opts,
+                    move |opts| async move {
+                        #[cfg(test)]
+                        crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                        pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                    },
+                ));
+            }
+            join_all(futures).await
+        };
 
         let mut errs = Vec::with_capacity(self.pools.len());
         let mut ress = Vec::with_capacity(self.pools.len());
@@ -631,7 +674,7 @@ mod tests {
     };
     use crate::core::sets::HealFormatAfterSaveBarrier;
     use crate::disk::error::Result as DiskResult;
-    use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
+    use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::services::rebalance::{
@@ -784,6 +827,30 @@ mod tests {
         }
     }
 
+    async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
+        let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
+        let missing_disk = target_set.disks.read().await[0]
+            .clone()
+            .expect("pool metadata fixture disk should be online");
+        missing_disk
+            .delete(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("one pool metadata shard should be removable");
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
+            "pool metadata fixture must start with one missing shard"
+        );
+        missing_disk
+    }
+
     #[tokio::test]
     async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
         let store = minimal_heal_store().await;
@@ -1243,6 +1310,121 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_read_repair_reuses_its_write_fence() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let missing_disk = remove_pool_meta_shard(&store, 0).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    read_repair: true,
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata read repair must not wait on its own capacity fence")
+        .expect("pool metadata read repair should complete");
+
+        assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "pool metadata read repair should restore the missing shard"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_preserves_typed_lock_timeout() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let lock = store.pools[0]
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("pool metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("pool metadata lock should be acquired");
+
+        let (_, err) = temp_env::async_with_vars(
+            [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata lock timeout should be mapped into the heal result");
+
+        assert!(
+            matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
+            "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
+        );
+        drop(guard);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let object = "pool.bin.backup";
+        save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
+            .await
+            .expect("neighbor metadata fixture should be written");
+
+        let target_set = store.pools[0].get_disks_by_key(object);
+        let lock = target_set
+            .new_ns_lock(RUSTFS_META_BUCKET, object)
+            .await
+            .expect("neighbor metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("neighbor metadata lock should be acquired");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    object,
+                    "",
+                    &HealOpts {
+                        pool: Some(0),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        barrier.wait_until_external_heal_target_lock_attempted().await;
+        tokio::task::yield_now().await;
+        assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
+        drop(guard);
+
+        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
+            .await
+            .expect("neighbor metadata heal should finish after the object lock is released")
+            .expect("neighbor metadata heal task should not panic")
+            .expect("neighbor metadata heal should complete");
+        assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
+        assert_eq!(result.object, object);
+        drop(barrier);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
@@ -1322,6 +1504,72 @@ mod tests {
             missing_disk.read_xl(&bucket, object, false).await.is_err(),
             "capacity-blocked targeted heal must not rewrite the missing shard"
         );
+
+        let (_, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata admission should not recurse on its namespace lock")
+        .expect("pool metadata capacity rejection should be mapped");
+        assert!(
+            matches!(err, Some(Error::SlowDown)),
+            "pool metadata heal must preserve target reservation admission: {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_per_target_capacity_admission() {
+        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        set_decommission_capacity_info_overrides_for_test(
+            store.id,
+            vec![vec![
+                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
+                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
+            ]],
+        );
+        store
+            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
+            .await
+            .expect("pool metadata reservation should activate");
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let reservation = pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .expect("pool metadata reservation should be durable");
+            assert_eq!(reservation.targets.len(), 1);
+            assert_eq!(reservation.targets[0].pool_index, 1);
+        }
+
+        let missing_disk = remove_pool_meta_shard(&store, 2).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
+        )
+        .await
+        .expect("unscoped pool metadata heal should complete")
+        .expect("unscoped pool metadata heal should return a mapped result");
+
+        assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "unscoped metadata heal should repair the admitted target while another target is reserved"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/store/list_objects.rs

@@ -20,7 +20,9 @@ fn to_filemeta_err(err: Error) -> rustfs_filemeta::Error {
     err.narrow_to_filemeta().unwrap_or_else(rustfs_filemeta::Error::other)
 }
 
-use crate::bucket::metadata_sys::{get_versioning_config, has_authoritative_never_versioned_state};
+use crate::bucket::metadata_sys::{
+    get_versioning_config, has_authoritative_never_versioned_state, has_authoritative_never_versioned_state_in,
+};
 use crate::bucket::utils::check_list_objs_args;
 use crate::bucket::versioning::VersioningApi;
 use crate::cache_value::metacache_set::{FallbackClaimTracker, ListPathRawOptions, list_path_raw_with_claim_tracker};
@@ -314,6 +316,25 @@ async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
         .unwrap_or(false)
 }
 
+fn should_purge_empty_directory_listing(
+    prefix: &str,
+    marker: Option<&str>,
+    delimiter: Option<&str>,
+    max_keys: i32,
+    incl_deleted: bool,
+    result: &ListObjectsInfo,
+) -> bool {
+    !prefix.is_empty()
+        && prefix.ends_with(SLASH_SEPARATOR)
+        && marker.is_none()
+        && delimiter.is_none_or(str::is_empty)
+        && max_keys == 1
+        && !incl_deleted
+        && !result.is_truncated
+        && result.objects.is_empty()
+        && result.prefixes.is_empty()
+}
+
 const MARKER_TAG_VERSION: &str = "v2";
 const LEGACY_MARKER_TAG_VERSIONS: &[&str] = &["v1", MARKER_TAG_VERSION];
 const LIST_CACHE_MARKER_PREFIX: &str = "[rustfs_cache:";
@@ -2501,6 +2522,7 @@ fn list_metadata_resolution_params(
     listing_quorum: usize,
     latest_object_quorum: usize,
     versioned: bool,
+    write_quorum_slack: usize,
 ) -> MetadataResolutionParams {
     let quorum = if versioned {
         listing_quorum
@@ -2511,6 +2533,7 @@ fn list_metadata_resolution_params(
         dir_quorum: quorum,
         obj_quorum: quorum,
         bucket,
+        write_quorum_slack,
         ..Default::default()
     };
 
@@ -2811,7 +2834,10 @@ fn cached_entry_needs_supplement(
 
     let mut selected_object_versions = 0;
     for version in cached.versions.iter() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions) {
             return true;
         }
@@ -2836,8 +2862,15 @@ fn listing_entries_supplement_target(
         return None;
     }
 
-    for entry in entries.0.iter().flatten() {
-        if entry.is_dir() {
+    for (idx, entry) in entries.0.iter().enumerate() {
+        let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
+            continue;
+        };
+        if entries.0[..idx]
+            .iter()
+            .flatten()
+            .any(|previous| previous.name == entry.name && previous.is_object())
+        {
             continue;
         }
 
@@ -2850,6 +2883,15 @@ fn listing_entries_supplement_target(
                     .is_some_and(|candidate| candidate.name == entry.name && candidate.is_object())
             })
             .count();
+        let entries_disagree = entries.0.iter().flatten().any(|candidate| {
+            candidate.name == entry.name && candidate.is_object() && !entry.matches(Some(candidate), resolver.strict).1
+        });
+        // A split primary sample can fall back to an older version even when a
+        // newer version reaches write quorum only after the fallback disks join.
+        if entries_disagree {
+            return Some(entry.name.clone());
+        }
+
         let mut entry = entry.clone();
         if let Ok(cached) = entry.xl_meta()
             && cached_entry_needs_supplement(&cached, reader_disks, resolver, enforce_write_quorum)
@@ -2946,7 +2988,10 @@ fn resolve_agreed_listing_entry(
     let mut needs_supplement = false;
 
     for (idx, version) in cached.versions.iter().enumerate() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if reader_disks < required_quorum {
             needs_supplement |=
                 version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions);
@@ -3000,21 +3045,51 @@ fn latest_listing_object_quorum(
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
-    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum)
+    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum, unreachable_disks)
 }
 
+/// Object quorum a listed "latest" version must reach among the drives the
+/// listing can actually consult.
+///
+/// An object legally committed at write quorum can have up to
+/// `unreachable_disks` of its metadata copies on drives that are offline for
+/// this listing, so the write-quorum requirement is relaxed by that amount.
+/// The result is floored at the erasure read quorum (data drives): below that
+/// the object could not be read back either, and a quorum-deleted object
+/// leaves at most `drive_count - write_quorum < read_quorum` stale copies, so
+/// the floor also keeps deleted objects from resurfacing.
+///
+/// Trade-off: while a drive is offline, a torn overwrite that reached only
+/// `write_quorum - 1` drives becomes indistinguishable from a committed write
+/// whose missing copy sits on the offline drive, so it can be listed as
+/// latest. GET at read quorum serves that same version in that state, so the
+/// listing stays consistent with reads instead of hiding readable objects.
 fn latest_listing_required_object_quorum(
     listing_quorum: usize,
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
     if !enforce_write_quorum {
         return listing_quorum;
     }
 
-    write_quorum_for_drive_count(drive_count, parity_count).max(listing_quorum)
+    let read_quorum = drive_count.saturating_sub(parity_count);
+    write_quorum_for_drive_count(drive_count, parity_count)
+        .saturating_sub(unreachable_disks)
+        .max(read_quorum)
+        .max(listing_quorum)
+}
+
+fn latest_listing_write_quorum_slack(enforce_write_quorum: bool, drive_count: usize, online_disks: usize) -> usize {
+    if !enforce_write_quorum {
+        return 0;
+    }
+
+    drive_count.saturating_sub(online_disks)
 }
 
 fn enforce_latest_listing_write_quorum(strict_latest: bool, ask_disks: &str) -> bool {
@@ -3772,6 +3847,19 @@ impl ECStore {
                 .list_objects_from_opt_in_key_only_provider(&opts, mode, max_keys, incl_deleted)
                 .await?
         {
+            if should_purge_empty_directory_listing(
+                prefix,
+                opts.marker.as_deref(),
+                delimiter.as_deref(),
+                max_keys,
+                incl_deleted,
+                &result,
+            ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+                .await
+                .unwrap_or(false)
+            {
+                self.purge_orphan_dir_object(bucket, prefix).await;
+            }
             return Ok(result);
         }
 
@@ -3802,6 +3890,7 @@ impl ECStore {
         };
 
         let mut list_result = self
+            .clone()
             .list_path(&opts)
             .await
             .unwrap_or_else(|err| MetaCacheEntriesSortedResult {
@@ -3820,7 +3909,7 @@ impl ECStore {
         }
 
         if let Some(result) = list_result.entries.as_mut() {
-            result.forward_past(opts.marker);
+            result.forward_past(opts.marker.clone());
         }
 
         // contextCanceled
@@ -3848,12 +3937,26 @@ impl ECStore {
         );
         let _ = next_version_idmarker;
 
-        Ok(ListObjectsInfo {
+        let result = ListObjectsInfo {
             is_truncated,
             next_marker,
             objects,
             prefixes,
-        })
+        };
+        if should_purge_empty_directory_listing(
+            prefix,
+            opts.marker.as_deref(),
+            delimiter.as_deref(),
+            max_keys,
+            incl_deleted,
+            &result,
+        ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+            .await
+            .unwrap_or(false)
+        {
+            self.purge_orphan_dir_object(bucket, prefix).await;
+        }
+        Ok(result)
     }
 
     pub async fn inner_list_object_versions(
@@ -4306,6 +4409,9 @@ impl ECStore {
         for eset in self.pools.iter() {
             for set in eset.disk_set.iter() {
                 let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+                // Captured before any quorum-based filtering: only genuinely
+                // unreachable drives may relax the write-quorum requirement.
+                let online_disks = disks.len();
                 let opts = opts.clone();
 
                 let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
@@ -4331,11 +4437,14 @@ impl ECStore {
                     let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                     let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                     let write_quorum_parity = set.default_parity_count;
+                    let write_quorum_slack =
+                        latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                     let required_obj_quorum = latest_listing_required_object_quorum(
                         listing_quorum,
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                     let fallback_disks = {
@@ -4357,11 +4466,17 @@ impl ECStore {
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
 
-                    let resolver =
-                        list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                    let resolver = list_metadata_resolution_params(
+                        bucket.to_owned(),
+                        listing_quorum,
+                        obj_quorum,
+                        !opts.latest_only,
+                        write_quorum_slack,
+                    );
                     let agreed_resolver = resolver.clone();
                     let partial_resolver = resolver.clone();
                     let reader_disks = disks.len();
@@ -5548,6 +5663,9 @@ impl Sets {
 
         for set in &self.disk_set {
             let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+            // Captured before any quorum-based filtering: only genuinely
+            // unreachable drives may relax the write-quorum requirement.
+            let online_disks = disks.len();
             let opts = opts.clone();
             let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
             inputs.push(list_out_rx);
@@ -5573,11 +5691,14 @@ impl Sets {
                 let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                 let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                 let write_quorum_parity = set.default_parity_count;
+                let write_quorum_slack =
+                    latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                 let required_obj_quorum = latest_listing_required_object_quorum(
                     listing_quorum,
                     set.set_drive_count,
                     write_quorum_parity,
                     enforce_write_quorum,
+                    write_quorum_slack,
                 );
                 ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                 let fallback_disks = if let Some(asked_disks) = positive_ask_disks(ask_disks)
@@ -5592,10 +5713,21 @@ impl Sets {
                 let fallback_disks = Arc::new(fallback_disks);
                 let claim_tracker = FallbackClaimTracker::default();
 
-                let obj_quorum =
-                    latest_listing_object_quorum(listing_quorum, set.set_drive_count, write_quorum_parity, enforce_write_quorum);
+                let obj_quorum = latest_listing_object_quorum(
+                    listing_quorum,
+                    set.set_drive_count,
+                    write_quorum_parity,
+                    enforce_write_quorum,
+                    write_quorum_slack,
+                );
                 let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
-                let resolver = list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                let resolver = list_metadata_resolution_params(
+                    bucket.to_owned(),
+                    listing_quorum,
+                    obj_quorum,
+                    !opts.latest_only,
+                    write_quorum_slack,
+                );
                 let agreed_resolver = resolver.clone();
                 let partial_resolver = resolver.clone();
                 let reader_disks = disks.len();
@@ -6517,6 +6649,9 @@ impl SetDisks {
         let list_path_started = std::time::Instant::now();
 
         let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await;
+        // Captured before any quorum-based filtering: only genuinely
+        // unreachable drives may relax the write-quorum requirement.
+        let online_disks = disks.len();
 
         let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
         if ask_disks == -1 {
@@ -6540,11 +6675,13 @@ impl SetDisks {
 
         let enforce_write_quorum = enforce_latest_listing_write_quorum(!opts.versioned, &opts.ask_disks);
         let write_quorum_parity = self.default_parity_count;
+        let write_quorum_slack = latest_listing_write_quorum_slack(enforce_write_quorum, self.set_drive_count, online_disks);
         let required_obj_quorum = latest_listing_required_object_quorum(
             listing_quorum,
             self.set_drive_count,
             write_quorum_parity,
             enforce_write_quorum,
+            write_quorum_slack,
         );
         ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
         let mut fallback_disks = Vec::new();
@@ -6562,10 +6699,21 @@ impl SetDisks {
 
         let bucket = opts.bucket.clone();
         let base_dir = opts.base_dir.clone();
-        let latest_object_quorum =
-            latest_listing_object_quorum(listing_quorum, self.set_drive_count, write_quorum_parity, enforce_write_quorum);
+        let latest_object_quorum = latest_listing_object_quorum(
+            listing_quorum,
+            self.set_drive_count,
+            write_quorum_parity,
+            enforce_write_quorum,
+            write_quorum_slack,
+        );
         let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, latest_object_quorum, enforce_write_quorum);
-        let resolver = list_metadata_resolution_params(bucket.clone(), listing_quorum, latest_object_quorum, opts.versioned);
+        let resolver = list_metadata_resolution_params(
+            bucket.clone(),
+            listing_quorum,
+            latest_object_quorum,
+            opts.versioned,
+            write_quorum_slack,
+        );
         let agreed_resolver = resolver.clone();
         let partial_resolver = resolver.clone();
         let reader_disks = disks.len();
@@ -6577,6 +6725,7 @@ impl SetDisks {
             asked_disks = ask_disks,
             listing_quorum = listing_quorum,
             latest_object_quorum = latest_object_quorum,
+            write_quorum_slack = write_quorum_slack,
             raw_min_disks = raw_min_disks,
             fallback_disks = fallback_disks.len(),
             limit = opts.limit,
@@ -6818,34 +6967,36 @@ mod test {
         LIST_CURSOR_GENERATION_LIVE, LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY,
         LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, ListIndexFallbackReason, ListIndexLifecycle, ListIndexLifecycleState,
         ListIndexSourceDecision, ListMetadataAuthority, ListMetadataIndexHealth, ListObjectsIndexProviderKind,
-        ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution,
-        ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
+        ListObjectsIndexProviderState, ListObjectsInfo, ListPathOptions, ListPathRawOptions, ListSourceMode,
+        ListingEntryResolution, ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
         NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
         PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
         PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex,
         PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker,
-        current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum,
-        expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects,
-        latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key,
-        list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
+        cached_entry_needs_supplement, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
+        enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
+        latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
+        latest_listing_required_object_quorum, latest_listing_write_quorum_slack, list_marker_key, list_merged_entry_channel,
+        list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
         list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
         list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
         list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
         list_objects_metadata_fast_guardrails_from_env, list_objects_paginate, list_objects_quorum_from_env,
-        load_namespace_mutation_journal_state, load_persistent_key_only_index, max_keys_plus_one, merge_entry_channels,
-        namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env,
-        namespace_mutation_journal_chaos_enabled_from_env, namespace_mutation_journal_chaos_sequence_from_env,
-        namespace_mutation_journal_chaos_status_from_env, normalize_list_quorum, observe_list_objects_mutations_with_store,
-        parse_namespace_mutation_journal_state, parse_persistent_key_only_index, parse_persistent_list_metadata_object,
-        parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
+        listing_entries_supplement_target, load_namespace_mutation_journal_state, load_persistent_key_only_index,
+        max_keys_plus_one, merge_entry_channels, namespace_mutation_journal_chaos_bucket_from_env,
+        namespace_mutation_journal_chaos_config_from_env, namespace_mutation_journal_chaos_enabled_from_env,
+        namespace_mutation_journal_chaos_sequence_from_env, namespace_mutation_journal_chaos_status_from_env,
+        normalize_list_quorum, observe_list_objects_mutations_with_store, parse_namespace_mutation_journal_state,
+        parse_persistent_key_only_index, parse_persistent_list_metadata_object, parse_version_marker,
+        persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
         persistent_key_only_index_health, persistent_key_only_index_matches_provider,
         reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
-        scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
-        send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
-        write_persistent_key_only_index_with_metadata,
+        resolve_listing_entries_with_supplement, scanner_namespace_mutation_generation, select_list_index_provider_source_mode,
+        select_list_index_source_mode, send_or_cancel, should_purge_empty_directory_listing, version_marker_for_entries,
+        walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index_with_metadata,
     };
     use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
-    use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
+    use crate::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, error::DiskError, new_disk};
     use crate::error::StorageError;
     use crate::object_api::ObjectInfo;
     use rustfs_filemeta::{
@@ -7168,6 +7319,32 @@ mod test {
         }
     }
 
+    fn test_object_with_delete_marker_meta_entry(
+        name: &str,
+        object_mod_time: time::OffsetDateTime,
+        delete_mod_time: time::OffsetDateTime,
+    ) -> MetaCacheEntry {
+        let mut object = test_object_meta_entry_with_erasure_versions(name, &[(object_mod_time, "object-etag", 4, 2)]);
+        let delete = test_delete_marker_meta_entry(name, delete_mod_time);
+        let mut metadata = object.cached.take().expect("test object metadata should be cached");
+        let delete_version = delete
+            .cached
+            .expect("test delete marker metadata should be cached")
+            .versions
+            .into_iter()
+            .next()
+            .expect("test delete marker should contain one version");
+        metadata.versions.insert(0, delete_version);
+        let encoded = metadata.marshal_msg().expect("test metadata should marshal");
+
+        MetaCacheEntry {
+            name: name.to_owned(),
+            metadata: encoded,
+            cached: Some(metadata),
+            reusable: false,
+        }
+    }
+
     fn test_dir_meta_entry(name: &str) -> MetaCacheEntry {
         MetaCacheEntry {
             name: name.to_owned(),
@@ -8665,6 +8842,80 @@ mod test {
         assert_eq!(scanner_namespace_mutation_generation(), 2);
     }
 
+    #[test]
+    fn empty_directory_listing_purge_requires_complete_exact_recursive_request() {
+        let empty = ListObjectsInfo::default();
+        assert!(should_purge_empty_directory_listing("ghost/", None, None, 1, false, &empty));
+        assert!(should_purge_empty_directory_listing("ghost/", None, Some(""), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost", None, None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", Some("marker"), None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, Some("/"), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 0, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 2, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, true, &empty));
+
+        let mut live = ListObjectsInfo::default();
+        live.objects.push(ObjectInfo::default());
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &live));
+
+        let truncated = ListObjectsInfo {
+            is_truncated: true,
+            ..Default::default()
+        };
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &truncated));
+    }
+
+    #[tokio::test]
+    async fn empty_recursive_listing_purges_committed_delete_residue() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "listing-purge-bucket";
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created with authoritative metadata");
+        let data_dir = uuid::Uuid::new_v4();
+        let transaction = uuid::Uuid::new_v4();
+        for dir in &dirs {
+            let residue = dir
+                .path()
+                .join(bucket)
+                .join("ghost")
+                .join("nested")
+                .join("object")
+                .join(data_dir.to_string());
+            tokio::fs::create_dir_all(&residue)
+                .await
+                .expect("committed delete residue should be created");
+            tokio::fs::write(residue.join("part.1"), b"stale")
+                .await
+                .expect("stale part should be written");
+            tokio::fs::write(
+                residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+                [],
+            )
+            .await
+            .expect("committed delete marker should be written");
+        }
+
+        let result = store
+            .list_objects_generic(bucket, "ghost/", None, None, 1, false)
+            .await
+            .expect("empty recursive listing should succeed");
+
+        assert!(result.objects.is_empty());
+        assert!(result.prefixes.is_empty());
+        for dir in &dirs {
+            assert!(
+                !dir.path().join(bucket).join("ghost").exists(),
+                "the empty listing should reclaim its committed delete residue"
+            );
+        }
+    }
+
     #[test]
     fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
         let provider = ListObjectsIndexProviderState::walker_key_only();
@@ -8947,7 +9198,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_limits_plain_listing_to_latest_version() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8957,7 +9208,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_keeps_all_versions_for_version_listing() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8967,22 +9218,22 @@ mod test {
 
     #[test]
     fn latest_listing_object_quorum_uses_write_quorum_for_strict_latest_listing() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 5);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
     }
 
     #[test]
     fn latest_listing_object_quorum_calculates_low_parity_write_quorum() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 7);
         assert_eq!(ask_disks, 7);
-        assert_eq!(latest_listing_object_quorum(2, 8, 1, true), 7);
+        assert_eq!(latest_listing_object_quorum(2, 8, 1, true, 0), 7);
     }
 
     #[test]
@@ -8991,21 +9242,196 @@ mod test {
         assert!(!enforce_latest_listing_write_quorum(true, "disk"));
         assert!(enforce_latest_listing_write_quorum(true, "optimal"));
         assert!(!enforce_latest_listing_write_quorum(false, "optimal"));
-        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false), 2);
-        assert_eq!(latest_listing_object_quorum(2, 4, 2, false), 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false, 0), 2);
+        assert_eq!(latest_listing_object_quorum(2, 4, 2, false, 0), 2);
         assert_eq!(expand_ask_disks_for_object_quorum(2, 4, 2), 2);
     }
 
+    #[test]
+    fn latest_listing_object_quorum_relaxes_write_quorum_by_unreachable_drives() {
+        // 4-drive set, EC 2+2: with one drive offline an object committed at
+        // write quorum 3 can only ever show 2 metadata copies to the listing.
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        assert_eq!(slack, 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, slack), 2);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, 0), 3);
+
+        // The relaxed quorum never drops below the erasure read quorum, so a
+        // quorum-deleted object (at most one stale copy) stays hidden even
+        // with half the set unreachable.
+        let slack = latest_listing_write_quorum_slack(true, 4, 2);
+        assert_eq!(slack, 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, true, slack), 2);
+
+        assert_eq!(latest_listing_write_quorum_slack(false, 4, 2), 0);
+    }
+
+    #[test]
+    fn latest_listing_resolves_degraded_object_when_a_set_drive_is_unreachable() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]);
+
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        let obj_quorum = latest_listing_object_quorum(2, 4, 2, true, slack);
+        assert_eq!(obj_quorum, 2);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, obj_quorum, false, slack);
+        let resolved = resolve_listing_entries(entries(), resolver, true)
+            .expect("object committed at write quorum should stay listable with one holder drive offline");
+        assert_eq!(resolved.name, "object");
+
+        // Without the unreachable-drive slack the same sample is dropped even
+        // though the object still satisfies read quorum for GET.
+        let strict_quorum = latest_listing_object_quorum(2, 4, 2, true, 0);
+        let strict_resolver = list_metadata_resolution_params("bucket".to_string(), 2, strict_quorum, false, 0);
+        assert!(resolve_listing_entries(entries(), strict_resolver, true).is_none());
+    }
+
+    #[test]
+    fn agreed_listing_entry_relaxes_write_quorum_by_unreachable_drives() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry.clone(), 2, resolver.clone(), true),
+            ListingEntryResolution::Resolved(_)
+        ));
+
+        resolver.write_quorum_slack = 0;
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry, 2, resolver, true),
+            ListingEntryResolution::NeedsSupplement(_, _)
+        ));
+    }
+
+    #[test]
+    fn cached_entry_supplement_check_honors_write_quorum_slack() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let cached = entry.xl_meta().expect("test entry should decode");
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(!cached_entry_needs_supplement(&cached, 2, &resolver, true));
+
+        resolver.write_quorum_slack = 0;
+        assert!(cached_entry_needs_supplement(&cached, 2, &resolver, true));
+    }
+
     #[test]
     fn latest_listing_object_quorum_requires_write_quorum_when_degraded_cannot_satisfy_it() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 4, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 4);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
+    }
+
+    #[tokio::test]
+    async fn latest_listing_supplements_a_split_delete_marker_sample() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+            Some(deleted.clone()),
+        ]);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, false), None);
+
+        let mut primary = resolve_listing_entries(MetaCacheEntries(entries.0.clone()), resolver.clone(), true)
+            .expect("the partial sample should fall back to the stale object version");
+        assert!(!primary.is_latest_delete_marker());
+
+        let mut fallback_disks = Vec::new();
+        let mut fallback_tempdirs = Vec::new();
+        for _ in 0..2 {
+            let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
+            let endpoint =
+                Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")).expect("valid endpoint");
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("fallback disk should be created");
+            disk.make_volume("bucket").await.expect("fallback bucket should be created");
+            disk.write_all(
+                "bucket",
+                &format!("object/{STORAGE_FORMAT_FILE}"),
+                bytes::Bytes::copy_from_slice(&deleted.metadata),
+            )
+            .await
+            .expect("fallback metadata should be written");
+            fallback_disks.push(disk);
+            fallback_tempdirs.push(tempdir);
+        }
+        let supplement = ListingSupplement::new(
+            ListingSupplementOptions {
+                bucket: "bucket".to_string(),
+                path: String::new(),
+                recursive: true,
+                incl_deleted: false,
+                skip_hidden_prefix_check: false,
+                filter_prefix: None,
+                forward_to: None,
+                per_disk_limit: 100,
+                skip_total_timeout: true,
+                walkdir_timeout: None,
+                walkdir_stall_timeout: None,
+            },
+            Arc::new(fallback_disks),
+            FallbackClaimTracker::default(),
+        );
+
+        let mut supplemented = resolve_listing_entries_with_supplement(entries, resolver, true, supplement)
+            .await
+            .expect("the supplemented sample should resolve the committed delete marker");
+        assert!(supplemented.is_latest_delete_marker());
+    }
+
+    #[test]
+    fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+        ]);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+
+        let mut resolved = resolve_listing_entries(
+            MetaCacheEntries(vec![
+                Some(stale.clone()),
+                Some(stale.clone()),
+                Some(stale),
+                Some(deleted.clone()),
+                Some(deleted.clone()),
+                Some(deleted),
+            ]),
+            resolver,
+            true,
+        )
+        .expect("the previous object version should remain visible below delete-marker write quorum");
+
+        assert!(!resolved.is_latest_delete_marker());
     }
 
     #[tokio::test]
@@ -9082,7 +9508,7 @@ mod test {
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
         let fallback_old_entry = test_object_meta_entry_with_erasure_versions("object", &[(old_mod_time, "old-etag", 4, 4)]);
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9137,7 +9563,7 @@ mod test {
             "object",
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9181,7 +9607,7 @@ mod test {
         let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
         let entry = test_object_meta_entry_with_erasure_versions("object", &[(new_mod_time, "new-etag", 7, 1)]);
         let fallback_entry = entry.clone();
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 

crates/ecstore/src/store/mod.rs

@@ -65,7 +65,7 @@ use http::HeaderMap;
 use lazy_static::lazy_static;
 use rand::RngExt as _;
 use rustfs_config::server_config::Config;
-use rustfs_filemeta::FileInfo;
+use rustfs_filemeta::{FileInfo, FileMeta};
 use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
 use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
 use rustfs_madmin::heal_commands::HealResultItem;
@@ -88,25 +88,105 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
 pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
+pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
+
+#[derive(Debug)]
+pub(crate) struct BucketDeleteDiagnosticBudget {
+    deadline: Option<tokio::time::Instant>,
+    max_elapsed: Duration,
+    entries_remaining: usize,
+}
+
+impl BucketDeleteDiagnosticBudget {
+    pub(crate) fn new() -> Self {
+        Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
+    }
+
+    fn with_limits(entries: usize, elapsed: Duration) -> Self {
+        Self {
+            deadline: None,
+            max_elapsed: elapsed,
+            entries_remaining: entries,
+        }
+    }
+
+    fn deadline(&mut self) -> tokio::time::Instant {
+        let max_elapsed = self.max_elapsed;
+        *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
+    }
+
+    fn claim_entry(&mut self) -> bool {
+        if self.entries_remaining == 0 {
+            return false;
+        }
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+        self.entries_remaining -= 1;
+        true
+    }
+
+    async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
+    where
+        F: std::future::Future<Output = std::io::Result<T>>,
+    {
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(None);
+        }
+        match tokio::time::timeout_at(deadline, future).await {
+            Ok(result) => result.map(Some),
+            Err(_) => Ok(None),
+        }
+    }
+}
 
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub(crate) struct BucketMetadataLessResidue {
     pub(crate) xlmeta_found: bool,
+    pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
     pub(crate) files: usize,
     pub(crate) uuid_data_dirs: usize,
+    pub(crate) entries_scanned: usize,
+    pub(crate) diagnostic_bytes_read: u64,
+    pub(crate) diagnostic_truncated: bool,
     pub(crate) sample: Option<String>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BucketDeleteBlockerKind {
+    VisibleVersion,
+    TierFreeVersion,
+    UnknownXlMeta,
+    OrphanDirectory,
+    DiagnosticBudgetExceeded,
+}
+
+impl BucketDeleteBlockerKind {
+    pub(crate) const fn as_str(self) -> &'static str {
+        match self {
+            Self::VisibleVersion => "visible_version",
+            Self::TierFreeVersion => "tier_free_version",
+            Self::UnknownXlMeta => "unknown_xlmeta",
+            Self::OrphanDirectory => "orphan_directory",
+            Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
+        }
+    }
+}
+
 impl BucketMetadataLessResidue {
     pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
-        !self.xlmeta_found && self.files > 0
+        !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
     }
 
     pub(crate) fn describe(&self) -> String {
         let sample = self.sample.as_deref().unwrap_or("<none>");
         format!(
-            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
-            self.files, self.uuid_data_dirs
+            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
+            self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
         )
     }
 }
@@ -145,28 +225,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
     Ok(false)
 }
 
+#[cfg(test)]
 pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
+    let mut budget = BucketDeleteDiagnosticBudget::new();
+    scan_metadata_less_residue_with_budget(path, &mut budget).await
+}
+
+async fn scan_metadata_less_residue_with_budget(
+    path: &std::path::Path,
+    budget: &mut BucketDeleteDiagnosticBudget,
+) -> std::io::Result<BucketMetadataLessResidue> {
     use crate::disk::STORAGE_FORMAT_FILE;
     use tokio::fs;
+    use tokio::io::AsyncReadExt as _;
 
     let mut scan = BucketMetadataLessResidue::default();
     let mut stack = vec![path.to_path_buf()];
 
+    let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
+        scan.diagnostic_truncated = true;
+        scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
+    };
+
     while let Some(current_path) = stack.pop() {
-        let mut entries = match fs::read_dir(&current_path).await {
-            Ok(entries) => entries,
+        let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
+            Ok(Some(entries)) => entries,
+            Ok(None) => {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
             Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
             Err(err) => return Err(err),
         };
 
-        while let Some(entry) = entries.next_entry().await? {
-            let file_type = entry.file_type().await?;
+        loop {
+            let entry = match budget.run_io(entries.next_entry()).await? {
+                Some(Some(entry)) => entry,
+                Some(None) => break,
+                None => {
+                    mark_budget_exhausted(&mut scan);
+                    return Ok(scan);
+                }
+            };
+            if !budget.claim_entry() {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
+            scan.entries_scanned = scan.entries_scanned.saturating_add(1);
+            let Some(file_type) = budget.run_io(entry.file_type()).await? else {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            };
             let file_name = entry.file_name();
             let file_name_str = file_name.to_string_lossy();
 
             if file_name_str == STORAGE_FORMAT_FILE {
                 scan.xlmeta_found = true;
-                continue;
+                if scan.xlmeta_blocker.is_none() {
+                    let entry_path = entry.path();
+                    let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
+                        mark_budget_exhausted(&mut scan);
+                        scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
+                        return Ok(scan);
+                    };
+                    scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
+                        BucketDeleteBlockerKind::UnknownXlMeta
+                    } else {
+                        match budget.run_io(fs::File::open(&entry_path)).await {
+                            Ok(Some(file)) => {
+                                let mut data = Vec::new();
+                                let read = budget
+                                    .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
+                                    .await;
+                                scan.diagnostic_bytes_read = data.len() as u64;
+                                match read {
+                                    Ok(Some(_)) => match FileMeta::load(&data) {
+                                        Ok(meta)
+                                            if !meta.versions.is_empty()
+                                                && meta.versions.iter().all(|version| version.header.free_version()) =>
+                                        {
+                                            BucketDeleteBlockerKind::TierFreeVersion
+                                        }
+                                        Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
+                                        Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                    },
+                                    Ok(None) => {
+                                        mark_budget_exhausted(&mut scan);
+                                        BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                                    }
+                                    Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                }
+                            }
+                            Ok(None) => {
+                                mark_budget_exhausted(&mut scan);
+                                BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                            }
+                            Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                        }
+                    });
+                    let sample = entry_path
+                        .strip_prefix(path)
+                        .unwrap_or(entry_path.as_path())
+                        .to_string_lossy()
+                        .replace(std::path::MAIN_SEPARATOR, "/");
+                    scan.sample = Some(sample);
+                }
+                return Ok(scan);
             }
 
             if file_type.is_dir() {
@@ -222,7 +386,10 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
+pub(crate) use object::{
+    DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
+    SourceCleanupMutationFence, tiered_data_movement_source_matches,
+};
 pub use object::{
     PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
     SnapshotConsistencyError,

crates/ecstore/src/store/multipart.rs

@@ -846,6 +846,7 @@ impl ECStore {
             .await
     }
 
+    #[cfg(all(test, feature = "test-util"))]
     pub(crate) async fn complete_multipart_upload_for_data_movement(
         self: Arc<Self>,
         target: (usize, Option<&ObjectLockDiagGuard>),
@@ -854,6 +855,44 @@ impl ECStore {
         upload_id: &str,
         uploaded_parts: Vec<CompletePart>,
         opts: &ObjectOptions,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
+            .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
+        self: Arc<Self>,
+        target_pool_idx: usize,
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(
+            (target_pool_idx, None),
+            bucket,
+            object,
+            upload_id,
+            uploaded_parts,
+            opts,
+            Some(publication_fence),
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn complete_multipart_upload_for_data_movement_inner(
+        self: Arc<Self>,
+        target: (usize, Option<&ObjectLockDiagGuard>),
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: Option<RemoteTuplePublicationFence>,
     ) -> Result<ObjectInfo> {
         let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
@@ -885,8 +924,36 @@ impl ECStore {
             snapshot.add_lock_fences(&mut opts);
             opts.object_lock_config_snapshot = Some(snapshot);
         }
-        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+        let fixed_read_anchor = publication_fence
+            .as_ref()
+            .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
             .await;
+        // NewMultipart/UploadPart are staging only. Acquire and consume the
+        // non-cloneable publication capability immediately before Complete,
+        // then retain its guards until Complete has drained the commit path.
+        let publication_object = encode_dir_object(object);
+        let publication_guard = match publication_fence {
+            Some(publication_fence) => {
+                let guard = publication_fence
+                    .into_commit_guard(target_pool_idx, bucket, &publication_object)
+                    .await?;
+                guard.add_namespace_lock_fence(&mut opts);
+                opts.no_lock = true;
+                Some(guard)
+            }
+            None => {
+                if rustfs_utils::http::metadata_compat::contains_key_str(
+                    &opts.user_defined,
+                    rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
+                ) {
+                    return Err(Error::other(
+                        "data movement multipart completion cannot publish transition ownership without a publication capability",
+                    ));
+                }
+                None
+            }
+        };
         #[cfg(test)]
         pause_data_movement_multipart_before_selected_completion(bucket).await;
         let pool = self
@@ -911,6 +978,7 @@ impl ECStore {
                 },
             )
             .await;
+        drop(publication_guard);
         let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
         if result.is_ok() {
             list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;

crates/ecstore/src/store/rebalance.rs

@@ -309,6 +309,17 @@ impl ECStore {
     }
 
     pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
+            let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+            let authorization = opts
+                .tier_delete_dispatch_authorization
+                .as_ref()
+                .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
+            authorization.ensure_current(bucket, incarnation, object)?;
+            Some((authorization, incarnation))
+        } else {
+            None
+        };
         if opts.lifecycle_delete_all.is_some() {
             let mut preflight_opts = opts.clone();
             preflight_opts
@@ -317,6 +328,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
                 pool.delete_object(bucket, object, preflight_opts.clone()).await?;
@@ -326,6 +340,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .lock()
                 .mark_mutation_started();
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.mark_mutation_started(bucket, incarnation, object)?;
+            }
             let mut non_trigger_opts = opts.clone();
             non_trigger_opts
                 .lifecycle_delete_all
@@ -333,6 +350,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::History;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
                 let mut pool_opts = non_trigger_opts.clone();
@@ -348,6 +368,9 @@ impl ECStore {
                 .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
             let mut trigger_pools = Vec::new();
             for (pool_index, pool) in self.pools.iter().enumerate() {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
                 let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
@@ -366,6 +389,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
             for pool_index in trigger_pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
                 let mut pool_opts = trigger_opts.clone();
@@ -378,7 +404,13 @@ impl ECStore {
         let mut first_error = None;
         let mut first_volume_error = None;
         let mut has_success = false;
+        if let Some((authorization, incarnation)) = dispatch_scope {
+            authorization.mark_mutation_started(bucket, incarnation, object)?;
+        }
         for pool in &self.pools {
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.ensure_current(bucket, incarnation, object)?;
+            }
             let mut opts = opts.clone();
             opts.delete_prefix = true;
             match pool.delete_object(bucket, object, opts).await {

crates/filemeta/src/filemeta/version.rs

@@ -3235,16 +3235,17 @@ pub fn merge_file_meta_versions(
     requested_versions: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
-    merge_file_meta_versions_inner(quorum, strict, requested_versions, false, versions)
+    merge_file_meta_versions_inner(quorum, strict, requested_versions, false, 0, versions)
 }
 
 pub(crate) fn merge_file_meta_versions_with_write_quorum(
     quorum: usize,
     strict: bool,
     requested_versions: usize,
+    write_quorum_slack: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
-    merge_file_meta_versions_inner(quorum, strict, requested_versions, true, versions)
+    merge_file_meta_versions_inner(quorum, strict, requested_versions, true, write_quorum_slack, versions)
 }
 
 fn merge_file_meta_versions_inner(
@@ -3252,6 +3253,7 @@ fn merge_file_meta_versions_inner(
     mut strict: bool,
     requested_versions: usize,
     enforce_write_quorum: bool,
+    write_quorum_slack: usize,
     versions: &[Vec<FileMetaShallowVersion>],
 ) -> Vec<FileMetaShallowVersion> {
     if quorum == 0 {
@@ -3269,7 +3271,7 @@ fn merge_file_meta_versions_inner(
 
         let required_quorum = versions[0]
             .first()
-            .map(|version| version.write_quorum(quorum).max(quorum))
+            .map(|version| version.write_quorum(quorum).saturating_sub(write_quorum_slack).max(quorum))
             .unwrap_or(quorum);
         if versions.len() >= required_quorum {
             return versions[0].clone();
@@ -3283,7 +3285,7 @@ fn merge_file_meta_versions_inner(
 
     let required_quorum = |version: &FileMetaShallowVersion| {
         if enforce_write_quorum {
-            version.write_quorum(quorum).max(quorum)
+            version.write_quorum(quorum).saturating_sub(write_quorum_slack).max(quorum)
         } else {
             quorum
         }

crates/filemeta/src/metacache.rs

@@ -53,6 +53,12 @@ pub struct MetadataResolutionParams {
     pub requested_versions: usize,
     pub bucket: String,
     pub strict: bool,
+    /// Number of set drives that were unreachable when the listing snapshot was
+    /// taken. Write-quorum enforcement relaxes each version's required quorum by
+    /// this amount (never below `obj_quorum`): a version legally committed at
+    /// write quorum can have that many of its metadata copies on drives no
+    /// reader could consult, and must not be dropped for it.
+    pub write_quorum_slack: usize,
     pub candidates: Vec<Vec<FileMetaShallowVersion>>,
 }
 
@@ -416,6 +422,7 @@ impl MetaCacheEntries {
             requested_versions: 0,
             bucket: bucket.to_string(),
             strict: false,
+            write_quorum_slack: 0,
             candidates: Vec::new(),
         })
     }
@@ -693,7 +700,12 @@ impl MetaCacheEntries {
                     .cached
                     .as_ref()
                     .and_then(|cached| cached.versions.first())
-                    .map(|version| version.write_quorum(params.obj_quorum).max(params.obj_quorum))
+                    .map(|version| {
+                        version
+                            .write_quorum(params.obj_quorum)
+                            .saturating_sub(params.write_quorum_slack)
+                            .max(params.obj_quorum)
+                    })
                     .unwrap_or(params.obj_quorum)
             } else {
                 params.obj_quorum
@@ -720,6 +732,7 @@ impl MetaCacheEntries {
                 params.obj_quorum,
                 params.strict,
                 params.requested_versions,
+                params.write_quorum_slack,
                 &params.candidates,
             )
         } else {
@@ -2415,6 +2428,97 @@ mod tests {
         assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
     }
 
+    #[test]
+    fn resolve_with_write_quorum_relaxes_requirement_by_unreachable_drives() {
+        let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // EC 2+2 version, write quorum 3 of 4: committed with one holder drive
+        // now offline, so only 2 of the 3 reachable drives carry the metadata.
+        let entry = metacache_entry_with_erasure(mod_time, "etag", 2, 2);
+        let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]);
+
+        let resolved = entries()
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed version should resolve when the missing copy sits on an unreachable drive");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(mod_time));
+
+        // Without the slack the same sample is rejected outright.
+        let rejected = entries().resolve_with_write_quorum(MetadataResolutionParams {
+            obj_quorum: 3,
+            requested_versions: 1,
+            bucket: "bucket".to_string(),
+            strict: true,
+            ..Default::default()
+        });
+        assert!(rejected.is_none());
+    }
+
+    #[test]
+    fn resolve_with_write_quorum_slack_accepts_committed_latest_during_merge() {
+        let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // The newer EC 2+2 version committed at write quorum 3 of 4; one holder
+        // drive is offline, and a third reachable drive still carries only the
+        // previous version. The candidates disagree, so the merge path (not the
+        // all-agree branch) must honor the slack.
+        let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2);
+        let new_and_old_entry =
+            metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]);
+
+        let resolved = MetaCacheEntries(vec![Some(new_and_old_entry.clone()), Some(new_and_old_entry), Some(old_entry)])
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed latest should survive the merge when its missing copy is on an unreachable drive");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(new_mod_time));
+        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new-etag"));
+    }
+
+    #[test]
+    fn resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge() {
+        let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        // The newer EC 2+2 version (write quorum 3) reached only ONE drive: even
+        // with one drive unreachable (slack 1) it cannot prove the relaxed
+        // quorum of 2, so the committed previous version must win the merge.
+        let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2);
+        let new_and_old_entry =
+            metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]);
+
+        let resolved = MetaCacheEntries(vec![Some(new_and_old_entry), Some(old_entry.clone()), Some(old_entry)])
+            .resolve_with_write_quorum(MetadataResolutionParams {
+                obj_quorum: 2,
+                requested_versions: 1,
+                bucket: "bucket".to_string(),
+                strict: true,
+                write_quorum_slack: 1,
+                ..Default::default()
+            })
+            .expect("committed previous version should resolve after rejecting the partial latest");
+        let info = resolved
+            .to_fileinfo("bucket")
+            .expect("resolved committed metadata should decode as file info");
+        assert_eq!(info.mod_time, Some(old_mod_time));
+        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
+    }
+
     #[test]
     fn resolve_rejects_partial_directory_below_dir_quorum() {
         let partial_dir = metacache_dir_entry("prefix/");

crates/heal/src/heal/channel.rs

@@ -697,6 +697,11 @@ impl HealChannelProcessor {
             | HealRequestSource::Mrf => true,
         });
 
+        // `no_lock` is an internal coordination hint. An admin request can
+        // carry the legacy field over the wire, but cannot use it as ambient
+        // authority to bypass storage namespace locking.
+        let no_lock = request.no_lock.unwrap_or(false) && request.source != HealRequestSource::Admin;
+
         // Build HealOptions with all available fields
         let options = HealOptions {
             scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
@@ -705,7 +710,7 @@ impl HealChannelProcessor {
             update_parity: request.update_parity.unwrap_or(true),
             recursive,
             dry_run: request.dry_run.unwrap_or(false),
-            no_lock: request.no_lock.unwrap_or(false),
+            no_lock,
             timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
             pool_index: request.pool_index,
             set_index: request.set_index,
@@ -990,6 +995,23 @@ mod tests {
         assert!(heal_request.options.no_lock);
     }
 
+    #[tokio::test]
+    async fn test_convert_to_heal_request_admin_cannot_bypass_object_lock() {
+        let heal_manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(heal_manager);
+        let channel_request = HealChannelRequest {
+            id: "admin-no-lock".to_string(),
+            bucket: "test-bucket".to_string(),
+            object_prefix: Some("test-object".to_string()),
+            no_lock: Some(true),
+            source: HealRequestSource::Admin,
+            ..Default::default()
+        };
+
+        let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
+        assert!(!heal_request.options.no_lock, "admin nolock must not become storage lock authority");
+    }
+
     #[tokio::test]
     async fn test_convert_to_heal_request_scanner_defaults_recreate_missing_false() {
         let heal_manager = create_test_heal_manager();

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -492,6 +492,7 @@ impl HealTask {
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
+            Err(e) if e.is_recoverable_heal() => Err(e),
             Err(e) => {
                 error!(
                     target: "rustfs::heal::task",

crates/heal/src/heal/task/tests.rs

@@ -2428,6 +2428,37 @@ async fn erasure_set_format_slowdown_is_propagated() {
     assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
 }
 
+#[tokio::test]
+async fn erasure_set_retry_signal_remains_typed_across_task_boundary() {
+    let temp = TempDir::new().expect("temporary directory should be created");
+    let disk = make_resume_disk(&temp).await;
+    let storage = Arc::new(MockStorage {
+        heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::RetryableSlowDown)),
+        resume_disk: Mutex::new(Some(disk)),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::ErasureSet {
+            buckets: vec!["bucket-a".to_string()],
+            set_disk_id: "pool_0_set_0".to_string(),
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage);
+
+    let error = task
+        .execute()
+        .await
+        .expect_err("an incomplete resumable pass must remain retryable");
+
+    assert!(
+        matches!(&error, Error::TransientSkip { message } if message.contains("retry scheduled")),
+        "the resumable retry signal must keep its typed identity: {error}"
+    );
+    assert!(error.is_recoverable_heal(), "the scheduler must accept the preserved retry signal");
+}
+
 #[tokio::test]
 async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let temp = TempDir::new().expect("temporary directory should be created");

crates/license/Cargo.toml

@@ -0,0 +1,29 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-license"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "License and entitlement provider contracts for RustFS"
+
+[lints]
+workspace = true
+
+[dependencies]
+thiserror = { workspace = true }

crates/license/src/lib.rs

@@ -0,0 +1,133 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Provider-neutral license and entitlement contracts.
+
+use std::{fmt, sync::Arc};
+
+use thiserror::Error;
+
+pub type LicenseResult<T> = Result<T, LicenseError>;
+pub type SharedLicenseProvider = Arc<dyn LicenseProvider>;
+
+/// Entitlement required by the existing server-wide license gate.
+pub const SERVER_ENTITLEMENT: &str = "rustfs.server";
+
+/// Provider-neutral metadata exposed through existing RustFS status APIs.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct LicenseMetadata {
+    pub subject: String,
+    pub expires_at: Option<u64>,
+}
+
+/// Sanitized provider state suitable for status and diagnostics output.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub enum LicenseStatus {
+    #[default]
+    Uninitialized,
+    Valid,
+    Missing,
+    Invalid(String),
+    Unavailable,
+}
+
+impl fmt::Display for LicenseStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Uninitialized => write!(f, "uninitialized"),
+            Self::Valid => write!(f, "valid"),
+            Self::Missing => write!(f, "missing"),
+            Self::Invalid(message) => write!(f, "{message}"),
+            Self::Unavailable => write!(f, "unavailable"),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum LicenseError {
+    #[error("License state is unavailable")]
+    StatePoisoned,
+    #[error("License is required when building with feature `license`.")]
+    Missing,
+    #[error("Incorrect license, please contact RustFS. {0}")]
+    Invalid(String),
+    #[error("Incorrect license, please contact RustFS. expired_at={expired_at}, now={now}")]
+    Expired { expired_at: u64, now: u64 },
+    #[error("Failed to read system time: {0}")]
+    Clock(String),
+    #[error("Entitlement is not granted: {entitlement}")]
+    Denied { entitlement: String },
+    #[error("License provider is unavailable: {0}")]
+    Unavailable(String),
+    #[error("Entitlement identifier is empty or not normalized")]
+    InvalidEntitlement,
+}
+
+/// Runtime boundary between RustFS and a license implementation.
+///
+/// Providers must sanitize all strings returned in errors, status, and
+/// metadata. In particular, they must never include raw license material.
+/// `initialize` must be safe to call repeatedly with the same input. `check`
+/// must be idempotent and must not consume quota, acquire a lease, or mutate
+/// external state. Providers that require a license must fail closed before
+/// successful initialization.
+pub trait LicenseProvider: Send + Sync {
+    fn initialize(&self, raw_license: Option<&str>) -> LicenseResult<()>;
+
+    fn check(&self, entitlement: &str) -> LicenseResult<()>;
+
+    fn status(&self) -> LicenseStatus;
+
+    fn metadata(&self) -> Option<LicenseMetadata> {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug)]
+    struct TestProvider;
+
+    impl LicenseProvider for TestProvider {
+        fn initialize(&self, _raw_license: Option<&str>) -> LicenseResult<()> {
+            Ok(())
+        }
+
+        fn check(&self, _entitlement: &str) -> LicenseResult<()> {
+            Ok(())
+        }
+
+        fn status(&self) -> LicenseStatus {
+            LicenseStatus::Valid
+        }
+    }
+
+    #[test]
+    fn provider_is_object_safe() {
+        let provider: SharedLicenseProvider = Arc::new(TestProvider);
+
+        assert_eq!(provider.status(), LicenseStatus::Valid);
+    }
+
+    #[test]
+    fn status_display_is_stable() {
+        assert_eq!(LicenseStatus::Uninitialized.to_string(), "uninitialized");
+        assert_eq!(LicenseStatus::Valid.to_string(), "valid");
+        assert_eq!(LicenseStatus::Missing.to_string(), "missing");
+        assert_eq!(LicenseStatus::Invalid("invalid key".to_string()).to_string(), "invalid key");
+        assert_eq!(LicenseStatus::Unavailable.to_string(), "unavailable");
+    }
+}

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1521,6 +1521,8 @@ pub struct TierMutationControlResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
     #[prost(bytes = "bytes", tag = "5")]
     pub response_proof: ::prost::bytes::Bytes,
+    #[prost(enumeration = "TierMutationFailureClass", tag = "6")]
+    pub failure_class: i32,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetLiveEventsRequest {
@@ -1611,6 +1613,35 @@ impl TierMutationPeerState {
         }
     }
 }
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum TierMutationFailureClass {
+    Unspecified = 0,
+    PreDispatchRejected = 1,
+    Ambiguous = 2,
+}
+impl TierMutationFailureClass {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Unspecified => "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED",
+            Self::PreDispatchRejected => "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED",
+            Self::Ambiguous => "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED" => Some(Self::Unspecified),
+            "TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED" => Some(Self::PreDispatchRejected),
+            "TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS" => Some(Self::Ambiguous),
+            _ => None,
+        }
+    }
+}
 /// Generated client implementations.
 pub mod node_service_client {
     #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]

crates/protos/src/lib.rs

@@ -177,6 +177,7 @@ pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-re
 pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
 pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
 pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
+pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
 
 pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
@@ -342,7 +343,16 @@ impl TierMutationRpcPhase {
     }
 }
 
-pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
+// Version 2 required peer Prepare to block new tier-reference creators and
+// drain their in-flight operation leases. Version 3 additionally binds Abort
+// to the canonical Prepare intent so a missing-record Abort can persist an
+// identity-bound tombstone and linearize against a delayed Prepare. Version 4
+// signs a typed failure classification while retaining the exact v3 proof
+// bytes for rolling compatibility.
+pub const TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION: u32 = 3;
+pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 4;
+pub const TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE: usize = 1024;
+pub const TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE: usize = 4096;
 
 pub fn canonical_tier_mutation_rpc_body(
     version: u32,
@@ -374,19 +384,23 @@ pub struct TierMutationRpcResponseProofInput<'a> {
     pub state: i32,
     pub applied: bool,
     pub error_info: Option<&'a str>,
+    pub failure_class: i32,
 }
 
 pub fn canonical_tier_mutation_rpc_response_body(
     input: TierMutationRpcResponseProofInput<'_>,
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
+    const V3_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
+    const V4_DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v2\0";
 
     let phase = input.phase.as_wire_str().as_bytes();
     let mutation_id = input.mutation_id.as_bytes();
     let error_info = input.error_info.map(str::as_bytes);
     let error_info_len = error_info.map_or(0, <[u8]>::len);
+    let is_v4 = input.version >= TIER_MUTATION_RPC_PROTOCOL_VERSION;
+    let domain = if is_v4 { V4_DOMAIN } else { V3_DOMAIN };
     let mut body = Vec::with_capacity(
-        DOMAIN.len()
+        domain.len()
             + 4
             + 8
             + phase.len()
@@ -398,9 +412,10 @@ pub fn canonical_tier_mutation_rpc_response_body(
             + 1
             + 1
             + 8
-            + error_info_len,
+            + error_info_len
+            + if is_v4 { 4 } else { 0 },
     );
-    body.extend_from_slice(DOMAIN);
+    body.extend_from_slice(domain);
     body.extend_from_slice(&input.version.to_be_bytes());
     body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
     body.extend_from_slice(phase);
@@ -415,6 +430,9 @@ pub fn canonical_tier_mutation_rpc_response_body(
     if let Some(error_info) = error_info {
         body.extend_from_slice(error_info);
     }
+    if is_v4 {
+        body.extend_from_slice(&input.failure_class.to_be_bytes());
+    }
     Ok(body)
 }
 
@@ -2137,10 +2155,10 @@ mod heal_control_tests {
 #[cfg(test)]
 mod tier_mutation_rpc_tests {
     use super::{
-        TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
-        canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
+        TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase,
+        TierMutationRpcResponseProofInput, canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
     };
-    use crate::proto_gen::node_service::TierMutationPeerState;
+    use crate::proto_gen::node_service::{TierMutationFailureClass, TierMutationPeerState};
     use uuid::uuid;
 
     #[test]
@@ -2155,7 +2173,7 @@ mod tier_mutation_rpc_tests {
         )
         .expect("small mutation body should encode");
         let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
-        golden.extend_from_slice(&1_u32.to_be_bytes());
+        golden.extend_from_slice(&TIER_MUTATION_RPC_PROTOCOL_VERSION.to_be_bytes());
         golden.extend_from_slice(&7_u64.to_be_bytes());
         golden.extend_from_slice(b"prepare");
         golden.extend_from_slice(mutation_id.as_bytes());
@@ -2165,8 +2183,13 @@ mod tier_mutation_rpc_tests {
 
         assert_ne!(
             baseline,
-            canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
-                .expect("small mutation body should encode")
+            canonical_tier_mutation_rpc_body(
+                TIER_MUTATION_RPC_PROTOCOL_VERSION + 1,
+                TierMutationRpcPhase::Prepare,
+                mutation_id,
+                payload,
+            )
+            .expect("small mutation body should encode")
         );
         assert_ne!(
             baseline,
@@ -2201,11 +2224,27 @@ mod tier_mutation_rpc_tests {
     }
 
     #[test]
-    fn canonical_tier_mutation_response_binds_request_state_and_error() {
+    fn tier_mutation_v3_request_and_response_golden_bytes_are_unchanged() {
         let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
         let payload = b"canonical-intent-record";
-        let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
-            version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+        let request = canonical_tier_mutation_rpc_body(
+            TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+        )
+        .expect("v3 request should encode");
+        let mut request_golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
+        request_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
+        request_golden.extend_from_slice(&7_u64.to_be_bytes());
+        request_golden.extend_from_slice(b"prepare");
+        request_golden.extend_from_slice(mutation_id.as_bytes());
+        request_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
+        request_golden.extend_from_slice(payload);
+        assert_eq!(request, request_golden);
+
+        let response = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
+            version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
             phase: TierMutationRpcPhase::Prepare,
             mutation_id,
             canonical_payload: payload,
@@ -2213,49 +2252,97 @@ mod tier_mutation_rpc_tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
+            // v3 must ignore the field so its authenticated bytes stay exact.
+            failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
         })
-        .expect("small mutation response should encode");
+        .expect("v3 response should encode");
+        let mut response_golden = b"rustfs-tier-mutation-rpc-response-v1\0".to_vec();
+        response_golden.extend_from_slice(&TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION.to_be_bytes());
+        response_golden.extend_from_slice(&7_u64.to_be_bytes());
+        response_golden.extend_from_slice(b"prepare");
+        response_golden.extend_from_slice(mutation_id.as_bytes());
+        response_golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
+        response_golden.extend_from_slice(payload);
+        response_golden.push(1);
+        response_golden.extend_from_slice(&(TierMutationPeerState::Prepared as i32).to_be_bytes());
+        response_golden.push(1);
+        response_golden.push(0);
+        response_golden.extend_from_slice(&0_u64.to_be_bytes());
+        assert_eq!(response, response_golden);
+    }
+
+    #[test]
+    fn canonical_tier_mutation_v4_response_binds_request_result_and_failure_class() {
+        let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
+        let payload = b"canonical-intent-record";
+        let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
+            version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            phase: TierMutationRpcPhase::Prepare,
+            mutation_id,
+            canonical_payload: payload,
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: Some("failure"),
+            failure_class: TierMutationFailureClass::Ambiguous as i32,
+        })
+        .expect("small v4 mutation response should encode");
 
         let cases = [
             TierMutationRpcResponseProofInput {
-                version: 2,
+                version: TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Commit,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: b"tampered-intent-record",
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
                 success: true,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -2263,39 +2350,43 @@ mod tier_mutation_rpc_tests {
                 mutation_id,
                 canonical_payload: payload,
                 success: false,
-                state: TierMutationPeerState::Prepared as i32,
-                applied: true,
-                error_info: None,
-            },
-            TierMutationRpcResponseProofInput {
-                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
-                phase: TierMutationRpcPhase::Prepare,
-                mutation_id,
-                canonical_payload: payload,
-                success: true,
                 state: TierMutationPeerState::Committed as i32,
-                applied: true,
-                error_info: None,
-            },
-            TierMutationRpcResponseProofInput {
-                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
-                phase: TierMutationRpcPhase::Prepare,
-                mutation_id,
-                canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
                 applied: false,
-                error_info: None,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
             },
             TierMutationRpcResponseProofInput {
                 version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
                 phase: TierMutationRpcPhase::Prepare,
                 mutation_id,
                 canonical_payload: payload,
-                success: true,
-                state: TierMutationPeerState::Prepared as i32,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
                 applied: true,
-                error_info: Some("error"),
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("other failure"),
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+            },
+            TierMutationRpcResponseProofInput {
+                version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
+                phase: TierMutationRpcPhase::Prepare,
+                mutation_id,
+                canonical_payload: payload,
+                success: false,
+                state: TierMutationPeerState::Unspecified as i32,
+                applied: false,
+                error_info: Some("failure"),
+                failure_class: TierMutationFailureClass::PreDispatchRejected as i32,
             },
         ];
         for case in cases {

crates/protos/src/node.proto

@@ -1068,12 +1068,19 @@ enum TierMutationPeerState {
   TIER_MUTATION_PEER_STATE_ABORTED = 3;
 }
 
+enum TierMutationFailureClass {
+  TIER_MUTATION_FAILURE_CLASS_UNSPECIFIED = 0;
+  TIER_MUTATION_FAILURE_CLASS_PRE_DISPATCH_REJECTED = 1;
+  TIER_MUTATION_FAILURE_CLASS_AMBIGUOUS = 2;
+}
+
 message TierMutationControlResponse {
   bool success = 1;
   TierMutationPeerState state = 2;
   bool applied = 3;
   optional string error_info = 4;
   bytes response_proof = 5;
+  TierMutationFailureClass failure_class = 6;
 }
 
 message GetLiveEventsRequest {

crates/s3select-api/src/input_stream.rs

@@ -326,13 +326,13 @@ fn invalid_gzip_header_error() -> io::Error {
     io::Error::new(io::ErrorKind::InvalidData, CompressionFormat::Gzip.invalid_header_error())
 }
 
-struct BlockingChannelReader {
+pub(crate) struct BlockingChannelReader {
     receiver: mpsc::Receiver<io::Result<Bytes>>,
     current: Bytes,
 }
 
 impl BlockingChannelReader {
-    fn new(receiver: mpsc::Receiver<io::Result<Bytes>>) -> Self {
+    pub(crate) fn new(receiver: mpsc::Receiver<io::Result<Bytes>>) -> Self {
         Self {
             receiver,
             current: Bytes::new(),
@@ -639,7 +639,7 @@ fn error_chain_contains<T: StdError + 'static>(error: &(dyn StdError + 'static))
     find_error_source::<T>(error).is_some()
 }
 
-fn find_error_source<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> {
+pub(crate) fn find_error_source<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> {
     let mut current = Some(error);
     for _ in 0..MAX_ERROR_SOURCE_DEPTH {
         let Some(error) = current else {

crates/s3select-api/src/metrics.rs

@@ -87,7 +87,7 @@ impl SelectInputMetricsRecorder {
 
 fn saturating_add(counter: &AtomicU64, bytes: usize) {
     let increment = u64::try_from(bytes).unwrap_or(u64::MAX);
-    let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(increment)));
+    let _ = counter.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(increment)));
 }
 
 #[cfg(test)]

crates/s3select-query/Cargo.toml

@@ -51,6 +51,8 @@ derive_builder = { workspace = true }
 futures = { workspace = true }
 parking_lot = { workspace = true }
 s3s = { workspace = true, features = ["minio"] }
+serde.workspace = true
+serde_json.workspace = true
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
 tracing = { workspace = true }
 

crates/s3select-query/src/dispatcher/json_document_schema.rs

@@ -0,0 +1,494 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use datafusion::{
+    arrow::{
+        datatypes::{DataType, Field, Schema},
+        error::ArrowError,
+    },
+    common::{DataFusionError, Result as DFResult},
+};
+use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor};
+use std::{
+    collections::HashMap,
+    fmt,
+    io::{self, Cursor},
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    },
+};
+
+const CANCELLATION_CHECK_ITEMS: usize = 1024;
+const READER_BUFFER_BYTES: usize = 64 * 1024;
+
+pub(super) fn infer_schema(
+    bytes: &[u8],
+    expected_records: usize,
+    cancellation: &AtomicBool,
+    max_schema_bytes: usize,
+) -> DFResult<(Schema, usize, usize)> {
+    let mut context = InferenceContext {
+        cancellation,
+        items_since_check: 0,
+    };
+    let mut fields = InferredObject::default();
+    let mut records = 0;
+
+    for row in bytes.split_inclusive(|byte| *byte == b'\n') {
+        if row.is_empty() {
+            continue;
+        }
+        context.check_now()?;
+        let reader = io::BufReader::with_capacity(
+            READER_BUFFER_BYTES,
+            CancellableReader {
+                inner: Cursor::new(row),
+                cancellation,
+            },
+        );
+        let mut deserializer = serde_json::Deserializer::from_reader(reader);
+        let inferred = InferSeed { context: &mut context }
+            .deserialize(&mut deserializer)
+            .map_err(|error| schema_parse_error(error, cancellation))?;
+        deserializer.end().map_err(|error| schema_parse_error(error, cancellation))?;
+        let InferredType::Object(record) = inferred else {
+            return Err(DataFusionError::Execution(
+                "JSON DOCUMENT schema inference expected an object row".to_string(),
+            ));
+        };
+        if record.retained_size() > max_schema_bytes {
+            return Err(schema_complexity_error());
+        }
+        fields.merge(record, &mut context)?;
+        if fields.retained_size() > max_schema_bytes {
+            return Err(schema_complexity_error());
+        }
+        records += 1;
+    }
+
+    if records != expected_records {
+        return Err(DataFusionError::Execution(format!(
+            "JSON DOCUMENT schema prefix contained {expected_records} records but inference read {records}"
+        )));
+    }
+    let schema = Schema::new(fields.into_fields(&mut context)?);
+    let schema_bytes = std::mem::size_of::<Schema>()
+        .checked_add(schema.fields().size())
+        .ok_or_else(schema_complexity_error)?;
+    if schema_bytes > max_schema_bytes {
+        return Err(schema_complexity_error());
+    }
+    Ok((schema, records, schema_bytes))
+}
+
+fn schema_parse_error(error: serde_json::Error, cancellation: &AtomicBool) -> DataFusionError {
+    if cancellation.load(Ordering::Acquire) {
+        DataFusionError::Execution("JSON DOCUMENT schema inference canceled".to_string())
+    } else {
+        schema_json_error(format!("JSON DOCUMENT schema inference failed: {error}"))
+    }
+}
+
+fn schema_json_error(message: impl Into<String>) -> DataFusionError {
+    DataFusionError::ArrowError(Box::new(ArrowError::JsonError(message.into())), None)
+}
+
+fn schema_complexity_error() -> DataFusionError {
+    DataFusionError::ResourcesExhausted("JSON DOCUMENT schema exceeds the inference complexity limit".to_string())
+}
+
+struct CancellableReader<'a> {
+    inner: Cursor<&'a [u8]>,
+    cancellation: &'a AtomicBool,
+}
+
+impl io::Read for CancellableReader<'_> {
+    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+        if self.cancellation.load(Ordering::Acquire) {
+            return Err(io::Error::new(io::ErrorKind::Interrupted, "JSON DOCUMENT schema inference canceled"));
+        }
+        io::Read::read(&mut self.inner, buffer)
+    }
+}
+
+struct InferenceContext<'a> {
+    cancellation: &'a AtomicBool,
+    items_since_check: usize,
+}
+
+impl InferenceContext<'_> {
+    fn checkpoint<E: de::Error>(&mut self) -> Result<(), E> {
+        self.items_since_check += 1;
+        if self.items_since_check < CANCELLATION_CHECK_ITEMS {
+            return Ok(());
+        }
+        self.items_since_check = 0;
+        if self.cancellation.load(Ordering::Acquire) {
+            Err(E::custom("JSON DOCUMENT schema inference canceled"))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn check_now(&self) -> DFResult<()> {
+        if self.cancellation.load(Ordering::Acquire) {
+            Err(DataFusionError::Execution("JSON DOCUMENT schema inference canceled".to_string()))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn checkpoint_df(&mut self) -> DFResult<()> {
+        self.items_since_check += 1;
+        if self.items_since_check < CANCELLATION_CHECK_ITEMS {
+            return Ok(());
+        }
+        self.items_since_check = 0;
+        self.check_now()
+    }
+}
+
+struct InferSeed<'a, 'b> {
+    context: &'a mut InferenceContext<'b>,
+}
+
+impl<'de> DeserializeSeed<'de> for InferSeed<'_, '_> {
+    type Value = InferredType;
+
+    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        deserializer.deserialize_any(InferVisitor { context: self.context })
+    }
+}
+
+struct InferVisitor<'a, 'b> {
+    context: &'a mut InferenceContext<'b>,
+}
+
+impl<'de> Visitor<'de> for InferVisitor<'_, '_> {
+    type Value = InferredType;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a JSON value")
+    }
+
+    fn visit_unit<E>(self) -> Result<Self::Value, E> {
+        Ok(InferredType::Any)
+    }
+
+    fn visit_none<E>(self) -> Result<Self::Value, E> {
+        Ok(InferredType::Any)
+    }
+
+    fn visit_bool<E>(self, _value: bool) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(ScalarKinds::BOOLEAN))
+    }
+
+    fn visit_i64<E>(self, _value: i64) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(ScalarKinds::INTEGER))
+    }
+
+    fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(if i64::try_from(value).is_ok() {
+            ScalarKinds::INTEGER
+        } else {
+            ScalarKinds::FLOAT
+        }))
+    }
+
+    fn visit_f64<E>(self, _value: f64) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(ScalarKinds::FLOAT))
+    }
+
+    fn visit_str<E>(self, _value: &str) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(ScalarKinds::STRING))
+    }
+
+    fn visit_string<E>(self, _value: String) -> Result<Self::Value, E> {
+        Ok(InferredType::Scalar(ScalarKinds::STRING))
+    }
+
+    fn visit_seq<A>(self, mut sequence: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        let mut inner = InferredType::Any;
+        let mut first = true;
+        while let Some(value) = sequence.next_element_seed(InferSeed { context: self.context })? {
+            self.context.checkpoint()?;
+            if first && matches!(value, InferredType::Any) {
+                inner = InferredType::Scalar(ScalarKinds::default());
+            } else {
+                inner.merge_array_element(value, self.context).map_err(de::Error::custom)?;
+            }
+            first = false;
+        }
+        Ok(InferredType::Array(Box::new(inner)))
+    }
+
+    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+    where
+        A: MapAccess<'de>,
+    {
+        let mut object = InferredObject::default();
+        while let Some(key) = map.next_key::<String>()? {
+            self.context.checkpoint()?;
+            let value = map.next_value_seed(InferSeed { context: self.context })?;
+            object.insert_record_field(key, value);
+        }
+        Ok(InferredType::Object(object))
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct ScalarKinds(u8);
+
+impl ScalarKinds {
+    const BOOLEAN: Self = Self(1 << 0);
+    const INTEGER: Self = Self(1 << 1);
+    const FLOAT: Self = Self(1 << 2);
+    const STRING: Self = Self(1 << 3);
+
+    fn merge(&mut self, other: Self) {
+        self.0 |= other.0;
+    }
+
+    fn data_type(self) -> DataType {
+        match self.0 {
+            value if value == Self::BOOLEAN.0 => DataType::Boolean,
+            value if value == Self::INTEGER.0 => DataType::Int64,
+            value if value == Self::FLOAT.0 || value == (Self::INTEGER.0 | Self::FLOAT.0) => DataType::Float64,
+            value if value == Self::STRING.0 => DataType::Utf8,
+            _ => DataType::Utf8,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum InferredType {
+    Scalar(ScalarKinds),
+    Array(Box<InferredType>),
+    Object(InferredObject),
+    Any,
+}
+
+impl InferredType {
+    fn merge(&mut self, other: Self, context: &mut InferenceContext<'_>) -> DFResult<()> {
+        match (self, other) {
+            (Self::Array(current), Self::Array(other)) => current.merge(*other, context),
+            (Self::Scalar(current), Self::Scalar(other)) => {
+                current.merge(other);
+                Ok(())
+            }
+            (Self::Object(current), Self::Object(other)) => current.merge(other, context),
+            (current @ Self::Any, value) => {
+                *current = value;
+                Ok(())
+            }
+            (_, Self::Any) => Ok(()),
+            (Self::Array(inner), scalar @ Self::Scalar(_)) => inner.merge(scalar, context),
+            (current @ Self::Scalar(_), Self::Array(mut inner)) => {
+                let scalar = std::mem::replace(current, Self::Any);
+                inner.merge(scalar, context)?;
+                *current = Self::Array(inner);
+                Ok(())
+            }
+            (current, other) => Err(schema_json_error(format!(
+                "incompatible JSON types during schema inference: {} and {}",
+                current.kind(),
+                other.kind()
+            ))),
+        }
+    }
+
+    fn merge_array_element(&mut self, other: Self, context: &mut InferenceContext<'_>) -> DFResult<()> {
+        match (self, other) {
+            (current @ Self::Any, value) => {
+                *current = value;
+                Ok(())
+            }
+            (_, Self::Any) => Ok(()),
+            (Self::Scalar(current), Self::Scalar(other)) => {
+                current.merge(other);
+                Ok(())
+            }
+            (Self::Object(current), Self::Object(other)) => current.merge(other, context),
+            (Self::Array(current), Self::Array(other)) => current.merge(*other, context),
+            (current, other) => Err(schema_json_error(format!(
+                "incompatible JSON array elements during schema inference: {} and {}",
+                current.kind(),
+                other.kind()
+            ))),
+        }
+    }
+
+    fn kind(&self) -> &'static str {
+        match self {
+            Self::Scalar(_) => "scalar",
+            Self::Array(_) => "array",
+            Self::Object(_) => "object",
+            Self::Any => "null",
+        }
+    }
+
+    fn into_data_type(self, context: &mut InferenceContext<'_>) -> DFResult<DataType> {
+        context.check_now()?;
+        match self {
+            Self::Scalar(kinds) => Ok(kinds.data_type()),
+            Self::Array(inner) => Ok(DataType::List(Arc::new(Field::new_list_field(inner.into_data_type(context)?, true)))),
+            Self::Object(object) => Ok(DataType::Struct(object.into_fields(context)?.into())),
+            Self::Any => Ok(DataType::Null),
+        }
+    }
+
+    fn retained_size(&self) -> usize {
+        std::mem::size_of::<Self>()
+            + match self {
+                Self::Array(inner) => inner.retained_size(),
+                Self::Object(object) => object.retained_size(),
+                Self::Scalar(_) | Self::Any => 0,
+            }
+    }
+}
+
+#[derive(Debug, Default)]
+struct InferredObject {
+    indexes: HashMap<Arc<str>, usize>,
+    fields: Vec<(Arc<str>, InferredType)>,
+}
+
+impl InferredObject {
+    fn insert_record_field(&mut self, key: String, value: InferredType) {
+        if let Some(index) = self.indexes.get(key.as_str()).copied() {
+            self.fields[index].1 = value;
+            return;
+        }
+        let key: Arc<str> = Arc::from(key);
+        let index = self.fields.len();
+        self.indexes.insert(Arc::clone(&key), index);
+        self.fields.push((key, value));
+    }
+
+    fn merge(&mut self, other: Self, context: &mut InferenceContext<'_>) -> DFResult<()> {
+        for (key, value) in other.fields {
+            context.checkpoint_df()?;
+            if let Some(index) = self.indexes.get(key.as_ref()).copied() {
+                self.fields[index].1.merge(value, context)?;
+            } else {
+                let index = self.fields.len();
+                self.indexes.insert(Arc::clone(&key), index);
+                self.fields.push((key, value));
+            }
+        }
+        Ok(())
+    }
+
+    fn into_fields(self, context: &mut InferenceContext<'_>) -> DFResult<Vec<Field>> {
+        self.fields
+            .into_iter()
+            .map(|(name, inferred)| {
+                context.check_now()?;
+                Ok(Field::new(name.as_ref(), inferred.into_data_type(context)?, true))
+            })
+            .collect()
+    }
+
+    fn retained_size(&self) -> usize {
+        let index_entry = std::mem::size_of::<(Arc<str>, usize)>() + 16;
+        let mut bytes = std::mem::size_of::<Self>()
+            .saturating_add(self.indexes.capacity().saturating_mul(index_entry))
+            .saturating_add(
+                self.fields
+                    .capacity()
+                    .saturating_mul(std::mem::size_of::<(Arc<str>, InferredType)>()),
+            );
+        for (name, inferred) in &self.fields {
+            bytes = bytes.saturating_add(name.len()).saturating_add(inferred.retained_size());
+        }
+        bytes
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::json::reader::infer_json_schema as arrow_infer_json_schema;
+
+    const TEST_SCHEMA_LIMIT: usize = 4 * 1024 * 1024;
+
+    fn assert_matches_arrow(input: &[u8]) {
+        let records = input.iter().filter(|byte| **byte == b'\n').count();
+        let (expected, expected_records) = arrow_infer_json_schema(io::BufReader::new(input), Some(records))
+            .expect("Arrow should infer the compatibility fixture");
+        let (actual, actual_records, _) = infer_schema(input, records, &AtomicBool::new(false), TEST_SCHEMA_LIMIT)
+            .expect("streaming inference should accept the compatibility fixture");
+        assert_eq!(actual_records, expected_records);
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn inference_matches_arrow_for_nested_and_coerced_types() {
+        assert_matches_arrow(
+            br#"{"a":1,"values":[1,2],"nested":{"enabled":true},"nullable":null}
+{"a":1.5,"values":3,"nested":{"name":"ok"},"nullable":"set"}
+"#,
+        );
+        assert_matches_arrow(
+            br#"{"matrix":[[1,2],[3]],"objects":[{"id":1},{"name":"two"}],"empty":[]}
+{"matrix":[[4.5]],"objects":[],"empty":[null]}
+"#,
+        );
+    }
+
+    #[test]
+    fn inference_matches_arrow_for_null_first_arrays_and_duplicate_keys() {
+        assert_matches_arrow(
+            br#"{"values":[null,1,2],"duplicate":1,"duplicate":"last"}
+"#,
+        );
+
+        let input = br#"{"values":[null,{"id":1}]}
+"#;
+        let records = 1;
+        assert!(arrow_infer_json_schema(io::BufReader::new(input.as_slice()), Some(records)).is_err());
+        assert!(infer_schema(input, records, &AtomicBool::new(false), TEST_SCHEMA_LIMIT).is_err());
+    }
+
+    #[test]
+    fn inference_checkpoints_observe_cancellation() {
+        let cancellation = AtomicBool::new(true);
+        let mut context = InferenceContext {
+            cancellation: &cancellation,
+            items_since_check: CANCELLATION_CHECK_ITEMS - 1,
+        };
+        let error = context
+            .checkpoint::<serde_json::Error>()
+            .expect_err("the item checkpoint must observe cancellation");
+        assert!(error.to_string().contains("schema inference canceled"));
+    }
+
+    #[test]
+    fn incompatible_types_preserve_arrow_json_error_classification() {
+        let error = infer_schema(b"{\"a\":{}}\n{\"a\":1}\n", 2, &AtomicBool::new(false), TEST_SCHEMA_LIMIT)
+            .expect_err("object and scalar fields must remain incompatible");
+
+        assert!(matches!(
+            error,
+            DataFusionError::ArrowError(source, None) if matches!(*source, ArrowError::JsonError(_))
+        ));
+    }
+}

crates/s3select-query/src/dispatcher/manager.rs

@@ -61,7 +61,7 @@ use tokio::{
 };
 
 use crate::{
-    dispatcher::parquet_table::ParquetSelectTable,
+    dispatcher::{json_document_table::JsonDocumentTable, parquet_table::ParquetSelectTable},
     execution::factory::QueryExecutionFactoryRef,
     instance::{DEFAULT_MAX_CONCURRENT_QUERIES, DEFAULT_QUERY_TIMEOUT_SECS},
     metadata::{ContextProviderExtension, MetadataProvider, TableHandleProviderRef, base_table::BaseTableProvider},
@@ -181,7 +181,9 @@ impl QueryDispatcher for SimpleQueryDispatcher {
                 Ok(logical_plan)
             })
             .await?;
-        query_state_machine.query.input_metrics().reset();
+        if !is_json_document_input(&self.input) {
+            query_state_machine.query.input_metrics().reset();
+        }
         if !query_tracker.mark_planned(&self.query_execution_owner) {
             drop(logical_plan);
             return Err(self.query_tracker_error(&query_tracker));
@@ -415,6 +417,17 @@ impl SimpleQueryDispatcher {
             return Ok(metadata_provider);
         }
 
+        if is_json_document_input(&self.input) {
+            let provider = JsonDocumentTable::try_new(session.inner(), &self.input.bucket, &self.input.key).await?;
+            let current_session_table_provider = self.build_table_handle_provider()?;
+            return Ok(MetadataProvider::new(
+                provider,
+                current_session_table_provider,
+                self.func_manager.clone(),
+                session.clone(),
+            ));
+        }
+
         let path = format!("s3://{}/{}", self.input.bucket, self.input.key);
         let table_path = ListingTableUrl::parse(path)?;
         let compressed_input = self
@@ -542,19 +555,24 @@ impl SimpleQueryDispatcher {
     }
 }
 
+fn is_json_document_input(input: &SelectObjectContentInput) -> bool {
+    input
+        .request
+        .input_serialization
+        .json
+        .as_ref()
+        .and_then(|json| json.type_.as_ref())
+        .is_some_and(|json_type| json_type.as_str() == JSONType::DOCUMENT)
+}
+
 fn validate_json_source_path_input(input: &SelectObjectContentInput, source_path: &[JsonPathSegment]) -> QueryResult<()> {
     if source_path.is_empty() {
         return Ok(());
     }
-    let Some(json) = input.request.input_serialization.json.as_ref() else {
+    if input.request.input_serialization.json.is_none() {
         return Err(SelectError::DataSourcePathUnsupported.into());
-    };
-    if !source_path_requires_expansion(source_path)
-        || json
-            .type_
-            .as_ref()
-            .is_some_and(|json_type| json_type.as_str() == JSONType::DOCUMENT)
-    {
+    }
+    if !source_path_requires_expansion(source_path) || is_json_document_input(input) {
         return Ok(());
     }
     Err(SelectError::DataSourcePathUnsupported.into())
@@ -1803,6 +1821,63 @@ mod tests {
         }
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn json_document_schema_prefix_is_counted_once() {
+        const DOCUMENT: &[u8] = br#"["one","two"]"#;
+        let mut input = test_input();
+        input.bucket = "s3select-json-document-metrics".to_string();
+        input.key = "input.json".to_string();
+        input.request.expression = "SELECT _1 FROM S3Object[*]".to_string();
+        input.request.input_serialization = InputSerialization {
+            json: Some(JSONInput {
+                type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
+            }),
+            ..Default::default()
+        };
+        input.request.output_serialization = OutputSerialization {
+            json: Some(JSONOutput::default()),
+            ..Default::default()
+        };
+        let input = Arc::new(input);
+        let env = snapshot_test_env().await;
+        env.make_bucket(&input.bucket, false).await;
+        env.put_object_bytes(&input.bucket, &input.key, DOCUMENT.to_vec()).await;
+        let snapshot = env.prepare_select_object_snapshot(&input.bucket, &input.key).await;
+        let dispatcher = production_dispatcher(Arc::clone(&input));
+        let query = Query::new_with_snapshot(
+            QueryContext {
+                input: Arc::clone(&input),
+            },
+            input.request.expression.clone(),
+            snapshot,
+        );
+        let state_machine = dispatcher
+            .build_query_state_machine(query)
+            .await
+            .expect("build tracked JSON DOCUMENT query");
+        let input_metrics = Arc::clone(state_machine.query.input_metrics());
+
+        let logical_plan = dispatcher
+            .build_logical_plan(Arc::clone(&state_machine))
+            .await
+            .expect("JSON DOCUMENT query should plan")
+            .expect("SELECT should produce a logical plan");
+        let expected_bytes = u64::try_from(DOCUMENT.len()).expect("fixture size should fit in u64");
+        assert_eq!(input_metrics.snapshot().bytes_scanned, expected_bytes);
+        assert_eq!(input_metrics.snapshot().bytes_processed, expected_bytes);
+
+        let values = collect_utf8_output(
+            dispatcher
+                .execute_logical_plan(logical_plan, state_machine)
+                .await
+                .expect("JSON DOCUMENT query should execute"),
+        )
+        .await;
+        assert_eq!(values, ["one", "two"]);
+        assert_eq!(input_metrics.snapshot().bytes_scanned, expected_bytes);
+        assert_eq!(input_metrics.snapshot().bytes_processed, expected_bytes);
+    }
+
     #[tokio::test]
     async fn staged_query_rejects_unbound_session() {
         let admission = Arc::new(Semaphore::new(1));

crates/s3select-query/src/dispatcher/mod.rs

@@ -12,5 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod json_document_schema;
+mod json_document_table;
 pub mod manager;
 mod parquet_table;

crates/scanner/src/data_usage_define.rs

@@ -131,6 +131,39 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
     }
 }
 
+pub(crate) fn usage_floor_primary_read_error_allows_backup(err: &Error) -> bool {
+    match err {
+        Error::FileCorrupt
+        | Error::CorruptedFormat
+        | Error::CorruptedBackend
+        | Error::PartMissingOrCorrupt
+        | Error::LessData
+        | Error::MoreData => true,
+        Error::Io(io_error) => {
+            matches!(io_error.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof)
+                || error_chain_has_usage_floor_corruption_signature(io_error)
+        }
+        _ => false,
+    }
+}
+
+fn error_chain_has_usage_floor_corruption_signature(error: &(dyn std::error::Error + 'static)) -> bool {
+    let mut current = Some(error);
+    while let Some(err) = current {
+        let message = err.to_string();
+        if message.contains("InlineData value out of range")
+            || message.contains("InlineData key out of range")
+            || message.contains("insufficient data for metadata")
+            || message.contains("insufficient data for meta length")
+            || message.contains("insufficient data for CRC")
+        {
+            return true;
+        }
+        current = err.source();
+    }
+    false
+}
+
 /// Read only the object revision without materializing its body.
 pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
     match store

crates/scanner/src/lib.rs

@@ -83,9 +83,9 @@ pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_st
 pub use rustfs_scanner_contracts::last_minute;
 pub use scanner::{
     ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
-    ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, init_data_scanner,
-    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status,
-    scanner_topology_digest,
+    ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
+    init_data_scanner, reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
+    scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
 };
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,

crates/scanner/src/scanner.rs

@@ -3385,7 +3385,8 @@ pub use backlog::{
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
 pub use cycle_state::{
-    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerUsageStateResetResult, reset_scanner_cycle_recovery,
+    reset_scanner_usage_state_for_full_rebuild, scanner_cycle_recovery_status,
 };
 pub(crate) use cycle_state::{
     current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,

crates/scanner/src/scanner/cycle_state.rs

@@ -14,7 +14,9 @@
 /// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence.
 use super::*;
 use crate::ScannerGetObjectReader;
-use crate::data_usage_define::{DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH};
+use crate::data_usage_define::{
+    DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup,
+};
 use crate::storage_api::owner::ObjectIO as _;
 use tokio::io::AsyncReadExt as _;
 
@@ -27,6 +29,8 @@ const USAGE_FLOOR_LOAD_FAILED: &str = "usage_floor_load_failed";
 const LEGACY_EMPTY_USAGE_FLOOR_RECOVERY: &str = "legacy_empty_usage_floor";
 const CACHE_CYCLE_AHEAD: &str = "cache_cycle_ahead";
 
+const SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD: &str = "full-rebuild";
+
 #[derive(Clone, Debug, Default, Serialize)]
 pub struct ScannerCycleRecoveryStatus {
     /// The immutable primary object whose revision is being guarded.
@@ -48,6 +52,23 @@ pub struct ScannerCycleRecoveryStatus {
     pub reason: Option<String>,
 }
 
+#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
+pub struct ScannerUsageStateResetResult {
+    pub status: String,
+    pub mode: String,
+    pub usage_state: String,
+    pub leader_epoch: u64,
+    pub next_cycle: u64,
+    pub reset_paths: Vec<String>,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct ScannerUsageStateResetSlot {
+    path: String,
+    data: Option<Vec<u8>>,
+    revision: DataUsageCacheRevision,
+}
+
 static SCANNER_CYCLE_RECOVERY_STATUS: LazyLock<RwLock<ScannerCycleRecoveryStatus>> = LazyLock::new(|| {
     RwLock::new(ScannerCycleRecoveryStatus {
         path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
@@ -1286,6 +1307,292 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
     Ok(())
 }
 
+fn scanner_usage_state_reset_paths() -> Vec<String> {
+    vec![
+        DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
+        format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
+        format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
+    ]
+}
+
+pub(super) async fn read_usage_state_reset_slots(
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+) -> Result<Vec<ScannerUsageStateResetSlot>, ScannerError> {
+    let mut slots = Vec::new();
+    for path in scanner_usage_state_reset_paths() {
+        let (data, revision) = read_config_with_revision(storeapi.clone(), &path)
+            .await
+            .map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset slot {path}: {err}")))?;
+        slots.push(ScannerUsageStateResetSlot { path, data, revision });
+    }
+    Ok(slots)
+}
+
+fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> {
+    let mut floor = PersistedUsageFloor::default();
+    for slot in slots {
+        let Some(data) = slot.data.as_deref() else {
+            continue;
+        };
+        let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
+            continue;
+        };
+        update_persisted_usage_floor(&mut floor, &usage, &slot.path)?;
+    }
+    Ok(floor)
+}
+
+async fn read_cycle_state_for_usage_reset(
+    storeapi: Arc<ECStore>,
+) -> Result<(CurrentCycle, u64, DataUsageCacheRevision), ScannerError> {
+    let mut reader = match storeapi
+        .get_object_reader(
+            RUSTFS_META_BUCKET,
+            DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+            None,
+            http::HeaderMap::new(),
+            &ScannerObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(reader) => reader,
+        Err(
+            EcstoreError::FileNotFound
+            | EcstoreError::VolumeNotFound
+            | EcstoreError::ObjectNotFound(_, _)
+            | EcstoreError::BucketNotFound(_)
+            | EcstoreError::ConfigNotFound,
+        ) => return Ok((CurrentCycle::default(), 0, DataUsageCacheRevision::Missing)),
+        Err(err) => return Err(ScannerError::Other(format!("failed to inspect scanner cycle state: {err}"))),
+    };
+    if reader.object_info.is_dir || reader.object_info.size < 0 {
+        return Err(ScannerError::Other(
+            "scanner usage reset requires a regular scanner cycle state object".to_string(),
+        ));
+    }
+    let revision = reader
+        .object_info
+        .etag
+        .as_ref()
+        .filter(|etag| !etag.is_empty())
+        .cloned()
+        .map(DataUsageCacheRevision::Etag)
+        .ok_or_else(|| ScannerError::Other("scanner cycle state has no revision".to_string()))?;
+    let data = read_cycle_state_body(&mut reader)
+        .await
+        .map_err(|err| ScannerError::Other(format!("failed to read scanner cycle state for usage reset: {err}")))?;
+    let (cycle, leader_epoch) = decode_scanner_cycle_state(&data).map_err(|err| {
+        ScannerError::Other(format!(
+            "scanner usage reset requires a valid scanner cycle state; reset scanner cycle state first: {err}"
+        ))
+    })?;
+    Ok((cycle, leader_epoch, revision))
+}
+
+async fn delete_usage_state_reset_slot(
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    slot: &ScannerUsageStateResetSlot,
+    expected_epoch: u64,
+) -> Result<bool, ScannerError> {
+    if matches!(slot.revision, DataUsageCacheRevision::Missing) {
+        return Ok(false);
+    }
+    let delete_result = delete_config_with_publication_admission_for_epoch(
+        storeapi.clone(),
+        RUSTFS_META_BUCKET,
+        &slot.path,
+        ScannerObjectOptions {
+            delete_prefix: false,
+            http_preconditions: Some(slot.revision.preconditions()),
+            ..Default::default()
+        },
+        expected_epoch,
+    )
+    .await;
+    match delete_result {
+        Ok(_) => Ok(true),
+        Err(EcstoreError::FileNotFound | EcstoreError::ConfigNotFound | EcstoreError::ObjectNotFound(_, _)) => Ok(false),
+        Err(EcstoreError::PreconditionFailed) => Err(ScannerError::Other(format!(
+            "scanner usage reset slot changed while deleting {}",
+            slot.path
+        ))),
+        Err(err) if scanner_publication_epoch_changed(&err) => {
+            Err(ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()))
+        }
+        Err(err) => Err(ScannerError::Other(format!(
+            "failed to delete scanner usage reset slot {}: {err}",
+            slot.path
+        ))),
+    }
+}
+
+pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    slots: &[ScannerUsageStateResetSlot],
+    expected_epoch: u64,
+    leader_epoch: u64,
+) -> Result<Vec<String>, ScannerError> {
+    let mut reset_paths = Vec::new();
+    let primary = slots
+        .iter()
+        .find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
+    let marker = DataUsageInfo {
+        last_update: Some(std::time::SystemTime::now()),
+        scanner_epoch: Some(leader_epoch),
+        usage_snapshot_converged: Some(false),
+        usage_snapshot_bootstrap_pending: true,
+        ..Default::default()
+    };
+    let data = serde_json::to_vec(&marker)
+        .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage reset bootstrap marker: {err}")))?;
+    let save_result = save_config_with_publication_admission_for_epoch(
+        storeapi.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        data.clone(),
+        primary.revision.preconditions(),
+        expected_epoch,
+    )
+    .await;
+    if save_result
+        .as_ref()
+        .ok()
+        .and_then(|info| info.etag.as_deref())
+        .is_some_and(|etag| !etag.is_empty())
+    {
+        reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
+    } else {
+        let (persisted, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+            .await
+            .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage reset bootstrap marker: {err}")))?;
+        if persisted.as_deref() != Some(data.as_slice()) || !matches!(revision, DataUsageCacheRevision::Etag(_)) {
+            return Err(ScannerError::Other(match save_result {
+                Ok(_) => "scanner usage reset bootstrap returned no ETag and could not be confirmed".to_string(),
+                Err(err) if scanner_publication_epoch_changed(&err) => {
+                    "scanner usage reset deferred by a movement epoch change".to_string()
+                }
+                Err(EcstoreError::PreconditionFailed) => {
+                    "scanner usage reset primary slot changed before bootstrap publish".to_string()
+                }
+                Err(err) => format!("failed to persist scanner usage reset bootstrap: {err}"),
+            }));
+        }
+        reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
+    }
+
+    for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
+        if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
+            reset_paths.push(slot.path.clone());
+        }
+    }
+    invalidate_admin_data_usage_snapshot_cache().await;
+    invalidate_data_usage_snapshot_cache().await;
+    Ok(reset_paths)
+}
+
+pub async fn reset_scanner_usage_state_for_full_rebuild(
+    ctx: CancellationToken,
+    storeapi: Arc<ECStore>,
+) -> Result<ScannerUsageStateResetResult, ScannerError> {
+    let lock = storeapi
+        .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock")
+        .await
+        .map_err(|err| ScannerError::Other(format!("failed to acquire scanner leader lock: {err}")))?;
+    let guard = lock
+        .get_write_lock_quiet(Duration::from_secs(5))
+        .await
+        .map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
+    if guard.is_lock_lost() {
+        return Err(ScannerError::Other("scanner leader lock was lost before usage reset".to_string()));
+    }
+    if ctx.is_cancelled() {
+        return Err(ScannerError::Other("scanner usage reset was cancelled".to_string()));
+    }
+
+    let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
+        return Err(ScannerError::Other("scanner usage reset is blocked by data movement".to_string()));
+    };
+    let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
+    let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
+    let usage_floor = usage_state_reset_floor(&slots)?;
+    let leader_epoch = cycle_epoch
+        .max(usage_floor.leader_epoch)
+        .checked_add(1)
+        .filter(|epoch| *epoch < u64::MAX)
+        .ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?;
+    let rebuilt_cycle = CurrentCycle {
+        next: cycle.next.max(usage_floor.next_cycle),
+        ..Default::default()
+    };
+    let cycle_data = encode_scanner_cycle_state(&rebuilt_cycle, leader_epoch)
+        .map_err(|err| ScannerError::Other(format!("failed to encode scanner cycle state for usage reset: {err}")))?;
+
+    if guard.is_lock_lost() {
+        return Err(ScannerError::Other(
+            "scanner leader lock was lost before fencing usage reset cycle state".to_string(),
+        ));
+    }
+    save_config_with_publication_admission_for_epoch(
+        storeapi.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        cycle_data,
+        cycle_revision.preconditions(),
+        reset_epoch,
+    )
+    .await
+    .map_err(|err| {
+        if scanner_publication_epoch_changed(&err) {
+            ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
+        } else {
+            ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
+        }
+    })?;
+
+    if guard.is_lock_lost() {
+        return Err(ScannerError::Other(
+            "scanner leader lock was lost before publishing usage reset marker".to_string(),
+        ));
+    }
+    let reset_paths =
+        reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?;
+    if guard.is_lock_lost() {
+        return Err(ScannerError::Other(
+            "scanner leader lock was lost after publishing usage reset marker".to_string(),
+        ));
+    }
+
+    clear_scanner_usage_floor_failure();
+    clear_legacy_empty_usage_floor_recovery_status();
+    set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
+    super::notify_scanner_cycle_recovery_wake();
+    info!(
+        target: "rustfs::scanner",
+        event = EVENT_SCANNER_PERSIST_STATE,
+        component = LOG_COMPONENT_SCANNER,
+        subsystem = LOG_SUBSYSTEM_RUNTIME,
+        state = "usage_state_reset",
+        mode = SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD,
+        leader_epoch,
+        next_cycle = rebuilt_cycle.next,
+        reset_paths = reset_paths.len(),
+        "Scanner usage state reset was published"
+    );
+
+    Ok(ScannerUsageStateResetResult {
+        status: "reset".to_string(),
+        mode: SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD.to_string(),
+        usage_state: "bootstrap-pending".to_string(),
+        leader_epoch,
+        next_cycle: rebuilt_cycle.next,
+        reset_paths,
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 pub(super) enum ScannerCycleStateError {
     #[error("failed to encode scanner cycle state: {0}")]
@@ -1679,6 +1986,97 @@ pub(super) async fn persisted_usage_floor(
     Ok(floor)
 }
 
+fn usage_epoch(usage: &DataUsageInfo) -> u64 {
+    usage.scanner_epoch.unwrap_or_default()
+}
+
+fn usage_is_older_than_bootstrap(usage: &DataUsageInfo, bootstrap_epoch: Option<u64>) -> bool {
+    bootstrap_epoch.is_some_and(|epoch| usage_epoch(usage) < epoch)
+}
+
+fn decode_usage_floor_slot(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> {
+    serde_json::from_slice::<DataUsageInfo>(data)
+        .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage floor from {path}: {err}")))
+}
+
+fn update_persisted_usage_floor(floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str) -> Result<(), ScannerError> {
+    floor.leader_epoch = floor.leader_epoch.max(usage_epoch(usage));
+    if let Some(completed_cycle) = usage.scanner_cycle {
+        let next_cycle = completed_cycle
+            .checked_add(1)
+            .filter(|next| *next < u64::MAX)
+            .ok_or_else(|| ScannerError::Other(format!("persisted scanner usage cycle is exhausted in {path}")))?;
+        floor.next_cycle = floor.next_cycle.max(next_cycle);
+    }
+    Ok(())
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum BootstrapBackupAction {
+    Resume,
+    StopV2Pair,
+}
+
+struct BootstrapBackupSlot<'a> {
+    data: &'a [u8],
+    usage: &'a DataUsageInfo,
+    backup_path: &'a str,
+    primary_path: &'a str,
+    bootstrap_epoch: Option<u64>,
+    recovered_bootstrap: bool,
+    recovered_primary_companion_epoch: Option<u64>,
+}
+
+struct BootstrapBackupResolution<'a> {
+    floor: &'a mut PersistedUsageFloor,
+    unrecoverable_baseline_path: &'a mut Option<String>,
+}
+
+fn resolve_bootstrap_backup_slot(
+    slot: BootstrapBackupSlot<'_>,
+    resolution: BootstrapBackupResolution<'_>,
+) -> Result<BootstrapBackupAction, ScannerError> {
+    if usage_is_older_than_bootstrap(slot.usage, slot.bootstrap_epoch) {
+        return Ok(BootstrapBackupAction::Resume);
+    }
+    if let Some(primary_epoch) = slot.recovered_primary_companion_epoch {
+        if data_usage_info_has_persisted_baseline_identity(slot.usage) {
+            let backup_epoch = usage_epoch(slot.usage);
+            if backup_epoch >= primary_epoch {
+                update_persisted_usage_floor(resolution.floor, slot.usage, slot.backup_path)?;
+            }
+        } else if let Some(epoch) = legacy_empty_usage_fence_epoch(slot.data, slot.usage) {
+            if let Some(epoch) = epoch {
+                resolution.floor.leader_epoch = resolution.floor.leader_epoch.max(epoch);
+            }
+        } else {
+            resolution
+                .unrecoverable_baseline_path
+                .get_or_insert_with(|| slot.backup_path.to_string());
+        }
+        return Ok(BootstrapBackupAction::Resume);
+    }
+    let compatible_empty_fence = legacy_empty_usage_fence_epoch(slot.data, slot.usage).is_some_and(|epoch| {
+        epoch.is_none_or(|epoch| slot.bootstrap_epoch.is_some_and(|bootstrap_epoch| epoch <= bootstrap_epoch))
+    });
+    if compatible_empty_fence {
+        return Ok(BootstrapBackupAction::Resume);
+    }
+    if slot.recovered_bootstrap && data_usage_info_has_persisted_baseline_identity(slot.usage) {
+        let epoch = usage_epoch(slot.usage);
+        if epoch >= resolution.floor.leader_epoch {
+            update_persisted_usage_floor(resolution.floor, slot.usage, slot.backup_path)?;
+            if slot.primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() {
+                return Ok(BootstrapBackupAction::StopV2Pair);
+            }
+            return Ok(BootstrapBackupAction::Resume);
+        }
+    }
+    Err(ScannerError::Other(
+        "scanner usage bootstrap conflicts with a persisted backup".to_string(),
+    ))
+}
+
 pub(super) async fn persisted_usage_floor_for_startup(
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
     allow_missing_for_bootstrap: bool,
@@ -1702,80 +2100,98 @@ pub(super) async fn persisted_usage_floor_for_startup(
     let mut unrecoverable_baseline_path: Option<String> = None;
     let mut stale_authoritative_path: Option<String> = None;
     let mut legacy_empty_primary: Option<LegacyEmptyUsageFloorPrimary> = None;
-    let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
-        floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
-        if let Some(completed_cycle) = usage.scanner_cycle {
-            let next_cycle = completed_cycle
-                .checked_add(1)
-                .filter(|next| *next < u64::MAX)
-                .ok_or_else(|| ScannerError::Other(format!("persisted scanner usage cycle is exhausted in {path}")))?;
-            floor.next_cycle = floor.next_cycle.max(next_cycle);
-        }
-        Ok(())
-    };
     for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
         let backup_path = format!("{primary_path}.bkp");
         let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str();
         let mut recovered_primary_companion_epoch = None;
+        let mut primary_read_error = None;
         let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await {
             Ok((Some(data), revision)) => {
-                let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
-                    ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
-                })?;
-                if data_usage_info_is_bootstrap_pending(&usage) && primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() {
-                    if bootstrap_pending {
-                        return Err(ScannerError::Other("multiple scanner usage bootstrap markers were found".to_string()));
-                    }
-                    bootstrap_pending = true;
-                    bootstrap_epoch = usage.scanner_epoch;
-                    if let Some((marker, _)) = recovery_marker.as_ref() {
-                        if usage.scanner_epoch.is_none_or(|epoch| epoch < marker.leader_epoch) {
-                            return Err(ScannerError::Other(
-                                "scanner usage bootstrap is older than its recovery marker".to_string(),
-                            ));
-                        }
-                        recovered_bootstrap = true;
-                    }
-                    update_floor(&mut floor, &usage, primary_path)?;
-                    None
-                } else if !data_usage_info_has_persisted_baseline_identity(&usage) {
-                    invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
-                    invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
-                    match legacy_empty_usage_fence_epoch(&data, &usage) {
-                        Some(Some(epoch)) if is_v2_path => {
-                            legacy_empty_primary = Some(LegacyEmptyUsageFloorPrimary { revision, epoch });
-                        }
-                        Some(_) => {}
-                        None => {
-                            unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
-                        }
-                    }
-                    None
-                } else {
-                    let epoch = usage.scanner_epoch.unwrap_or_default();
-                    if recovered_bootstrap && !is_v2_path {
-                        if epoch < floor.leader_epoch {
-                            unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
-                        } else {
-                            update_floor(&mut floor, &usage, primary_path)?;
-                            recovered_primary_companion_epoch = Some(epoch);
-                        }
+                let usage = match decode_usage_floor_slot(&data, primary_path) {
+                    Ok(usage) => Some(usage),
+                    Err(err) if bootstrap_pending && !is_v2_path => {
+                        warn!(
+                            target: "rustfs::scanner",
+                            event = EVENT_SCANNER_PERSIST_STATE,
+                            component = LOG_COMPONENT_SCANNER,
+                            subsystem = LOG_SUBSYSTEM_RUNTIME,
+                            state = "usage_reset_cleanup_deferred",
+                            path = %primary_path,
+                            error = %err,
+                            "Scanner usage reset ignored a stale legacy usage primary after bootstrap marker"
+                        );
                         None
-                    // A legacy snapshot may be structurally valid but older
-                    // than an incomplete v2 snapshot left by a newer leader.
-                    // Do not let that candidate regress the startup floor.
-                    } else if invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch)
-                        && (!is_v2_path || recovery_marker.is_some())
-                    {
-                        stale_authoritative_path.get_or_insert_with(|| primary_path.to_string());
+                    }
+                    Err(err) => {
+                        return Err(err);
+                    }
+                };
+                if let Some(usage) = usage {
+                    if data_usage_info_is_bootstrap_pending(&usage) && primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() {
+                        if bootstrap_pending {
+                            return Err(ScannerError::Other("multiple scanner usage bootstrap markers were found".to_string()));
+                        }
+                        bootstrap_pending = true;
+                        bootstrap_epoch = usage.scanner_epoch;
+                        if let Some((marker, _)) = recovery_marker.as_ref() {
+                            if usage.scanner_epoch.is_none_or(|epoch| epoch < marker.leader_epoch) {
+                                return Err(ScannerError::Other(
+                                    "scanner usage bootstrap is older than its recovery marker".to_string(),
+                                ));
+                            }
+                            recovered_bootstrap = true;
+                        }
+                        update_persisted_usage_floor(&mut floor, &usage, primary_path)?;
+                        None
+                    } else if bootstrap_pending && !is_v2_path && usage_is_older_than_bootstrap(&usage, bootstrap_epoch) {
+                        None
+                    } else if !data_usage_info_has_persisted_baseline_identity(&usage) {
+                        invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                        invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
+                        match legacy_empty_usage_fence_epoch(&data, &usage) {
+                            Some(Some(epoch)) if is_v2_path => {
+                                legacy_empty_primary = Some(LegacyEmptyUsageFloorPrimary { revision, epoch });
+                            }
+                            Some(_) => {}
+                            None => {
+                                unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                            }
+                        }
                         None
                     } else {
-                        update_floor(&mut floor, &usage, primary_path)?;
-                        Some(epoch)
+                        let epoch = usage_epoch(&usage);
+                        if recovered_bootstrap && !is_v2_path {
+                            if epoch < floor.leader_epoch {
+                                unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                            } else {
+                                update_persisted_usage_floor(&mut floor, &usage, primary_path)?;
+                                recovered_primary_companion_epoch = Some(epoch);
+                            }
+                            None
+                        // A legacy snapshot may be structurally valid but older
+                        // than an incomplete v2 snapshot left by a newer leader.
+                        // Do not let that candidate regress the startup floor.
+                        } else if invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch)
+                            && (!is_v2_path || recovery_marker.is_some())
+                        {
+                            stale_authoritative_path.get_or_insert_with(|| primary_path.to_string());
+                            None
+                        } else {
+                            update_persisted_usage_floor(&mut floor, &usage, primary_path)?;
+                            Some(epoch)
+                        }
                     }
+                } else {
+                    None
                 }
             }
             Ok((None, _)) => None,
+            Err(err) if !is_v2_path && usage_floor_primary_read_error_allows_backup(&err) => {
+                primary_read_error = Some(format!("failed to read scanner usage epoch floor from {primary_path}: {err}"));
+                invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                None
+            }
             Err(err) => {
                 return Err(ScannerError::Other(format!(
                     "failed to read scanner usage epoch floor from {primary_path}: {err}"
@@ -1785,44 +2201,44 @@ pub(super) async fn persisted_usage_floor_for_startup(
         let mut any_found = primary_epoch.is_some();
         match read_config_with_revision(storeapi.clone(), &backup_path).await {
             Ok((Some(data), _)) => {
-                let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
-                    ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
-                })?;
+                let usage = match decode_usage_floor_slot(&data, &backup_path) {
+                    Ok(usage) => usage,
+                    Err(err) if bootstrap_pending => {
+                        warn!(
+                            target: "rustfs::scanner",
+                            event = EVENT_SCANNER_PERSIST_STATE,
+                            component = LOG_COMPONENT_SCANNER,
+                            subsystem = LOG_SUBSYSTEM_RUNTIME,
+                            state = "usage_reset_cleanup_deferred",
+                            path = %backup_path,
+                            error = %err,
+                            "Scanner usage reset ignored a stale usage backup after bootstrap marker"
+                        );
+                        continue;
+                    }
+                    Err(err) => {
+                        return Err(err);
+                    }
+                };
                 if bootstrap_pending {
-                    if let Some(primary_epoch) = recovered_primary_companion_epoch {
-                        if data_usage_info_has_persisted_baseline_identity(&usage) {
-                            let backup_epoch = usage.scanner_epoch.unwrap_or_default();
-                            if backup_epoch >= primary_epoch {
-                                update_floor(&mut floor, &usage, &backup_path)?;
-                            }
-                        } else if let Some(epoch) = legacy_empty_usage_fence_epoch(&data, &usage) {
-                            if let Some(epoch) = epoch {
-                                floor.leader_epoch = floor.leader_epoch.max(epoch);
-                            }
-                        } else {
-                            unrecoverable_baseline_path.get_or_insert_with(|| backup_path.clone());
-                        }
-                        continue;
+                    match resolve_bootstrap_backup_slot(
+                        BootstrapBackupSlot {
+                            data: &data,
+                            usage: &usage,
+                            backup_path: &backup_path,
+                            primary_path,
+                            bootstrap_epoch,
+                            recovered_bootstrap,
+                            recovered_primary_companion_epoch,
+                        },
+                        BootstrapBackupResolution {
+                            floor: &mut floor,
+                            unrecoverable_baseline_path: &mut unrecoverable_baseline_path,
+                        },
+                    )? {
+                        BootstrapBackupAction::Resume => continue,
+                        BootstrapBackupAction::StopV2Pair => break,
                     }
-                    let compatible_empty_fence = legacy_empty_usage_fence_epoch(&data, &usage).is_some_and(|epoch| {
-                        epoch.is_none_or(|epoch| bootstrap_epoch.is_some_and(|bootstrap_epoch| epoch <= bootstrap_epoch))
-                    });
-                    if compatible_empty_fence {
-                        continue;
-                    }
-                    if recovered_bootstrap && data_usage_info_has_persisted_baseline_identity(&usage) {
-                        let epoch = usage.scanner_epoch.unwrap_or_default();
-                        if epoch >= floor.leader_epoch {
-                            update_floor(&mut floor, &usage, &backup_path)?;
-                            if primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() {
-                                break;
-                            }
-                            continue;
-                        }
-                    }
-                    return Err(ScannerError::Other(
-                        "scanner usage bootstrap conflicts with a persisted backup".to_string(),
-                    ));
                 }
                 if !data_usage_info_has_persisted_baseline_identity(&usage) {
                     invalid_baseline_path.get_or_insert_with(|| backup_path.clone());
@@ -1834,14 +2250,14 @@ pub(super) async fn persisted_usage_floor_for_startup(
                     // missing-state bootstrap.  Continue to a legacy pair in
                     // case it contains a complete, fenced snapshot.
                 } else {
-                    let backup_epoch = usage.scanner_epoch.unwrap_or_default();
+                    let backup_epoch = usage_epoch(&usage);
                     // A backup write from an older leader may complete after the
                     // primary epoch has been fenced. It must not advance the startup
                     // floor unless its epoch is at least as new as the primary.
                     if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch)
                         && invalid_baseline_epoch.is_none_or(|epoch| backup_epoch >= epoch)
                     {
-                        update_floor(&mut floor, &usage, &backup_path)?;
+                        update_persisted_usage_floor(&mut floor, &usage, &backup_path)?;
                         any_found = true;
                     } else {
                         stale_authoritative_path.get_or_insert_with(|| backup_path.clone());
@@ -1855,6 +2271,14 @@ pub(super) async fn persisted_usage_floor_for_startup(
                 )));
             }
         }
+        if let Some(primary_read_error) = primary_read_error
+            && !any_found
+        {
+            return Err(ScannerError::Other(format!(
+                "{}; no valid scanner usage floor backup was available at {backup_path}",
+                primary_read_error
+            )));
+        }
         if any_found {
             if bootstrap_pending {
                 return Err(ScannerError::Other(

crates/scanner/src/scanner/leadership.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 /// Leader-lock claiming, usage-epoch fencing, and lock-loss handling.
 use super::*;
+use crate::data_usage_define::usage_floor_primary_read_error_allows_backup;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub(super) enum ScannerLeadershipClaimReconcile {
@@ -142,20 +143,34 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
         }
     }
 
-    for path in [
-        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
-        format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
-    ] {
-        let (legacy, _) = read_config_with_revision(storeapi.clone(), &path)
-            .await
-            .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
+    let legacy_primary_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string();
+    let legacy_backup_path = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let mut legacy_primary_read_error = None;
+    for path in [&legacy_primary_path, &legacy_backup_path] {
+        let legacy = match read_config_with_revision(storeapi.clone(), path).await {
+            Ok((legacy, _)) => legacy,
+            Err(err) if path == &legacy_primary_path && usage_floor_primary_read_error_allows_backup(&err) => {
+                legacy_primary_read_error = Some(format!("failed to read legacy scanner usage epoch fence from {path}: {err}"));
+                continue;
+            }
+            Err(err) => {
+                return Err(ScannerError::Other(format!(
+                    "failed to read legacy scanner usage epoch fence from {path}: {err}"
+                )));
+            }
+        };
         if let Some(legacy) = legacy.as_deref() {
-            let usage = decode_usage_snapshot_for_epoch_fence(legacy, &path, false)?;
+            let usage = decode_usage_snapshot_for_epoch_fence(legacy, path, false)?;
             if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) {
                 return Ok(Some(usage));
             }
         }
     }
+    if let Some(legacy_primary_read_error) = legacy_primary_read_error {
+        return Err(ScannerError::Other(format!(
+            "{legacy_primary_read_error}; no valid legacy scanner usage epoch fence backup was available at {legacy_backup_path}"
+        )));
+    }
     // A missing usage snapshot is an uninitialized state, not an empty
     // snapshot. Leadership fencing may proceed without creating a plausible
     // default; the first authoritative scanner publication will create it.
@@ -258,6 +273,12 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
             Some(epoch) if epoch == claimed_epoch => return Ok(()),
             Some(_) | None => {}
         }
+        // A validated pre-marker legacy baseline needs an explicit complete
+        // identity before acquiring an epoch. Otherwise the v2 reader would
+        // reject the fenced value on its next startup.
+        if !usage.usage_snapshot_bootstrap_pending {
+            usage.usage_snapshot_complete = true;
+        }
         usage.scanner_epoch = Some(claimed_epoch);
         let data = serde_json::to_vec(&usage)
             .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?;

crates/scanner/src/scanner/tests.rs

@@ -574,6 +574,7 @@ struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
     insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
+    read_errors: Mutex<HashMap<String, EcstoreError>>,
     delayed_gets: Mutex<HashMap<String, Duration>>,
     non_regular_objects: Mutex<HashSet<String>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
@@ -623,6 +624,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
         _opts: &ObjectOptions,
     ) -> EcstoreResult<GetObjectReader> {
         let key = memory_config_key(bucket, object);
+        if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
+            return Err(error);
+        }
         if let Some(delay) = self.delayed_gets.lock().await.remove(&key) {
             tokio::time::sleep(delay).await;
         }
@@ -2902,6 +2906,147 @@ async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_ba
     );
 }
 
+async fn seed_legacy_primary_read_error_with_backup(store: &Arc<MemoryConfigStore>, error: EcstoreError, epoch: u64, cycle: u64) {
+    let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str();
+    let legacy_backup = format!("{legacy_primary}.bkp");
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    backup.scanner_epoch = Some(epoch);
+    backup.scanner_cycle = Some(cycle);
+
+    store
+        .read_errors
+        .lock()
+        .await
+        .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), error);
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, &legacy_backup),
+        serde_json::to_vec(&backup).expect("legacy backup usage snapshot should encode"),
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_recovers_legacy_backup_after_primary_decode_error() {
+    let store = Arc::new(MemoryConfigStore::default());
+    seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await;
+
+    let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect("valid legacy backup should recover the startup floor");
+    assert_eq!(state, PersistedUsageFloorStartup::Authoritative);
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 42,
+            leader_epoch: 19,
+        }
+    );
+    assert_eq!(
+        persisted_usage_floor(store)
+            .await
+            .expect("valid legacy backup should recover the authoritative floor"),
+        floor
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_does_not_bootstrap_over_corrupt_legacy_primary_without_backup() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str();
+    store
+        .read_errors
+        .lock()
+        .await
+        .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), EcstoreError::FileCorrupt);
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("corrupt legacy primary without a valid backup must remain fail-closed");
+    assert!(err.to_string().contains("no valid scanner usage floor backup"), "unexpected error: {err}");
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_does_not_fallback_to_legacy_after_corrupt_v2_primary() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let mut v2_backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    v2_backup.scanner_epoch = Some(8);
+    v2_backup.scanner_cycle = Some(11);
+    let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    legacy.scanner_epoch = Some(3);
+    legacy.scanner_cycle = Some(7);
+    store.read_errors.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        EcstoreError::FileCorrupt,
+    );
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str())),
+        serde_json::to_vec(&v2_backup).expect("v2 backup usage snapshot should encode"),
+    );
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"),
+    );
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("corrupt v2 primary must not recover without a primary revision");
+    assert!(
+        err.to_string().contains(&format!(
+            "failed to read scanner usage epoch floor from {}",
+            DATA_USAGE_OBJ_NAME_PATH.as_str()
+        )),
+        "unexpected error: {err}"
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_keeps_transient_primary_read_error_fail_closed() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str();
+    seed_legacy_primary_read_error_with_backup(
+        &store,
+        EcstoreError::Io(std::io::Error::new(
+            std::io::ErrorKind::ConnectionReset,
+            "connection reset while reading usage primary",
+        )),
+        19,
+        41,
+    )
+    .await;
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("transient primary errors must not be converted into backup recovery");
+    assert!(
+        err.to_string()
+            .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")),
+        "unexpected error: {err}"
+    );
+    assert!(
+        !err.to_string().contains("no valid scanner usage floor backup"),
+        "transient error should not enter corrupt-primary fallback: {err}"
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_keeps_outdated_primary_metadata_fail_closed() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str();
+    seed_legacy_primary_read_error_with_backup(&store, EcstoreError::OutdatedXLMeta, 19, 41).await;
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("outdated primary metadata must not be converted into backup recovery");
+    assert!(
+        err.to_string()
+            .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")),
+        "unexpected error: {err}"
+    );
+    assert!(
+        !err.to_string().contains("no valid scanner usage floor backup"),
+        "outdated metadata should not enter corrupt-primary fallback: {err}"
+    );
+}
+
 #[tokio::test]
 async fn scanner_usage_floor_does_not_bootstrap_over_incomplete_v2_primary() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -3004,6 +3149,19 @@ async fn scanner_leadership_fencing_recovers_incomplete_v2_primary_from_backup()
     assert_eq!(recovered.scanner_cycle, Some(103));
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_leadership_fencing_recovers_legacy_backup_after_primary_decode_error() {
+    let store = Arc::new(MemoryConfigStore::default());
+    seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await;
+
+    let recovered = usage_snapshot_for_epoch_fence(store, None, false)
+        .await
+        .expect("a valid legacy backup should provide the fencing baseline")
+        .expect("the fencing baseline should be present");
+    assert_eq!(recovered.scanner_epoch, Some(19));
+    assert_eq!(recovered.scanner_cycle, Some(41));
+}
+
 #[tokio::test]
 async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -3955,6 +4113,162 @@ async fn scanner_defers_leadership_when_usage_snapshots_are_stably_absent() {
     assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_leadership_claim_recovers_legacy_backup_after_primary_decode_error() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let ctx = CancellationToken::new();
+    seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await;
+
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle::default();
+    let mut persisted_epoch = 19;
+    assert!(
+        claim_scanner_leadership(
+            &ctx,
+            store.clone(),
+            &mut cycle,
+            &mut revision,
+            &mut persisted_epoch,
+            false,
+            ScannerCycleResetPolicy::None,
+        )
+        .await
+    );
+
+    let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("leadership claim should persist after legacy backup recovery");
+    let (_, claimed_epoch) = decode_scanner_cycle_state(&state).expect("leadership claim should decode");
+    assert_eq!(claimed_epoch, 20);
+    assert_eq!(persisted_epoch, 20);
+
+    let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("legacy backup recovery should publish a fenced v2 usage primary");
+    let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("fenced v2 usage primary should decode");
+    assert_eq!(usage.scanner_epoch, Some(20));
+    assert_eq!(usage.scanner_cycle, Some(41));
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_metadata_truncation() {
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    let (temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
+    let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    usage.usage_snapshot_complete = false;
+    usage.scanner_cycle = Some(41);
+    let mut data = serde_json::to_vec(&usage).expect("legacy usage should encode");
+    data.resize(data.len() + 16 * 1024, b' ');
+    let legacy_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str();
+    let backup_path = format!("{legacy_path}.bkp");
+    for path in [legacy_path, backup_path.as_str()] {
+        save_config(store.clone(), path, data.clone())
+            .await
+            .expect("legacy usage fixture should persist");
+    }
+    let mut truncated_files = Vec::new();
+    for disk_index in 0..4 {
+        let path = temp_dir
+            .path()
+            .join(format!("pool0/disk{disk_index}"))
+            .join(RUSTFS_META_BUCKET)
+            .join(legacy_path)
+            .join("xl.meta");
+        let file = tokio::fs::OpenOptions::new()
+            .write(true)
+            .open(&path)
+            .await
+            .expect("legacy inline metadata should exist");
+        assert!(file.metadata().await.expect("metadata should be readable").len() > 4096);
+        file.set_len(4096).await.expect("fixture should truncate at a page boundary");
+        truncated_files.push((
+            path.clone(),
+            tokio::fs::read(&path)
+                .await
+                .expect("truncated evidence should remain readable"),
+        ));
+    }
+
+    let store = restart_scanner_cycle_store_from(&store).await;
+    let error = read_config_with_revision(store.clone(), legacy_path)
+        .await
+        .expect_err("truncated primary must fail in the real object reader");
+    assert!(
+        error.to_string().contains("InlineData value out of range"),
+        "unexpected truncated-primary error: {error}"
+    );
+    assert_eq!(
+        read_config_with_revision(store.clone(), &backup_path)
+            .await
+            .expect("backup should remain readable")
+            .0,
+        Some(data.clone()),
+    );
+    let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect("intact legacy backup must recover startup despite truncated primary");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 42,
+            leader_epoch: 0
+        }
+    );
+    assert_eq!(state, PersistedUsageFloorStartup::Authoritative);
+
+    let baseline = read_data_usage_persist_baseline(store.clone())
+        .await
+        .expect("publication must also read the intact backup");
+    assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
+    assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
+    fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false)
+        .await
+        .expect("legacy backup must be fenced into v2");
+    let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("fencing must publish a v2 usage primary");
+    let fenced = serde_json::from_slice::<DataUsageInfo>(&fenced).expect("fenced v2 usage primary should decode");
+    assert!(
+        fenced.usage_snapshot_complete,
+        "the fenced pre-marker baseline must become a complete v2 identity"
+    );
+
+    let store = restart_scanner_cycle_store_from(&store).await;
+    let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect("a restart after fencing must preserve the recovered floor");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 42,
+            leader_epoch: 7
+        }
+    );
+    assert_eq!(state, PersistedUsageFloorStartup::Authoritative);
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    assert_eq!(
+        persisted_usage_floor(restarted)
+            .await
+            .expect("fenced floor must survive another restart"),
+        PersistedUsageFloor {
+            next_cycle: 42,
+            leader_epoch: 7
+        }
+    );
+    for (path, bytes) in truncated_files {
+        assert_eq!(tokio::fs::read(path).await.expect("legacy evidence must not be removed"), bytes);
+    }
+    assert_eq!(
+        read_config(store, &backup_path)
+            .await
+            .expect("legacy backup must remain intact"),
+        data
+    );
+    global_metrics().set_cycle(None).await;
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+}
+
 #[tokio::test]
 async fn usage_bootstrap_pending_unblocks_first_leadership_claim() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -4063,6 +4377,265 @@ async fn usage_bootstrap_does_not_overwrite_concurrent_replacement() {
     );
 }
 
+#[tokio::test]
+#[serial]
+async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+    let cycle = CurrentCycle {
+        current: 41,
+        next: 42,
+        cycle_completed: vec![Utc::now()],
+        started: Utc::now(),
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_BLOOM_NAME_PATH.as_str(),
+        encode_scanner_cycle_state(&cycle, 7).expect("cycle state should encode"),
+    )
+    .await
+    .expect("cycle state should persist");
+
+    let usage_backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let legacy_backup_path = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str());
+    for (path, epoch, cycle) in [
+        (usage_backup_path.as_str(), 6, 40),
+        (LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), 5, 39),
+        (legacy_backup_path.as_str(), 4, 38),
+        (DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), 8, 41),
+    ] {
+        let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+        usage.scanner_epoch = Some(epoch);
+        usage.scanner_cycle = Some(cycle);
+        save_config(store.clone(), path, serde_json::to_vec(&usage).expect("usage slot should encode"))
+            .await
+            .expect("usage slot should persist");
+    }
+
+    let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
+        .await
+        .expect("usage state reset should publish a fenced bootstrap marker");
+
+    assert_eq!(result.status, "reset");
+    assert_eq!(result.mode, "full-rebuild");
+    assert_eq!(result.usage_state, "bootstrap-pending");
+    assert_eq!(result.leader_epoch, 9);
+    assert_eq!(result.next_cycle, 42);
+    assert_eq!(result.reset_paths.len(), 5);
+
+    let cycle_state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
+        .await
+        .expect("reset cycle state should remain");
+    let (reset_cycle, reset_epoch) = decode_scanner_cycle_state(&cycle_state).expect("reset cycle state should decode");
+    assert_eq!(reset_cycle.next, 42);
+    assert_eq!(reset_cycle.current, 0);
+    assert!(reset_cycle.cycle_completed.is_empty());
+    assert_eq!(reset_epoch, 9);
+
+    let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("reset usage marker should remain");
+    let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("reset usage marker should decode");
+    assert!(data_usage_info_is_bootstrap_pending(&usage));
+    assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
+    assert_eq!(usage.scanner_epoch, Some(9));
+
+    for path in [
+        usage_backup_path.as_str(),
+        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        legacy_backup_path.as_str(),
+        DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+    ] {
+        assert!(
+            matches!(read_config(store.clone(), path).await, Err(EcstoreError::ConfigNotFound)),
+            "reset should remove stale usage slot {path}"
+        );
+    }
+
+    let (floor, state) = persisted_usage_floor_for_startup(store, false)
+        .await
+        .expect("reset marker should be resumable");
+    assert_eq!(floor.leader_epoch, 9);
+    assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending);
+}
+
+#[tokio::test]
+async fn scanner_usage_state_reset_bootstrap_survives_stale_cleanup_slots_after_restart() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let mut marker = DataUsageInfo {
+        last_update: Some(std::time::SystemTime::UNIX_EPOCH),
+        scanner_epoch: Some(9),
+        usage_snapshot_converged: Some(false),
+        usage_snapshot_bootstrap_pending: true,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("usage reset marker should encode"),
+    )
+    .await
+    .expect("usage reset marker should persist");
+    save_config(
+        store.clone(),
+        format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()).as_str(),
+        b"{not-json".to_vec(),
+    )
+    .await
+    .expect("stale malformed backup should persist");
+
+    marker.usage_snapshot_bootstrap_pending = false;
+    marker.usage_snapshot_complete = true;
+    marker.scanner_epoch = Some(8);
+    marker.scanner_cycle = Some(41);
+    for path in [
+        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
+        format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+    ] {
+        save_config(
+            store.clone(),
+            &path,
+            serde_json::to_vec(&marker).expect("stale legacy usage should encode"),
+        )
+        .await
+        .expect("stale legacy usage should persist");
+    }
+
+    let (floor, state) = persisted_usage_floor_for_startup(store.clone(), false)
+        .await
+        .expect("restart should resume reset bootstrap while stale cleanup slots remain");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 0,
+            leader_epoch: 9,
+        }
+    );
+    assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending);
+    assert!(
+        persisted_usage_floor(store).await.is_err(),
+        "bootstrap marker must still not become an authoritative usage floor"
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_state_reset_bootstrap_survives_malformed_legacy_primary_after_restart() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let marker = DataUsageInfo {
+        last_update: Some(std::time::SystemTime::UNIX_EPOCH),
+        scanner_epoch: Some(9),
+        usage_snapshot_converged: Some(false),
+        usage_snapshot_bootstrap_pending: true,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("usage reset marker should encode"),
+    )
+    .await
+    .expect("usage reset marker should persist");
+    save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("stale malformed legacy primary should persist");
+
+    let (floor, state) = persisted_usage_floor_for_startup(store.clone(), false)
+        .await
+        .expect("restart should resume reset bootstrap when only stale malformed legacy primary remains");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 0,
+            leader_epoch: 9,
+        }
+    );
+    assert_eq!(state, PersistedUsageFloorStartup::BootstrapPending);
+    assert!(persisted_usage_floor(store).await.is_err());
+}
+
+#[tokio::test]
+async fn scanner_usage_state_reset_bootstrap_does_not_mask_newer_legacy_backup() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let marker = DataUsageInfo {
+        last_update: Some(std::time::SystemTime::UNIX_EPOCH),
+        scanner_epoch: Some(9),
+        usage_snapshot_converged: Some(false),
+        usage_snapshot_bootstrap_pending: true,
+        ..Default::default()
+    };
+    save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&marker).expect("usage reset marker should encode"),
+    )
+    .await
+    .expect("usage reset marker should persist");
+    save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{not-json".to_vec())
+        .await
+        .expect("malformed legacy primary should persist");
+
+    let mut newer_backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    newer_backup.scanner_epoch = Some(10);
+    newer_backup.scanner_cycle = Some(43);
+    save_config(
+        store.clone(),
+        format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()).as_str(),
+        serde_json::to_vec(&newer_backup).expect("newer legacy backup should encode"),
+    )
+    .await
+    .expect("newer legacy backup should persist");
+
+    let err = persisted_usage_floor_for_startup(store, false)
+        .await
+        .expect_err("newer legacy backup must not be hidden by an older bootstrap marker");
+    assert!(
+        err.to_string()
+            .contains("scanner usage bootstrap conflicts with a persisted backup"),
+        "unexpected conflict error: {err}"
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_state_reset_slots_reject_primary_aba() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    store.objects.lock().await.insert(key.clone(), b"not-json".to_vec());
+    store.revisions.lock().await.insert(key.clone(), 1);
+    let slots = read_usage_state_reset_slots(store.clone())
+        .await
+        .expect("usage reset slots should be inspected");
+
+    store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
+    store.revisions.lock().await.insert(key, 2);
+    let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
+        .await
+        .expect_err("stale primary revision must not be overwritten");
+    assert!(
+        err.to_string()
+            .contains("scanner usage reset primary slot changed before bootstrap publish"),
+        "unexpected primary ABA error: {err}"
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    store.objects.lock().await.insert(key, b"not-json".to_vec());
+    let slots = read_usage_state_reset_slots(store.clone())
+        .await
+        .expect("usage reset slots should be inspected");
+    store.publication_admission_blocked.store(true, Ordering::Release);
+
+    let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
+        .await
+        .expect_err("movement admission loss must defer reset");
+    assert!(
+        err.to_string()
+            .contains("scanner usage reset deferred by a movement epoch change"),
+        "unexpected movement defer error: {err}"
+    );
+}
+
 #[tokio::test]
 async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() {
     let store = Arc::new(MemoryConfigStore::default());

crates/scanner/src/scanner/usage_store.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 /// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
 use super::*;
+use crate::data_usage_define::usage_floor_primary_read_error_allows_backup;
 use crate::storage_api::owner::ScannerPublicationCommitState;
 use std::collections::HashMap;
 use std::sync::atomic::AtomicBool;
@@ -53,6 +54,30 @@ pub(super) struct DataUsagePersistBaseline {
     pub(super) revision: DataUsageCacheRevision,
 }
 
+async fn read_usage_persist_candidate(
+    storeapi: Arc<impl ScannerObjectIO>,
+    path: &str,
+) -> Result<(Option<Vec<u8>>, DataUsageCacheRevision), EcstoreError> {
+    let primary = read_config_with_revision(storeapi.clone(), path).await;
+    if path != LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() {
+        return primary;
+    }
+    let Err(primary_error) = &primary else {
+        return primary;
+    };
+    if !usage_floor_primary_read_error_allows_backup(primary_error) {
+        return primary;
+    }
+    let backup_path = format!("{path}.bkp");
+    let backup = read_config_with_revision(storeapi, &backup_path).await?;
+    if backup.0.as_deref().is_some_and(|data| {
+        serde_json::from_slice::<DataUsageInfo>(data).is_ok_and(|usage| data_usage_info_has_persisted_baseline_identity(&usage))
+    }) {
+        return Ok(backup);
+    }
+    primary
+}
+
 /// Read the bytes used as the baseline for a usage publication while keeping
 /// the v2 primary revision as the CAS fence. During an interrupted upgrade the
 /// primary can be valid JSON without a baseline identity; in that case a
@@ -68,7 +93,7 @@ pub(super) async fn read_data_usage_persist_baseline(
             LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
             format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
         ] {
-            let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
+            let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?;
             let Some(candidate) = candidate else {
                 continue;
             };
@@ -107,7 +132,7 @@ pub(super) async fn read_data_usage_persist_baseline(
         LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
         format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
     ] {
-        let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
+        let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?;
         let Some(candidate) = candidate else {
             continue;
         };

crates/scanner/tests/lifecycle_integration_test.rs

@@ -46,7 +46,7 @@ use storage_api::lifecycle::{
     STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
     enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
     get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
-    is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
+    is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
     register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
 };
 
@@ -562,6 +562,29 @@ where
     }
 }
 
+// Deep transition futures can overflow libtest's default stack before their
+// first assertion, so the serial ILM cases use one dedicated test thread.
+fn run_large_stack_async_test<F, Fut>(thread_name: &'static str, test_fn: F)
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = ()> + 'static,
+{
+    let handle = std::thread::Builder::new()
+        .name(thread_name.to_string())
+        .stack_size(32 * 1024 * 1024)
+        .spawn(move || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("large-stack scanner test runtime should build");
+            runtime.block_on(test_fn());
+        })
+        .expect("large-stack scanner test thread should spawn");
+    if let Err(payload) = handle.join() {
+        std::panic::resume_unwind(payload);
+    }
+}
+
 mod serial_tests {
     use super::*;
 
@@ -737,10 +760,17 @@ mod serial_tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[test]
     #[serial]
     #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
-    async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
+    fn rejected_transition_candidate_is_recovered_from_persisted_transaction() {
+        run_large_stack_async_test(
+            "scanner-rejected-transition-transaction",
+            rejected_transition_candidate_is_recovered_from_persisted_transaction_case,
+        );
+    }
+
+    async fn rejected_transition_candidate_is_recovered_from_persisted_transaction_case() {
         let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
 
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
@@ -790,21 +820,20 @@ mod serial_tests {
             "no cleanup path may delete the candidate while remove failures are enabled"
         );
 
-        let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+        let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
             .await
-            .expect("tier delete journal recovery should scan the persisted candidate");
-        assert_eq!(retained.scanned, 1);
-        assert_eq!(retained.deleted, 0);
-        assert_eq!(retained.failed, 1);
+            .expect("transition transaction recovery should scan the persisted candidate");
+        assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
         assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate");
 
         backend.set_remove_failure(false);
-        let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+        let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
             .await
-            .expect("tier delete journal recovery should delete the retained candidate");
-        assert_eq!(recovered.scanned, 1);
-        assert_eq!(recovered.deleted, 1);
-        assert_eq!(recovered.failed, 0);
+            .expect("transition transaction recovery should delete the retained candidate");
+        assert_eq!(
+            (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
+            (1, 1, 0, 0)
+        );
         let removed_versions = backend.remove_versions().await;
         assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete");
         assert!(
@@ -813,10 +842,10 @@ mod serial_tests {
         );
         assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate");
 
-        let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+        let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
             .await
-            .expect("a removed tier delete journal entry should no longer be listed");
-        assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry");
+            .expect("a removed transition transaction should no longer be listed");
+        assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted transaction");
         assert_eq!(
             read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
             payload,
@@ -824,10 +853,17 @@ mod serial_tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[test]
     #[serial]
     #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
-    async fn cancelled_before_cleanup_store_resolution_persists_journal() {
+    fn cancelled_before_cleanup_store_resolution_persists_transaction() {
+        run_large_stack_async_test(
+            "scanner-cancelled-transition-transaction",
+            cancelled_before_cleanup_store_resolution_persists_transaction_case,
+        );
+    }
+
+    async fn cancelled_before_cleanup_store_resolution_persists_transaction_case() {
         let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
         let backend = register_mock_tier(&tier_name).await;
@@ -876,9 +912,9 @@ mod serial_tests {
 
         let retained = tokio::time::timeout(Duration::from_secs(30), async {
             loop {
-                let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
                     .await
-                    .expect("the cancelled transition journal should be readable");
+                    .expect("the cancelled transition transaction should be readable");
                 if recovery.scanned > 0 {
                     break recovery;
                 }
@@ -887,29 +923,36 @@ mod serial_tests {
         })
         .await
         .expect("Drop should persist the rejected candidate through the saved instance context");
-        assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
+        assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
         tokio::time::timeout(Duration::from_secs(5), async {
             while backend.exact_remove_count() < 2 {
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint");
+        .expect("Drop cleanup and failed transaction recovery must both preserve the exact version constraint");
         let failed_exact_attempts = backend.exact_remove_count();
+        assert_eq!(
+            failed_exact_attempts, 2,
+            "the cancelled task and the first failed recovery must each preserve the exact delete constraint"
+        );
         assert_eq!(backend.object_count().await, 1);
         assert!(backend.remove_versions().await.is_empty());
 
         backend.set_remove_failure(false);
-        let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+        let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
             .await
             .expect("recovery should delete the candidate retained by the cancelled transition");
-        assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
+        assert_eq!(
+            (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
+            (1, 1, 0, 0)
+        );
         assert_eq!(backend.remove_versions().await, backend.put_versions().await);
         assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1);
         assert_eq!(backend.object_count().await, 0);
-        let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+        let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
             .await
-            .expect("successful recovery should remove the cancellation journal");
+            .expect("successful recovery should remove the cancellation transaction");
         assert_eq!(empty.scanned, 0);
         assert_eq!(
             read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
@@ -918,10 +961,14 @@ mod serial_tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[test]
     #[serial]
     #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
-    async fn rejected_transition_cleanup_durability_matrix() {
+    fn rejected_transition_cleanup_durability_matrix() {
+        run_large_stack_async_test("scanner-transition-cleanup-matrix", rejected_transition_cleanup_durability_matrix_case);
+    }
+
+    async fn rejected_transition_cleanup_durability_matrix_case() {
         #[derive(Clone, Copy)]
         enum CleanupCase {
             Persisted,
@@ -1010,44 +1057,67 @@ mod serial_tests {
                 CleanupCase::Persisted | CleanupCase::DeleteFallback => {
                     assert_eq!(backend.remove_versions().await, backend.put_versions().await);
                     assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate");
-                    let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                    let recovery = recover_transition_transaction_records(ecstore.clone(), 100, None)
                         .await
-                        .expect("successful cleanup must not retain a journal entry");
-                    assert_eq!(recovery.scanned, 0);
+                        .expect("successful cleanup must leave no failed transition transaction");
+                    assert_eq!(recovery.failed, 0);
+                    assert_eq!(recovery.retained, 0);
+                    assert_eq!(recovery.recovered, recovery.scanned);
+                    let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
+                        .await
+                        .expect("successful reconciliation must remove every transition transaction");
+                    assert_eq!(empty.scanned, 0);
                 }
                 CleanupCase::RetryPersisted => {
                     assert!(
                         !err.to_string().contains("journal retry error"),
-                        "a successful journal retry must preserve the original version-constraint error"
+                        "the transaction path must not surface the removed journal fallback error"
                     );
                     assert_eq!(backend.object_count().await, 1);
-                    let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                    let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
                         .await
-                        .expect("the retried journal should be recoverable");
-                    assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
+                        .expect("the retained transaction should be recoverable");
+                    assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 0, 1));
                     backend.set_remove_failure(false);
-                    let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                    let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
                         .await
-                        .expect("recovery should delete the exact retried candidate");
-                    assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
+                        .expect("recovery should delete the exact transaction candidate");
+                    assert_eq!(
+                        (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
+                        (1, 1, 0, 0)
+                    );
                     assert_eq!(backend.remove_versions().await, backend.put_versions().await);
                     assert_eq!(backend.object_count().await, 0);
-                    let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                    let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
                         .await
-                        .expect("successful recovery must remove the retried journal");
+                        .expect("successful recovery must remove the retained transaction");
                     assert_eq!(empty.scanned, 0);
                 }
                 CleanupCase::FullyFailed => {
                     let message = err.to_string();
-                    assert!(message.contains("initial journal error"), "{message}");
                     assert!(message.contains("cleanup error"), "{message}");
-                    assert!(message.contains("journal retry error"), "{message}");
                     assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible");
                     assert!(backend.remove_versions().await.is_empty());
-                    let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
+                    let retained = recover_transition_transaction_records(ecstore.clone(), 100, None)
                         .await
-                        .expect("failed journal writes must not create partial recovery entries");
-                    assert_eq!(recovery.scanned, 0);
+                        .expect("the pre-upload transaction must retain ownership after cleanup failure");
+                    assert_eq!(retained.scanned, 1, "the failed cleanup must keep one durable transaction owner");
+                    assert_eq!(retained.recovered, 0);
+                    assert_eq!(retained.retained + retained.failed, 1);
+                    backend.set_remove_failure(false);
+                    let recovered = recover_transition_transaction_records(ecstore.clone(), 100, None)
+                        .await
+                        .expect("recovery should delete the candidate after the backend becomes available");
+                    assert_eq!(
+                        (recovered.scanned, recovered.recovered, recovered.retained, recovered.failed),
+                        (1, 1, 0, 0)
+                    );
+                    assert_eq!(backend.remove_versions().await, backend.put_versions().await);
+                    assert_eq!(backend.object_count().await, 0);
+                    let empty = recover_transition_transaction_records(ecstore.clone(), 100, None)
+                        .await
+                        .expect("successful recovery must remove the failed cleanup transaction");
+                    assert_eq!(empty.scanned, 0);
                 }
             }
             assert_eq!(
@@ -1062,20 +1132,7 @@ mod serial_tests {
     #[serial]
     #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
     fn test_transition_and_restore_flows() {
-        std::thread::Builder::new()
-            .name("scanner-transition-restore-flows".to_string())
-            .stack_size(32 * 1024 * 1024)
-            .spawn(|| {
-                let runtime = tokio::runtime::Builder::new_current_thread()
-                    .enable_all()
-                    .build()
-                    .expect("transition and restore test runtime should build");
-
-                runtime.block_on(test_transition_and_restore_flows_inner());
-            })
-            .expect("transition and restore test thread should spawn")
-            .join()
-            .expect("transition and restore test thread should finish");
+        run_large_stack_async_test("scanner-transition-restore-flows", test_transition_and_restore_flows_inner);
     }
 
     async fn test_transition_and_restore_flows_inner() {

crates/scanner/tests/storage_api/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
+pub(crate) use rustfs_ecstore::api::bucket::lifecycle::transition_transaction::recover_transition_transaction_records;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
     bucket_lifecycle_audit::LcEventSrc,
     bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry},
@@ -50,7 +50,7 @@ pub(crate) mod lifecycle {
         TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
         enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
         get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
-        is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
+        is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_transition_transaction_records,
         register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
     };
 }

deny.toml

@@ -36,8 +36,9 @@ unknown-registry = "deny"
 unknown-git = "deny"
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 allow-git = [
-    # Temporary tokio-tar fork pinned to the reviewed bounded extension parser
-    # change while astral-sh/tokio-tar#118 awaits an upstream release.
+    # Temporary tokio-tar fork pinned to the reviewed parser limits,
+    # cancellation safety, and error-fusing change while
+    # astral-sh/tokio-tar#118 awaits an upstream release.
     # owner: cxymds  review: 2026-10
     "https://github.com/cxymds/tokio-tar.git",
     # Official s3s repository. Temporarily pinned to the merged generic REST