test(e2e): rename stall timing variable flagged by typos

Add ListObjectsV2 paging, Range GET/HEAD, unversioned buckets, standard
and user metadata replay, ResponseStatus/TruncateBodyAt/Stall fault
actions, Range/User-Agent/prefix/continuation-token journal fields,
count_requests, direct seeding, and a configurable object cap to the
programmable fake S3 target, and add the on_demand_migration e2e
harness (OdmTestEnv, admin wrappers, source seeding, local-state
assertions, second RustFS source) with its self-test.

.agents/skills/arch-checks/SKILL.md

@@ -50,10 +50,11 @@ consider adding it to the script's `checked_files` list.
 
 ## `check_doc_paths.sh`
 
-Instruction/architecture docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`,
-`docs/architecture/*.md`) must not reference repo file paths that no longer
-exist. If your refactor moved code, update the docs that point at it — the
-error message lists `doc -> stale-path` pairs.
+Instruction docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`) and every
+Markdown file under `docs/` (architecture, operations, testing, index) must not
+reference repo file paths that no longer exist. If your refactor moved code,
+update the docs that point at it — the error message lists `doc -> stale-path`
+pairs. Cite paths plus symbol names, never line numbers (see `docs/README.md`).
 
 ## `check_no_planning_docs.sh`
 

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
-sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2
+sha256-darwin=9dccb0cd537cf79ae70c1c20e8281d36d03f2f09f81142a5341e26e3dc18709d
+sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4

.config/e2e-nightly-selection.txt

@@ -1 +1 @@
-sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105
+sha256=d06524b44de97ed8f62b0fd8cf9fa504e3cd520ffcaacc32691d6f890ebe7f20

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=dbebfbab9b9efd4eff31211e69dd32235dc00e207f2ab0dd919a1b2ac9e724c2
+sha256=db9bd8cdcb0abe43461aa6b36499b17cabd4098e5b34e300b1a0f0d0f34d9884

.config/make/coverage.mak

@@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow,
 	@mkdir -p target/llvm-cov
 	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
 	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
+	$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json

.config/make/lint-fmt.mak

@@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 .PHONY: test-wiring-check
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
-	python3 ./scripts/check_test_wiring.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -35,13 +35,14 @@ script-tests: ## Run shell script tests
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/test_fuzz_runner.sh
+	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
-	python3 ./scripts/check_test_wiring.py --self-test
-	python3 ./scripts/check_security_coverage.py --self-test
-	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-	python3 ./scripts/s3-tests/test_report_compat.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
-	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
 
 .PHONY: test

.config/nextest.toml

@@ -65,7 +65,7 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
 
 # --- default profile (local): serialize the flaky groups, never retry --------
 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.default.scripts]]
@@ -89,6 +89,29 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests. Every test in the module builds a
+# real-disk (TempDir-backed) hermetic erasure set and drives MiB-scale writes
+# plus deep-scan heal — the same load-sensitive cross-disk IO shape as the
+# crash_consistency scenarios above. Under a heavily parallel run a single
+# disk's IO can fail while write quorum still holds, which flips per-disk
+# readback and aggregate-outcome assertions nondeterministically (different
+# tests each round; all pass standalone). Preventive serialization only, no
+# retries. The matching ci-profile override is after [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair. Both carry
+# #[serial(metadata_cache_invalidation_probe)] — a no-op across nextest's
+# process boundary — and assert get_object_metadata_cache generation
+# semantics on a 4-disk hermetic set, the same load-sensitive shape that
+# forced the transition matrix tests into this group. Preventive
+# serialization only, no retries. The matching ci-profile override is after
+# [profile.ci].
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # The production-handler relocation regression builds an isolated 8-disk,
 # 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
 # from overlapping the ecstore commit fixtures above.
@@ -183,7 +206,7 @@ fail-fast = false
 path = "junit.xml"
 
 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
 setup = 'ecstore-large-stack'
 
 [[profile.ci.scripts]]
@@ -250,6 +273,20 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the heal result-report tests under the ci profile too (see the
+# matching default-profile override near the top). Not a quarantine: no
+# retries, just serialized real-disk heal IO.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
+test-group = 'ecstore-serial-flaky'
+
+# Serialize the metadata-cache generation-retirement pair under the ci
+# profile too (see the matching default-profile override near the top). Not a
+# quarantine: no retries.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
+test-group = 'ecstore-serial-flaky'
+
 # Match the default-profile embedded test isolation without quarantining or
 # retrying failures in CI.
 [[profile.ci.overrides]]
@@ -318,7 +355,8 @@ test-group = 'ecstore-serial-flaky'
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
 # regexes byte-identical. The committed profile selection digests make changes
-# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
+# visible in CI; list current membership with `cargo nextest list -p e2e_test
+# --profile <profile>` (platform-dependent; see docs/testing/README.md).
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
 # SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
@@ -440,7 +478,7 @@ path = "junit.xml"
 [profile.e2e-nightly]
 default-filter = """
     package(e2e_test)
-    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
 """
 fail-fast = false
 
@@ -471,7 +509,7 @@ path = "junit.xml"
 # quota, checksum, encryption,
 # security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
 # skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
-# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
+# --profile e2e-full -p e2e_test` (platform-dependent; see docs/testing/README.md).
 #
 # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
 #   * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
@@ -496,7 +534,7 @@ path = "junit.xml"
 default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
-    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
     & !test(/^replication_extension_test::/)
 """
 fail-fast = false

.github/workflows/audit.yml

@@ -24,8 +24,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
@@ -37,8 +40,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
+      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
     # Daily, not weekly. This schedule exists to catch RustSec advisories
@@ -146,6 +152,12 @@ jobs:
       - name: Check performance A/B workflow trust boundary
         run: ./scripts/security/check_performance_ab_workflow.sh
 
+      - name: Check tier evidence workflow isolation
+        run: ./scripts/security/check_tier_artifact_workflow.sh
+
+      - name: Check package version contract
+        run: ./scripts/test_package_versions.sh
+
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -244,7 +244,7 @@ jobs:
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 150
+    timeout-minutes: 180
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Release binaries ship without dial9 telemetry and therefore do not need
@@ -408,9 +408,9 @@ jobs:
 
           if [[ "${{ matrix.cross }}" == "true" ]]; then
             # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bins
+            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
           fi
 
       - name: Create release package
@@ -1065,7 +1065,7 @@ jobs:
           while IFS= read -r preview_tag; do
             [[ -n "$preview_tag" ]] || continue
             echo "🧹 Deleting preview release $preview_tag (tag kept)"
-            gh release delete "$preview_tag" --yes
+            gh release delete "$preview_tag" --repo "${GITHUB_REPOSITORY}" --yes
             DELETED=$((DELETED + 1))
           done < <(
             jq -r --arg tag "$TAG" '

.github/workflows/e2e-upgrade.yml

@@ -49,8 +49,20 @@ env:
   UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
 
 jobs:
-  direct-upgrade:
-    name: Direct upgrade from rc.2
+  upgrade:
+    name: ${{ matrix.name }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Direct upgrade from rc.2
+            cache_key: e2e-direct-upgrade
+            test: direct_upgrade_from_rc2_preserves_object_contracts
+            artifact: direct-upgrade
+          - name: Mixed-version rolling upgrade from rc.2
+            cache_key: e2e-mixed-version-upgrade
+            test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
+            artifact: mixed-version-upgrade
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -64,7 +76,7 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: e2e-direct-upgrade
+          cache-shared-key: ${{ matrix.cache_key }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: "false"
 
@@ -89,17 +101,17 @@ jobs:
           cargo build --locked -p rustfs --bin rustfs
           : > target/debug/rustfs.features
 
-      - name: Run direct-upgrade compatibility test
+      - name: Run upgrade compatibility test
         run: |
           cargo test --locked -p e2e_test \
-            upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \
+            "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: direct-upgrade-server-logs-${{ github.run_number }}
+          name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs
           if-no-files-found: warn
           retention-days: 14

.github/workflows/package.yml

@@ -21,10 +21,10 @@
 # - workflow_run: automatically package after "Build and Release" completes
 #   for a release tag (the mac/windows/linux binaries are already uploaded
 #   to the GitHub release before packaging starts)
-# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
+# - workflow_dispatch: manual fallback with a release tag and/or exact build run ID
 #
 # Flow:
-# 1. Resolve the triggering Build workflow run for the release tag
+# 1. Resolve and validate the selected Build workflow run and source identity
 # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
 # 3. Build DEB packages for amd64 and arm64
 # 4. Build RPM packages for x86_64 and aarch64
@@ -51,7 +51,7 @@ on:
         required: false
         type: string
       build_run_id:
-        description: "Build workflow run ID (overrides tag lookup)"
+        description: "Build workflow run ID (when combined with tag, both must identify the same release commit)"
         required: false
         type: string
 
@@ -82,6 +82,9 @@ jobs:
       version: ${{ steps.resolve.outputs.version }}
       build_type: ${{ steps.resolve.outputs.build_type }}
       build_run_id: ${{ steps.resolve.outputs.build_run_id }}
+      build_run_number: ${{ steps.resolve.outputs.build_run_number }}
+      head_sha: ${{ steps.resolve.outputs.head_sha }}
+      dev_sequence: ${{ steps.resolve.outputs.dev_sequence }}
       tag: ${{ steps.resolve.outputs.tag }}
     steps:
       - name: Resolve build run
@@ -89,90 +92,129 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
+          EVENT_NAME: ${{ github.event_name }}
+          REPOSITORY: ${{ github.repository }}
           INPUT_TAG: ${{ github.event.inputs.tag }}
           INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
         run: |
           set -euo pipefail
 
-          # Determine tag
-          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            TAG="${HEAD_BRANCH}"
-          elif [[ -n "$INPUT_TAG" ]]; then
-            TAG="$INPUT_TAG"
+          fail() {
+            echo "❌ $1" >&2
+            exit 1
+          }
+
+          TAG=""
+          BUILD_RUN_ID=""
+          case "$EVENT_NAME" in
+            workflow_run)
+              TAG="$HEAD_BRANCH"
+              BUILD_RUN_ID="$WORKFLOW_RUN_ID"
+              ;;
+            workflow_dispatch)
+              TAG="$INPUT_TAG"
+              BUILD_RUN_ID="$INPUT_RUN_ID"
+              ;;
+            *) fail "unsupported event: $EVENT_NAME" ;;
+          esac
+
+          # Validate and classify tags before using them in API paths or logs.
+          semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)'
+          prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)'
+          if [[ -n "$TAG" ]]; then
+            if [[ "$TAG" =~ ^${semver_core}-${prerelease_id}-preview\.(0|[1-9][0-9]*)$ ]]; then
+              BUILD_TYPE=preview
+            elif [[ "$TAG" =~ ^${semver_core}-${prerelease_id}$ ]]; then
+              BUILD_TYPE=prerelease
+            elif [[ "$TAG" =~ ^${semver_core}$ ]]; then
+              BUILD_TYPE=release
+            else
+              fail "tag is not a supported strict package version"
+            fi
           else
-            TAG=""
+            BUILD_TYPE=development
           fi
 
-          echo "Tag: ${TAG:-<none>}"
-
-          # Determine build run ID
-          BUILD_RUN_ID=""
-
-          if [[ -n "$INPUT_RUN_ID" ]]; then
-            # Explicit run ID takes priority
-            BUILD_RUN_ID="$INPUT_RUN_ID"
-            echo "Using explicit build run ID: $BUILD_RUN_ID"
-
-          elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            # Use the Build and Release run that triggered this workflow
-            BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
-            echo "Using triggering workflow run: $BUILD_RUN_ID"
-
+          if [[ -n "$BUILD_RUN_ID" ]]; then
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "build run ID must be a positive decimal integer"
+            echo "Using selected build run: $BUILD_RUN_ID"
           elif [[ -n "$TAG" ]]; then
-            # Find the build run that produced this tag
             echo "Looking for build run for tag: $TAG"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
 
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              # Tag might not be a branch; try event=push with head_branch matching
-              BUILD_RUN_ID=$(gh api \
-                "repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
-                --jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
-            fi
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful build run found for tag: $TAG"
-              exit 1
+            if [[ -z "$BUILD_RUN_ID" ]]; then
+              BUILD_RUN_ID=$(gh api --method GET \
+                "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+                -f event=push -f status=success -F per_page=100 2>/dev/null |
+                jq -r --arg tag "$TAG" \
+                  '[.workflow_runs[] | select(.head_branch == $tag)][0].id // empty' || true)
             fi
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful build run found for tag"
             echo "Found build run: $BUILD_RUN_ID"
-
           else
-            # No tag — latest successful main build
             echo "No tag specified, looking for latest main build"
-            BUILD_RUN_ID=$(gh api \
-              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
-              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
-
-            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
-              echo "❌ No successful main build found"
-              exit 1
-            fi
+            BUILD_RUN_ID=$(gh api --method GET \
+              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch=main -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
+            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful main build found"
             echo "Latest main build: $BUILD_RUN_ID"
           fi
 
-          # Determine version and build type
+          # Fetch once and use the same immutable run metadata for identity,
+          # ordering, workflow provenance, and release-channel validation.
+          RUN_JSON=$(gh api "repos/${REPOSITORY}/actions/runs/${BUILD_RUN_ID}") ||
+            fail "cannot read selected build run"
+          RUN_ID=$(jq -r '.id // empty' <<<"$RUN_JSON")
+          RUN_NUMBER=$(jq -r '.run_number // empty' <<<"$RUN_JSON")
+          RUN_STATUS=$(jq -r '.status // empty' <<<"$RUN_JSON")
+          RUN_CONCLUSION=$(jq -r '.conclusion // empty' <<<"$RUN_JSON")
+          RUN_PATH=$(jq -r '.path // empty' <<<"$RUN_JSON")
+          HEAD_SHA=$(jq -r '.head_sha // empty' <<<"$RUN_JSON")
+          RUN_HEAD_BRANCH=$(jq -r '.head_branch // empty' <<<"$RUN_JSON")
+
+          [[ "$RUN_ID" == "$BUILD_RUN_ID" ]] || fail "run metadata ID mismatch"
+          [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] || fail "build run number must be a positive decimal integer"
+          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]] || fail "selected build run is not successful"
+          [[ "$RUN_PATH" == .github/workflows/build.yml ]] || fail "selected run is not Build and Release"
+          [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "selected build run has an invalid head SHA"
+          [[ "$RUN_HEAD_BRANCH" != *$'\n'* && -n "$RUN_HEAD_BRANCH" ]] || fail "selected build run has an invalid head branch"
+
           if [[ -n "$TAG" ]]; then
+            [[ "$RUN_HEAD_BRANCH" == "$TAG" ]] || fail "tag and build run head branch do not match"
+
+            TAG_REF_JSON=$(gh api "repos/${REPOSITORY}/git/ref/tags/${TAG}") ||
+              fail "cannot resolve release tag ref"
+            TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_REF_JSON")
+            TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_REF_JSON")
+            depth=0
+            while [[ "$TAG_OBJECT_TYPE" == tag && $depth -lt 5 ]]; do
+              TAG_OBJECT_JSON=$(gh api "repos/${REPOSITORY}/git/tags/${TAG_OBJECT_SHA}") ||
+                fail "cannot peel annotated release tag"
+              TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_OBJECT_JSON")
+              TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_OBJECT_JSON")
+              depth=$((depth + 1))
+            done
+            [[ "$TAG_OBJECT_TYPE" == commit && "$TAG_OBJECT_SHA" =~ ^[0-9a-f]{40}$ ]] ||
+              fail "release tag does not resolve to a commit"
+            [[ "$TAG_OBJECT_SHA" == "$HEAD_SHA" ]] || fail "release tag commit and build run head SHA do not match"
             VERSION="$TAG"
-            if [[ "$TAG" == *"-preview"* ]]; then
-              BUILD_TYPE="preview"
-            elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
-              BUILD_TYPE="prerelease"
-            else
-              BUILD_TYPE="release"
-            fi
+            DEV_SEQUENCE=""
           else
-            SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
-              --jq '.head_sha' 2>/dev/null | head -c 7)
-            VERSION="dev-${SHORT_SHA}"
-            BUILD_TYPE="development"
+            VERSION="dev-${HEAD_SHA}"
+            DEV_SEQUENCE="$RUN_NUMBER"
           fi
 
           {
             echo "version=$VERSION"
             echo "build_type=$BUILD_TYPE"
             echo "build_run_id=$BUILD_RUN_ID"
+            echo "build_run_number=$RUN_NUMBER"
+            echo "head_sha=$HEAD_SHA"
+            echo "dev_sequence=$DEV_SEQUENCE"
             echo "tag=${TAG}"
           } >> "$GITHUB_OUTPUT"
 
@@ -180,6 +222,7 @@ jobs:
           echo "  Version: $VERSION"
           echo "  Build type: $BUILD_TYPE"
           echo "  Build run ID: $BUILD_RUN_ID"
+          echo "  Build run number: $RUN_NUMBER"
 
   # Build DEB and RPM packages for each architecture
   package:
@@ -206,6 +249,22 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Normalize package metadata
+        id: versions
+        shell: bash
+        env:
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
+          DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+        run: |
+          set -euo pipefail
+
+          normalized=$(./scripts/release/package_versions.sh \
+            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
+          printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
+
       - name: Download binary artifact from build run
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
@@ -245,18 +304,16 @@ jobs:
       - name: Build DEB package
         id: deb
         shell: bash
+        env:
+          DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
+          DEB_ARCH: ${{ matrix.deb_arch }}
+          DEB_FILE: ${{ steps.versions.outputs.deb_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          DEB_ARCH="${{ matrix.deb_arch }}"
-          # DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
-          # Use a variable for ~ to prevent tilde expansion by bash
-          TILDE='~'
-          DEB_VERSION="${VERSION/-/$TILDE}"
-          PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
+          PKG_DIR="${DEB_FILE%.deb}"
 
-          echo "Building DEB: ${PKG_DIR}.deb"
+          echo "Building DEB: ${DEB_FILE}"
 
           mkdir -p "${PKG_DIR}/DEBIAN"
           mkdir -p "${PKG_DIR}/usr/bin"
@@ -333,9 +390,12 @@ jobs:
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
           cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
 
-          fakeroot dpkg-deb --build "${PKG_DIR}"
+          fakeroot dpkg-deb --build "${PKG_DIR}" "$DEB_FILE"
 
-          DEB_FILE="${PKG_DIR}.deb"
+          [[ $(dpkg-deb -f "$DEB_FILE" Package) == rustfs ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Version) == "$DEB_VERSION" ]]
+          [[ $(dpkg-deb -f "$DEB_FILE" Architecture) == "$DEB_ARCH" ]]
+          dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -tf - | grep -Fx './usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
@@ -343,16 +403,19 @@ jobs:
       - name: Build RPM package
         id: rpm
         shell: bash
+        env:
+          RPM_VERSION: ${{ steps.versions.outputs.rpm_version }}
+          RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
+          RPM_ARCH: ${{ matrix.rpm_arch }}
+          RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
         run: |
           set -euo pipefail
 
-          VERSION="${{ needs.resolve.outputs.version }}"
-          RPM_ARCH="${{ matrix.rpm_arch }}"
-
           echo "Building RPM for ${RPM_ARCH}"
 
-          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
+          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
           sudo gem install fpm
+          ./scripts/test_package_versions.sh --require-package-managers
 
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
@@ -367,8 +430,10 @@ jobs:
 
           fpm -s dir -t rpm \
             --name rustfs \
-            --version "$VERSION" \
+            --version "$RPM_VERSION" \
+            --iteration "$RPM_RELEASE" \
             --architecture "$RPM_ARCH" \
+            --package "$RPM_FILE" \
             --depends "glibc >= 2.31" \
             --maintainer "RustFS Team <support@rustfs.com>" \
             --description "High-performance distributed object storage" \
@@ -410,13 +475,15 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
-            RPM_FILE="${RPM_FILE#./}"
-          if [[ -z "$RPM_FILE" ]]; then
+          if [[ ! -f "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
+          RPM_METADATA=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$RPM_FILE")
+          EXPECTED_METADATA=$(printf 'rustfs\n%s\n%s\n%s' "$RPM_VERSION" "$RPM_RELEASE" "$RPM_ARCH")
+          [[ "$RPM_METADATA" == "$EXPECTED_METADATA" ]]
+          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
@@ -438,6 +505,9 @@ jobs:
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
           R2_BUCKET: ${{ secrets.R2_BUCKET }}
           AWS_EC2_METADATA_DISABLED: true
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
@@ -455,7 +525,6 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
@@ -465,9 +534,6 @@ jobs:
 
           echo "📤 Uploading to $R2_PATH"
 
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           for f in "$DEB_FILE" "$RPM_FILE"; do
             if [[ -n "$f" && -f "$f" ]]; then
               echo "Uploading: $f"
@@ -493,14 +559,13 @@ jobs:
         if: needs.resolve.outputs.tag != ''
         env:
           GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.resolve.outputs.tag }}
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
 
-          TAG="${{ needs.resolve.outputs.tag }}"
-          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
-          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
-
           # Upload the packages, then refresh the release checksums so the new
           # assets are covered, matching the binary release flow.
           for f in "$DEB_FILE" "$RPM_FILE"; do
@@ -552,14 +617,19 @@ jobs:
     steps:
       - name: Print summary
         shell: bash
+        env:
+          SUMMARY_VERSION: ${{ needs.resolve.outputs.version }}
+          SUMMARY_BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+          SUMMARY_BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          SUMMARY_PACKAGE_STATUS: ${{ needs.package.result }}
         run: |
           {
             echo "## 📦 Package Summary"
             echo ""
             echo "| Item | Value |"
             echo "|------|-------|"
-            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
-            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
-            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
-            echo "| Package Status | ${{ needs.package.result }} |"
+            echo "| Version | \`${SUMMARY_VERSION}\` |"
+            echo "| Build Type | ${SUMMARY_BUILD_TYPE} |"
+            echo "| Build Run | #${SUMMARY_BUILD_RUN_ID} |"
+            echo "| Package Status | ${SUMMARY_PACKAGE_STATUS} |"
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-functional-chain.yml

@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Functional chain driver: runs the ten functional suites in a fixed order
+# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
+# replication, with performance on its own runner in parallel) and guarantees
+# the chain keeps moving even when individual suites fail.
+#
+# Each suite workflow can still be dispatched standalone (workflow_dispatch);
+# only chain-triggered runs forward to the next suite via repository_dispatch,
+# so a standalone run never drags the rest of the chain behind it.
+#
+# Why not workflow_run chaining: GitHub does not guarantee delivery of
+# workflow_run events (they are fire-and-forget), and the head-SHA filter made
+# newly added suites (storage) unable to trigger at all. Explicit
+# repository_dispatch handoffs are verifiable and re-drivable.
+
+name: RustFS Functional Chain
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    # Entry point: start the chain after the nightly build completes. The
+    # build's own conclusion does not gate the chain; each suite reports its
+    # own result to rustfs/backlog and the dashboard.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  start-chain:
+    name: Start functional chain (upgrade first)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    steps:
+      - name: Dispatch first suite (upgrade)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-upgrade' \
+            -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -23,6 +23,11 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the storage suite finishes. Heal runs
+    # exactly once per chain; the pool expansion workflow no longer embeds
+    # its own heal pass.
+    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -49,19 +54,33 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 480
-    # Manual-only standalone run. Nightly chain already runs heal in
-    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
-    if: ${{ github.event_name == 'workflow_dispatch' }}
+    # Standalone manual run, or one link of the nightly functional chain
+    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -116,8 +135,8 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
+            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
+            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
             --log-file /tmp/rustfs-heal-test.log
 
       - name: Generate report
@@ -175,155 +194,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'heal'
+          SUITE_LABEL: 'Heal'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-heal-report.md'
+          LOG_FILE: '/tmp/rustfs-heal-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -354,6 +282,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Pool expansion)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-pool' \
+              -F 'client_payload[from_suite]=heal'; then
+              echo "dispatched next suite Pool expansion (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-pool'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-kms-test.yml

@@ -23,10 +23,9 @@ on:
         description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after S3 compatibility test completes.
-    workflows: ["RustFS S3 Compatibility Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the S3 compatibility suite finishes.
+    types: [rustfs-chain-kms]
 
 permissions:
   contents: read
@@ -52,16 +51,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -240,105 +250,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'kms'
+          SUITE_LABEL: 'KMS'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-kms-report.md'
+          LOG_FILE: '/tmp/rustfs-kms.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -369,6 +338,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Tier)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-tier' \
+              -F 'client_payload[from_suite]=kms'; then
+              echo "dispatched next suite Tier (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-tier'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-performance-test.yml

@@ -48,10 +48,10 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Run after the nightly build completes; the nightly deb is what the test installs.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
+    types: [rustfs-chain-performance]
 
 permissions:
   contents: read
@@ -84,19 +84,33 @@ env:
 jobs:
   performance-test:
     runs-on: pf-testing
+    # Requirement: a failing benchmark must not fail the workflow;
+    # failures are filed to rustfs/backlog.
+    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -212,6 +226,65 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'performance'
+          SUITE_LABEL: 'Performance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-perf-report.md'
+          LOG_FILE: '/tmp/rustfs-perf-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
       - name: Upload test logs & results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion / Heal Test
+name: RustFS Pool Expansion Test
 
 on:
   workflow_dispatch:
@@ -33,14 +33,6 @@ on:
         description: 'Run the pool decommission step (3-pool topology only)'
         type: boolean
         default: true
-      stop_node_gb:
-        description: 'Heal: stop the outage node when surviving nodes reach N GiB'
-        required: false
-        default: '15'
-      warp_stop_gb:
-        description: 'Heal: stop warp when surviving nodes reach N GiB'
-        required: false
-        default: '40'
       cleanup_before:
         description: 'Reset the nodes before the test (DESTROYS existing data/config)'
         type: boolean
@@ -49,17 +41,16 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the heal suite finishes.
+    types: [rustfs-chain-pool]
 
 permissions:
   contents: read
 
-# Only one test run at a time: every job mutates the same shared test
+# Only one test run at a time: the job mutates the same shared test
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
-# other. Jobs inside a run are chained with needs to serialize them.
+# other.
 concurrency:
   group: rustfs-shared-functional-tests
   cancel-in-progress: false
@@ -80,330 +71,16 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  heal-test:
-    name: Heal test
-    runs-on: smoke-testing
-    timeout-minutes: 480
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        if: ${{ inputs.cleanup_before != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Install RustFS package & start cluster
-        run: |
-          ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Preflight checks
-        run: |
-          ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Run heal test (write -> outage -> heal -> verify)
-        id: test
-        run: |
-          ARGS=(--steps "3,4,5,6,7" -y \
-            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          {
-            echo "# RustFS heal test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-          SUITE: heal
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          fi
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-heal-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-heal-test.log
-            /tmp/rustfs-warp.*.log
-          if-no-files-found: warn
-
-      - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS heal test failed"
-          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
-          echo "See the uploaded log artifact for details."
-
-  # Pool expansion runs after heal regardless of heal outcome.
+  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
+  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
+    continue-on-error: true
     timeout-minutes: 360
-    needs: heal-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run') }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -411,14 +88,25 @@ jobs:
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Initialize pool test artifacts
         run: |
@@ -810,155 +498,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'pool'
+          SUITE_LABEL: 'Pool expansion'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
+          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload test logs
         if: always()
@@ -997,6 +594,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Security)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-security' \
+              -F 'client_payload[from_suite]=pool'; then
+              echo "dispatched next suite Security (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-security'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-replication-test.yml

@@ -0,0 +1,365 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Replication Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      suite:
+        description: 'Suite to run (all = bucket REP-* then site SITE-*)'
+        type: choice
+        options:
+          - all
+          - bucket
+          - site
+        default: all
+  repository_dispatch:
+    # Chain handoff: dispatched when the security suite finishes. This is the
+    # last link of the functional chain.
+    types: [rustfs-chain-replication]
+
+permissions:
+  contents: read
+
+# The replication suite uses the same shared VMs as the other functional
+# tests, so it must serialize with them instead of running in parallel.
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  replication-test:
+    runs-on: smoke-testing
+    # A failed replication run must not break the chain or the workflow: the
+    # failure is reported to rustfs/backlog instead (see the issue step).
+    continue-on-error: true
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    steps:
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          aws --version
+          df -h /data | tail -1 || true
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2 /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Run replication suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-replication-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          SUITE='${{ inputs.suite }}'
+          ARGS=(-y --log-file "${LOG_FILE}")
+          if [ "${SUITE}" = "all" ] || [ -z "${SUITE}" ] || [ "${SUITE}" = "null" ]; then
+            ARGS+=(--suite all)
+          else
+            ARGS+=(--suite "${SUITE}")
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
+          REPORT_FILE: /tmp/rustfs-replication-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          RUSTFS_VERSION_INFO="N/A"
+          if [ "${#NODES[@]}" -gt 0 ]; then
+            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
+            if [ -n "${DETECTED_VERSION}" ]; then
+              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
+            fi
+          fi
+          CASE_TABLE="/tmp/rustfs-replication-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = start_re.match(line)
+                      if m:
+                          case_id, name = m.group(1), m.group(2)
+                          if case_id not in index:
+                              index[case_id] = len(rows)
+                              rows.append([case_id, name, 'RUNNING'])
+                          continue
+                      m = done_re.match(line)
+                      if m:
+                          status, case_id = m.group(1), m.group(2)
+                          if case_id in index:
+                              rows[index[case_id]][2] = status
+                          else:
+                              rows.append([case_id, case_id, status])
+                              index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+              rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+              counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Case Summary\n\n')
+              out.write(f"- Total: {len(rows)}\\n")
+              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+              out.write('\\n')
+              out.write('| Case | Name | Status |\\n')
+              out.write('| --- | --- | --- |\\n')
+              for case_id, name, status in rows:
+                  out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS replication test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-replication-report.md
+          SUITE: replication
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'replication'
+          SUITE_LABEL: 'Replication (bucket + site)'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-replication-report.md'
+          LOG_FILE: '/tmp/rustfs-replication.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-replication-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-replication.log
+            /tmp/rustfs-replication-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2
+            '
+          done
+
+      - name: Chain complete
+        # Replication is the last link of the functional chain: nothing to
+        # dispatch after it. This step just records that the chain finished.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        run: |
+          echo "Functional chain complete: replication (final suite) finished."
+          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS replication suite failed"
+          echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-s3-compat-test.yml

@@ -11,10 +11,9 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Run after upgrade compatibility completes; the nightly deb is what the test installs.
-    workflows: ["RustFS Upgrade Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the upgrade suite finishes.
+    types: [rustfs-chain-s3]
 
 permissions:
   contents: read
@@ -40,16 +39,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -220,155 +230,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 's3'
+          SUITE_LABEL: 'S3 compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+          LOG_FILE: '/tmp/rustfs-s3-compat.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -399,6 +318,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: KMS)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-kms' \
+              -F 'client_payload[from_suite]=s3'; then
+              echo "dispatched next suite KMS (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-kms'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-security-test.yml

@@ -46,10 +46,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs last in the functional chain, after pool/heal, on the shared VMs.
-    workflows: ["RustFS Pool Expansion / Heal Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the pool expansion suite finishes.
+    types: [rustfs-chain-security]
 
 permissions:
   contents: read
@@ -77,16 +76,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Checkout repository (for the OIDC live gate script)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -193,155 +203,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'security'
+          SUITE_LABEL: 'Security'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-security-report.md'
+          LOG_FILE: ''
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -373,6 +292,24 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Replication)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Replication"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-replication' \
+            -F 'client_payload[from_suite]=security'
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-storage-test.yml

@@ -20,10 +20,9 @@ on:
           - single-multi
           - multi-multi
         default: all
-  workflow_run:
-    # Strict shared-environment order: run after tier test completes.
-    workflows: ["RustFS Tier Test"]
-    types: [completed]
+  repository_dispatch:
+    # Chain handoff: dispatched when the tier suite finishes.
+    types: [rustfs-chain-storage]
 
 permissions:
   contents: read
@@ -49,16 +48,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -235,156 +245,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'storage', label: 'Storage Engine' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'storage'
+          SUITE_LABEL: 'Storage engine'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-storage-report.md'
+          LOG_FILE: '/tmp/rustfs-storage.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -415,6 +333,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: Heal)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-heal' \
+              -F 'client_payload[from_suite]=storage'; then
+              echo "dispatched next suite Heal (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-heal'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-tier-test.yml

@@ -11,10 +11,18 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  workflow_run:
-    # Strict shared-environment order: run after KMS test completes.
-    workflows: ["RustFS KMS Test"]
-    types: [completed]
+      rc_sha256:
+        description: 'Optional SHA-256 for the preinstalled rc binary; mismatch is an infrastructure failure.'
+        required: false
+        type: string
+      force_case_failure:
+        description: 'Diagnostic only: rewrite single-single/TIER-101 to FAIL after execution to verify artifact and final-gate behavior.'
+        required: false
+        default: false
+        type: boolean
+  repository_dispatch:
+    # Chain handoff: dispatched when the KMS suite finishes.
+    types: [rustfs-chain-tier]
 
 permissions:
   contents: read
@@ -33,23 +41,50 @@ env:
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  RUSTFS_EXPECTED_RC_SHA256: ${{ inputs.rc_sha256 || vars.RUSTFS_TIER_RC_SHA256 }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
 
 jobs:
   tier-test:
     runs-on: smoke-testing
+    # Requirement: a failing suite must not fail the workflow; failures
+    # are filed to rustfs/backlog and the chain continues.
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      - name: Initialize run evidence directory
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then
+            echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2
+            exit 1
+          fi
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -110,13 +145,25 @@ jobs:
         id: test
         continue-on-error: true
         env:
-          LOG_FILE: /tmp/rustfs-tier.log
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
         run: |
           set -euo pipefail
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
           chmod +x auto-testing/rustfs-tier-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          RC_BIN="$(command -v rc)"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
+          ARGS=(
+            --all-topologies
+            -y
+            --log-file "${LOG_FILE}"
+            --rc-bin "${RC_BIN}"
+            --artifacts-dir "${TIER_ARTIFACTS_DIR}"
+          )
+          if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
+            ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
+          fi
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -126,15 +173,34 @@ jobs:
           fi
           ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
 
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-tier.log
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+      - name: Inject diagnostic case failure
+        if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }}
         run: |
           set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json"
+          test -s "${RESULT_FILE}"
+          TMP_FILE="$(mktemp "${TIER_ARTIFACTS_DIR}/cases/.forced.XXXXXX")"
+          jq '.status = "FAIL" | .case_rc = 97' "${RESULT_FILE}" > "${TMP_FILE}"
+          mv "${TMP_FILE}" "${RESULT_FILE}"
+
+      - name: Generate report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        env:
+          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
+          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          TRIGGER_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          test -d "${TIER_ARTIFACTS_DIR}"
+          test ! -L "${TIER_ARTIFACTS_DIR}"
+          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
+          REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md"
+          CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc"
+          PACKAGE_URL="${PACKAGE_URL_INPUT}"
+          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
           if [ -n "${PACKAGE_URL}" ]; then
             PACKAGE_SOURCE="${PACKAGE_URL}"
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -142,65 +208,31 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-tier-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
+          set +e
+          python3 auto-testing/rustfs_tier_report.py \
+            --results-dir "${TIER_ARTIFACTS_DIR}/cases" \
+            --provenance "${TIER_ARTIFACTS_DIR}/provenance.json" \
+            --output "${CASE_TABLE}"
+          CASE_GATE_RC=$?
+          set -e
+          printf '%s\n' "${CASE_GATE_RC}" > "${GATE_RC_FILE}"
+          if [ ! -s "${CASE_TABLE}" ]; then
+            {
+              echo "## Case Summary"
+              echo ""
+              echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
+            } > "${CASE_TABLE}"
+          fi
           {
             echo "# RustFS tier test report"
             echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${TRIGGER_NAME}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${TEST_OUTCOME}"
+            echo "- Structured Gate Exit: ${CASE_GATE_RC}"
             echo ""
-            cat "${CASE_TABLE}" || true
+            cat "${CASE_TABLE}"
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -210,11 +242,11 @@ jobs:
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Upload functional report to dashboard
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-tier-report.md
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
           SUITE: tier
         run: |
           set -euo pipefail
@@ -236,115 +268,42 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-                { key: 'upgrade', label: 'Upgrade' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('s3');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: Verify required tier evidence
+        id: evidence_verify
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          failed=0
+          for name in \
+            rustfs-tier.log \
+            rustfs-tier-report.md \
+            rustfs-tier-cases.md \
+            rustfs-tier-gate.rc \
+            provenance.json; do
+            if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence is missing or empty: ${name}" >&2
+              failed=1
+            fi
+          done
+          for name in cases logs; do
+            if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then
+              echo "required tier evidence directory is missing: ${name}" >&2
+              failed=1
+            fi
+          done
+          if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then
+            echo "no atomic tier case result was produced" >&2
+            failed=1
           fi
+          [ "${failed}" -eq 0 ]
 
       - name: Upload report and logs
-        if: always()
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-tier-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-tier.log
-            /tmp/rustfs-tier-report.md
-          if-no-files-found: warn
+          name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.TIER_ARTIFACTS_DIR }}/
+          if-no-files-found: error
 
       - name: Cleanup environment (after)
         if: always()
@@ -367,6 +326,157 @@ jobs:
             '
           done
 
+      - name: Enforce tier suite result
+        id: gate
+        if: always()
+        env:
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          TEST_OUTCOME: ${{ steps.test.outcome }}
+          GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
+        run: |
+          set -euo pipefail
+          failed=0
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${TEST_OUTCOME}" != "success" ]; then
+            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
+            failed=1
+          fi
+          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+            echo "structured gate result is unavailable because evidence initialization failed" >&2
+          elif [ ! -s "${GATE_RC_FILE}" ]; then
+            echo "structured gate result is missing" >&2
+            failed=1
+          else
+            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
+            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
+              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
+              failed=1
+            fi
+          fi
+          [ "${failed}" -eq 0 ]
+
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'tier'
+          SUITE_LABEL: 'Tier'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
+          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
+          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
+          GATE_OUTCOME: ${{ steps.gate.outcome }}
+          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
+          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
+          fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
+            echo "- Evidence verification: ${VERIFY_OUTCOME}"
+            echo "- Final gate: ${GATE_OUTCOME}"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
+              echo "(the run evidence directory was rejected; its contents were not read)"
+            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
+              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
+            elif [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
+
+      - name: "Continue functional chain (next: Storage engine)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-storage' \
+              -F 'client_payload[from_suite]=tier'; then
+              echo "dispatched next suite Storage engine (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-storage'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-upgrade-test.yml

@@ -53,11 +53,9 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Runs first in the functional chain: upgrade compatibility gates the
-    # nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
+  repository_dispatch:
+    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
+    types: [rustfs-chain-upgrade]
 
 permissions:
   contents: read
@@ -83,16 +81,27 @@ jobs:
     runs-on: smoke-testing
     continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
 
       - name: Show environment
         run: |
@@ -288,155 +297,64 @@ jobs:
               | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
           fi
 
-          cat > /tmp/rustfs-functional-index.html <<'EOF'
-          <!doctype html>
-          <html lang="en">
-          <head>
-            <meta charset="utf-8" />
-            <meta name="viewport" content="width=device-width, initial-scale=1" />
-            <title>RustFS Functional Test Reports</title>
-            <style>
-              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
-              * { box-sizing: border-box; }
-              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
-              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
-              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
-              h1 { margin: 0 0 8px; font-size: 26px; }
-              p { margin: 0 0 14px; color: var(--muted); }
-              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
-              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
-              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-              .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
-              ul { list-style: none; margin: 0; padding: 0; }
-              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
-              a { color: var(--accent); text-decoration: none; }
-              a:hover { text-decoration: underline; }
-              .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
-              .kv { margin: 6px 0; color: var(--text); }
-              .muted { color: var(--muted); }
-            </style>
-          </head>
-          <body>
-            <div class="wrap">
-              <div class="card">
-                <h1>RustFS Functional Test Reports</h1>
-                <p>Select a suite and date to view the build version used in that run.</p>
-                <div class="tabs" id="tabs"></div>
-                <ul id="list"></ul>
-                <div class="meta">
-                  <div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
-                  <div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
-                  <div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
-                </div>
-              </div>
-            </div>
-            <script>
-              const suites = [
-                { key: 'upgrade', label: 'Upgrade' },
-                { key: 's3', label: 'S3 Compatibility' },
-                { key: 'kms', label: 'KMS' },
-                { key: 'tier', label: 'Tier' },
-                { key: 'heal', label: 'Heal' },
-                { key: 'pool', label: 'Pool Expansion' },
-                { key: 'security', label: 'Security' },
-              ];
-              const tabs = document.getElementById('tabs');
-              const list = document.getElementById('list');
-              const reportDate = document.getElementById('report-date');
-              const reportVersion = document.getElementById('report-version');
-              const reportLink = document.getElementById('report-link');
-
-              function parseVersion(markdown) {
-                const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
-                return m ? m[1].trim() : 'N/A';
-              }
-
-              async function showReport(report) {
-                reportDate.textContent = report.name.replace('.md', '');
-                reportVersion.textContent = 'Loading...';
-                reportLink.href = report.html_url;
-                try {
-                  const res = await fetch(report.download_url, { cache: 'no-store' });
-                  if (!res.ok) {
-                    reportVersion.textContent = 'N/A';
-                    return;
-                  }
-                  const text = await res.text();
-                  reportVersion.textContent = parseVersion(text);
-                } catch (_e) {
-                  reportVersion.textContent = 'N/A';
-                }
-              }
-
-              async function loadSuite(suite) {
-                list.innerHTML = '<li>Loading...</li>';
-                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
-                try {
-                  const res = await fetch(api);
-                  if (!res.ok) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    return;
-                  }
-                  const data = await res.json();
-                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
-                  if (!files.length) {
-                    list.innerHTML = '<li>No reports yet.</li>';
-                    reportDate.textContent = 'N/A';
-                    reportVersion.textContent = 'N/A';
-                    reportLink.href = '#';
-                    return;
-                  }
-                  list.innerHTML = '';
-                  files.forEach((f) => {
-                    const li = document.createElement('li');
-                    const btn = document.createElement('button');
-                    btn.className = 'report-btn';
-                    btn.textContent = f.name.replace('.md', '');
-                    btn.addEventListener('click', () => showReport(f));
-                    li.appendChild(btn);
-                    list.appendChild(li);
-                  });
-                  showReport(files[0]);
-                } catch (_e) {
-                  list.innerHTML = '<li>Failed to load reports.</li>';
-                  reportDate.textContent = 'N/A';
-                  reportVersion.textContent = 'N/A';
-                  reportLink.href = '#';
-                }
-              }
-
-              function setActive(key) {
-                for (const btn of tabs.querySelectorAll('button')) {
-                  btn.classList.toggle('active', btn.dataset.key === key);
-                }
-                loadSuite(key);
-              }
-
-              for (const suite of suites) {
-                const btn = document.createElement('button');
-                btn.textContent = suite.label;
-                btn.dataset.key = suite.key;
-                btn.addEventListener('click', () => setActive(suite.key));
-                tabs.appendChild(btn);
-              }
-              setActive('upgrade');
-            </script>
-          </body>
-          </html>
-          EOF
-
-          INDEX_PATH="functional/index.html"
-          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
-          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${INDEX_SHA}" ]; then
-            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+      - name: File failure issue in rustfs/backlog
+        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'upgrade'
+          SUITE_LABEL: 'Upgrade compatibility'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+          LOG_FILE: '/tmp/rustfs-upgrade.log'
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            exit 0
           fi
+          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+          EXISTING="$(gh issue list -R rustfs/backlog --state all \
+            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+            --json number --jq '.[].number' || true)"
+          if [ -n "${EXISTING}" ]; then
+            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+            exit 0
+          fi
+          redact() {
+            sed -E \
+              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+          }
+          BODY_FILE="$(mktemp)"
+          {
+            echo "The **${SUITE_LABEL}** functional suite failed."
+            echo ""
+            echo "- Suite: \`${SUITE}\`"
+            echo "- Run: ${RUN_URL}"
+            echo "- Trigger: ${GITHUB_EVENT_NAME}"
+            echo "- Date: $(date -u +%Y-%m-%d)"
+            echo ""
+            echo "## Report (errors and symptoms)"
+            echo ""
+            if [ -s "${REPORT_FILE}" ]; then
+              redact < "${REPORT_FILE}"
+            elif [ -s "${LOG_FILE:-}" ]; then
+              echo "(report file missing; log tail below)"
+              echo ""
+              tail -n 200 "${LOG_FILE}" | redact
+            else
+              echo "(no report or log file was produced)"
+            fi
+          } | head -c 55000 > "${BODY_FILE}"
+          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test; then
+            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          fi
+          echo "filed backlog issue for suite ${SUITE}"
 
       - name: Upload report and logs
         if: always()
@@ -468,6 +386,55 @@ jobs:
             '
           done
 
+      - name: "Continue functional chain (next: S3 compatibility)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-s3' \
+              -F 'client_payload[from_suite]=upgrade'; then
+              echo "dispatched next suite S3 compatibility (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-s3'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/star-history.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
+      - uses: overtrue/repo-visuals-action@fd79cba437ecfac933d00a69add17eb95d3939c3 # v1.3.1
         with:
           github-token: ${{ github.token }}
           output-branch: star-history

.gitignore

@@ -57,9 +57,6 @@ docs/*
 !docs/operations/**
 !docs/testing/
 !docs/testing/**
-docs/heal-scanner-logging-governance.md
-docs/benchmark/rustfs-target-bench/
-docs/benchmark/*.md
 .codegraph/*
 .docker/test/compat/data/*
 .docker/test/compat/kms/*
@@ -83,6 +80,8 @@ worktrees/*
 
 # Local AI-agent review artifacts (omo evidence dumps)
 .omo/
+# Legacy per-tool skill dir; skills live in .agents/skills (shared by all agents)
+.mimocode/
 
 # insta scratch files; the accepted .snap files ARE the assertions and are committed
 *.snap.new

AGENTS.md

@@ -86,6 +86,7 @@ This file contains repository-wide rules. Use the nearest subdirectory
 - CI gates: `.github/workflows/ci.yml`.
 - PR format: `.github/pull_request_template.md`.
 - Architecture routing: `ARCHITECTURE.md` and `docs/architecture/README.md`.
+- Knowledge-base index and documentation rules: `docs/architecture/README.md`.
 - Agent skills: `.agents/skills/*/SKILL.md`.
 
 Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
@@ -123,12 +124,13 @@ runtime/build output:
 - Use `make pre-commit` only when its repository-wide fast checks add confidence
   beyond the focused checks.
 
-### Broad or High-Risk Changes
+### Broad Cross-Module Changes
 
-After the required adversarial review, run `make pre-pr` when targeted coverage
-cannot bound the impact, including dependency/toolchain/build-matrix changes,
-unbounded cross-crate APIs, or locking, durability, erasure coding, replication,
-RPC, IAM/KMS/auth, cryptography, on-disk/on-wire, and S3-visible behavior.
+Do not run `make pre-pr` by default before opening a PR. Consider it only when
+the final diff is broad, spans multiple modules, and targeted checks cannot
+bound the impact. Decide dynamically from the affected boundaries and risks;
+otherwise use the scoped formatting, linting, compilation, and test checks
+above.
 
 `make pre-pr` includes `make pre-commit`; never run both for the same unchanged
 diff. Do not repeat a check already covered by a successful umbrella gate.

ARCHITECTURE.md

@@ -62,7 +62,7 @@ rustfs/                      # Workspace root (virtual manifest)
 │   ├── utils/               # Pure utility functions
 │   ├── ...                  # (see "Crate Reference" below)
 │   └── e2e_test/            # End-to-end integration tests
-└── docs/                    # Design documents and analysis
+└── docs/                    # Agent knowledge base: contracts, runbooks, testing rules (index: docs/architecture/README.md)
 ```
 
 ### Main Crate Layers (`rustfs/src/`)

CLAUDE.md

@@ -15,7 +15,7 @@ cargo check -p <crate>               # fast type-check one crate
 cargo test -p <crate>                # test one crate
 cargo fmt --all                      # format (required before PR)
 make pre-commit                      # fast gate: fmt + arch checks + quick-check (NO clippy/tests)
-make pre-pr                          # full pre-PR gate: fmt + arch checks + clippy + tests
+make pre-pr                          # optional full gate for broad cross-module changes
 make build-docker BUILD_OS=ubuntu22.04
 ```
 
@@ -27,6 +27,7 @@ make build-docker BUILD_OS=ubuntu22.04
 
 ## Where to look (do not duplicate here)
 
+- Agent knowledge base index and doc-writing rules: [docs/architecture/README.md](docs/architecture/README.md)
 - Crate membership: `Cargo.toml` `[workspace].members`
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)

CONTRIBUTING.md

@@ -62,12 +62,20 @@ make test
 # Fast pre-commit gate — see below for exactly what it runs
 make pre-commit
 
-# Full pre-PR gate (pre-commit gates + clippy + tests)
+# Optional full gate for broad cross-module changes (pre-commit + clippy + tests)
 make pre-pr
 ```
 
 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.
 
+> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the
+> security-coverage and scheduled-validation self-tests in `make test`) and import
+> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through
+> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls
+> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a
+> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a
+> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`.
+
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
 > For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
@@ -88,14 +96,16 @@ make pre-pr
 8. `quick-check` — `cargo check --workspace --exclude e2e_test`
 
 **`make pre-commit` does NOT run clippy and does NOT run any tests.**
-A green `make pre-commit` is not enough to open a pull request.
+It does not replace the scoped Clippy and test checks applicable to a change.
 
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
 tests). Complete the applicable multi-role adversarial review described in
-`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
-updating a pull request. This is what CI enforces.
+`AGENTS.md` first. Do not run `make pre-pr` locally by default before opening or
+updating a pull request. Consider it only for a broad change that spans multiple
+modules and whose impact cannot be bounded by targeted checks; decide from the
+affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -114,8 +124,9 @@ Or manually:
 chmod +x .git/hooks/pre-commit
 ```
 
-With or without a hook, the expectation is the same: run `make pre-commit`
-before committing and `make pre-pr` before opening a pull request.
+With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
+applicable scoped checks, and reserve `make pre-pr` for broad cross-module
+changes whose impact cannot be bounded by those checks.
 
 ### 📝 Formatting Configuration
 
@@ -154,7 +165,9 @@ Example output when formatting fails:
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
-6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
+6. **Run applicable scoped checks before opening/updating a PR**; consider
+   `make pre-pr` only for broad cross-module changes whose impact cannot be
+   bounded by targeted checks
 7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration

Cargo.lock

@@ -347,9 +347,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
 
 [[package]]
 name = "arrow"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
+checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -368,9 +368,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
+checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -382,9 +382,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
+checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -401,9 +401,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
+checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182"
 dependencies = [
  "bytes",
  "half",
@@ -413,9 +413,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
+checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -435,9 +435,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
+checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -450,9 +450,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
+checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -463,9 +463,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
+checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -479,9 +479,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
+checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -504,9 +504,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
+checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -517,9 +517,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
+checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -530,9 +530,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
+checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd"
 dependencies = [
  "serde_core",
  "serde_json",
@@ -540,9 +540,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
+checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -554,9 +554,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
+checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -919,9 +919,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.18.0"
+version = "1.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
 dependencies = [
  "aws-lc-sys",
  "untrusted 0.7.1",
@@ -930,9 +930,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.44.0"
+version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483"
+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
 dependencies = [
  "cc",
  "cmake",
@@ -4210,7 +4210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5039,9 +5039,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284"
 
 [[package]]
 name = "hex"
@@ -5668,7 +5668,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -6106,9 +6106,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libredox"
-version = "0.1.21"
+version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96"
+checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed"
 dependencies = [
  "libc",
 ]
@@ -6650,10 +6650,11 @@ dependencies = [
 
 [[package]]
 name = "mysql_async"
-version = "0.37.0"
+version = "0.37.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2"
+checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4"
 dependencies = [
+ "arc-swap",
  "bytes",
  "crossbeam-queue",
  "crossbeam-utils",
@@ -6986,9 +6987,9 @@ checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
 
 [[package]]
 name = "nvml-wrapper"
-version = "0.12.1"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
+checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
 dependencies = [
  "bitflags 2.13.1",
  "libloading",
@@ -7000,9 +7001,9 @@ dependencies = [
 
 [[package]]
 name = "nvml-wrapper-sys"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b"
+checksum = "5d2079f4c9b6d2170bfb71c6355734ead6c47da75c179847395c31f9f2f66ede"
 dependencies = [
  "libloading",
 ]
@@ -7527,9 +7528,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "59.2.0"
+version = "59.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
+checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -8036,9 +8037,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppmd-rust"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
+checksum = "9e9219bcb9d7aca6b2f63c83cf100cf78bcd619ac46e6ecbd0dd90869a39345d"
 
 [[package]]
 name = "ppv-lite86"
@@ -8610,7 +8611,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -9461,6 +9462,7 @@ dependencies = [
  "rustfs-io-metrics",
  "rustfs-keystone",
  "rustfs-kms",
+ "rustfs-license",
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
@@ -9512,7 +9514,7 @@ dependencies = [
  "tokio-util",
  "tonic",
  "tower",
- "tower-http 0.7.0",
+ "tower-http 0.7.1",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -10063,6 +10065,13 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustfs-license"
+version = "1.0.0-rc.5"
+dependencies = [
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "rustfs-lifecycle"
 version = "1.0.0-rc.5"
@@ -10082,7 +10091,6 @@ dependencies = [
  "time",
  "tokio",
  "tracing",
- "url",
  "uuid",
 ]
 
@@ -10606,6 +10614,8 @@ dependencies = [
  "rustfs-s3select-api",
  "rustfs-test-utils",
  "s3s",
+ "serde",
+ "serde_json",
  "tokio",
  "tracing",
 ]
@@ -10943,7 +10953,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11016,7 +11026,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -11073,7 +11083,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11104,6 +11114,7 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
+ "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
  "serde",
@@ -11127,28 +11138,45 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "s3s-rfc2047"
+version = "0.16.0-alpha.1"
+source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
+dependencies = [
+ "base64-simd",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
 dependencies = [
+ "base64-simd",
+ "hmac 0.13.0",
  "jiff",
+ "sha1 0.11.0",
+ "smallvec",
  "thiserror 2.0.20",
 ]
 
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
+source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
 dependencies = [
  "arrayvec",
  "base64-simd",
  "hex-simd",
+ "hmac 0.13.0",
  "jiff",
  "nom 8.0.0",
  "serde",
+ "sha2 0.11.0",
  "smallvec",
+ "std-next",
  "thiserror 2.0.20",
+ "zeroize",
 ]
 
 [[package]]
@@ -11758,9 +11786,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.15.2"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
 dependencies = [
  "serde",
 ]
@@ -12040,9 +12068,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "10.0.2"
+version = "11.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
+checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
 dependencies = [
  "async-trait",
  "chrono",
@@ -12233,10 +12261,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -12729,9 +12757,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
 dependencies = [
  "async-compression",
  "bitflags 2.13.1",
@@ -13355,7 +13383,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]

Cargo.toml

@@ -29,6 +29,7 @@ members = [
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
     "crates/keystone", # OpenStack Keystone integration
+    "crates/license", # License and entitlement provider contracts
     "crates/lifecycle", # Lifecycle rule evaluation contracts
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
@@ -71,7 +72,7 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.97.1"
+rust-version = "1.98.0"
 version = "1.0.0-rc.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
@@ -105,6 +106,7 @@ rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
 rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
 rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
 rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
+rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
 rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
 rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
 rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
@@ -142,7 +144,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
-mysql_async = { default-features = false, version = "0.37" }
+mysql_async = { default-features = false, version = "0.37.1" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
@@ -174,7 +176,7 @@ tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
 tower = { version = "0.5.3" }
-tower-http = { version = "0.7.0" }
+tower-http = { version = "0.7.1" }
 
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
@@ -232,7 +234,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
+# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
 astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
@@ -283,7 +285,7 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.16" }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.12.1"
+nvml-wrapper = "0.13.0"
 parking_lot = "0.12.5"
 path-absolutize = "4.0.1"
 percent-encoding = "2.3.2"
@@ -305,11 +307,11 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
-smallvec = { version = "1.15.2" }
+smallvec = { version = "1.16.0" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.3.0" }
@@ -355,7 +357,7 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "10.0.2" }
+suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"

Makefile

@@ -23,6 +23,12 @@ SHELL := $(shell which bash)
 .SHELLFLAGS = -eu -o pipefail -c
 
 DOCKER_CLI ?= docker
+# Python interpreter for the repository's helper scripts. They import tomllib
+# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go
+# through a resolver that picks a new-enough interpreter (or falls back to uv).
+# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via
+# RUSTFS_PYTHON_BIN=<command>.
+RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh
 IMAGE_NAME ?= rustfs:v1.0.0
 CONTAINER_NAME ?= rustfs-dev
 # Docker build configurations

README.md

@@ -48,16 +48,33 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 - **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
 - **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
 
-| Feature                 | Status       | Feature                  | Status           |
-| :---------------------- | :----------- | :----------------------- | :--------------- |
-| **S3 Core Features**    | ✅ Available | **Bitrot Protection**    | ✅ Available     |
-| **Upload / Download**   | ✅ Available | **Single Node Mode**     | ✅ Available     |
-| **Versioning**          | ✅ Available | **Bucket Replication**   | ✅ Available     |
-| **Logging**             | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
-| **Event Notifications** | ✅ Available | **Distributed Mode**     | 🚧 Under Testing |
-| **K8s Helm Charts**     | ✅ Available | **RustFS KMS**           | 🚧 Under Testing |
-| **Keystone Auth**       | ✅ Available | **Multi-Tenancy**        | ✅ Available     |
-| **Swift API**           | ✅ Available | **Swift Metadata Ops**   | 🚧 Partial       |
+Status legend: ✅ Available — shipped and covered by CI gates; 🧪 Preview — shipped behind an opt-in flag or with a bounded compatibility claim.
+
+| Feature                          | Status       | Feature                            | Status       |
+| :------------------------------- | :----------- | :--------------------------------- | :----------- |
+| **S3 Core Features**             | ✅ Available | **Distributed Mode**               | ✅ Available |
+| **Upload / Download**            | ✅ Available | **Single Node Mode**               | ✅ Available |
+| **Versioning**                   | ✅ Available | **Bitrot Protection**              | ✅ Available |
+| **Object Lock (WORM)**           | ✅ Available | **Healing & Scanner**              | ✅ Available |
+| **Server-Side Encryption**       | ✅ Available | **Pool Expansion / Decommission**  | ✅ Available |
+| **RustFS KMS**                   | ✅ Available | **Bucket Replication**             | ✅ Available |
+| **Lifecycle Management (ILM)**   | ✅ Available | **Site Replication**               | ✅ Available |
+| **ILM Tiering (Remote S3)**      | ✅ Available | **Bucket Quota**                   | ✅ Available |
+| **S3 Select**                    | ✅ Available | **Event Notifications**            | ✅ Available |
+| **S3 Tables (Iceberg REST)**     | 🧪 Preview   | **Audit Logging**                  | ✅ Available |
+| **IAM / Policies**               | ✅ Available | **Logging & Observability**        | ✅ Available |
+| **OIDC / SSO**                   | ✅ Available | **Web Console**                    | ✅ Available |
+| **Keystone Auth**                | ✅ Available | **K8s Helm Charts**                | ✅ Available |
+| **Swift API**                    | ✅ Available | **FTPS / WebDAV**                  | ✅ Available |
+| **Multi-Tenancy**                | ✅ Available | **SFTP**                           | ✅ Available |
+| **MinIO On-Disk Compatibility**  | 🧪 Preview   |                                    |              |
+
+Notes:
+
+- **RustFS KMS**: Vault (KV2 / Transit) and AWS KMS backends are supported for production. The `Local` and `Static` backends are for development and testing only. See [KMS backend security properties](docs/operations/kms-backend-security.md).
+- **Swift API / SFTP**: opt-in cargo features (`--features swift`, `--features sftp`, or `full`). FTPS and WebDAV are enabled in the default build.
+- **S3 Tables**: ships as an Iceberg REST Catalog with automated PyIceberg and DuckDB coverage; other engines and vendor profiles carry bounded claims listed in the [S3 Tables support matrix](docs/architecture/s3-tables-support-matrix.md).
+- **MinIO On-Disk Compatibility**: gated behind the `rio-v2` feature and not part of the default build. Objects MinIO encrypted are not readable by RustFS. See [MinIO file-format interoperability](docs/architecture/minio-file-format-compat.md).
 
 ## RustFS vs MinIO Performance
 

crates/e2e_test/README.md

@@ -169,7 +169,7 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
@@ -233,8 +233,8 @@ spawn error. Install the pinned CI version before running their profiles.
   [`src/policy/README.md`](src/policy/README.md),
   [`src/protocols/README.md`](src/protocols/README.md),
   [`src/reliant/README.md`](src/reliant/README.md)
-- Authoritative per-module counts:
-  [`docs/testing/e2e-suite-inventory.md`](../../docs/testing/e2e-suite-inventory.md)
+- Per-module counts: `cargo nextest list -p e2e_test --profile <profile>`
+  (one-liner in [`docs/testing/README.md`](../../docs/testing/README.md))
 - Test pyramid & flake policy: [`docs/testing/README.md`](../../docs/testing/README.md)
 
 ## CI smoke subset (`--profile e2e-smoke`)
@@ -271,12 +271,12 @@ Note on `#[serial]`: nextest runs each test in its own process, so
 parallel-safe by construction (random port + isolated temp dir), which the
 current subset is.
 
-### Authoritative test inventory
+### Test inventory
 
-`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
-listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
-moving e2e tests so acceptance numbers in the test-strategy issues
-(backlog#1147–#1155) stay auditable. When a profile membership change is
+Per-module counts are not committed; list them with
+`cargo nextest list -p e2e_test --profile <profile>` (the result is
+platform-dependent because some modules are linux-only; the `jq` one-liner is
+in `docs/testing/README.md`). When a profile membership change is
 intentional, review its JSON listing before updating the matching
 `.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
 produced the listing:

crates/e2e_test/src/checksum_upload_test.rs

@@ -22,7 +22,7 @@ mod tests {
     use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
+    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption};
     use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
     use md5::{Digest as Md5Digest, Md5};
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
@@ -260,6 +260,117 @@ mod tests {
         info!("PASSED: HeadObject returns stored SHA256 digest");
     }
 
+    #[tokio::test]
+    async fn test_head_object_returns_sse_s3_checksum() {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server_with_env(
+            vec![],
+            &[
+                ("RUSTFS_SSE_S3_MASTER_KEY", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="),
+                ("RUSTFS_CONSOLE_ENABLE", "false"),
+            ],
+        )
+        .await
+        .expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-sse-s3-checksum-head";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let put = client
+            .put_object()
+            .bucket(bucket)
+            .key("encrypted.txt")
+            .body(ByteStream::from_static(b"encrypted checksum"))
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 PutObject with CRC32 failed");
+        let expected = put.checksum_crc32().expect("PutObject must return CRC32");
+
+        let head = client
+            .head_object()
+            .bucket(bucket)
+            .key("encrypted.txt")
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 HeadObject failed");
+
+        assert_eq!(head.checksum_crc32(), Some(expected));
+
+        client
+            .copy_object()
+            .bucket(bucket)
+            .key("encrypted-copy.txt")
+            .copy_source(format!("{bucket}/encrypted.txt"))
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .send()
+            .await
+            .expect("SSE-S3 CopyObject failed");
+        let copy_head = client
+            .head_object()
+            .bucket(bucket)
+            .key("encrypted-copy.txt")
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 copied HeadObject failed");
+
+        assert_eq!(copy_head.checksum_crc32(), Some(expected));
+
+        let multipart_key = "encrypted-multipart.txt";
+        let create = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .server_side_encryption(ServerSideEncryption::Aes256)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 CreateMultipartUpload with CRC32 failed");
+        let upload_id = create.upload_id().expect("CreateMultipartUpload must return an upload ID");
+        let part = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from_static(b"encrypted multipart checksum"))
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("SSE-S3 UploadPart with CRC32 failed");
+        let completed_part = CompletedPart::builder()
+            .part_number(1)
+            .e_tag(part.e_tag().expect("UploadPart must return an ETag"))
+            .checksum_crc32(part.checksum_crc32().expect("UploadPart must return CRC32"))
+            .build();
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(upload_id)
+            .multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
+            .send()
+            .await
+            .expect("SSE-S3 CompleteMultipartUpload with CRC32 failed");
+        let expected_multipart = complete.checksum_crc32().expect("CompleteMultipartUpload must return CRC32");
+        let multipart_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("SSE-S3 multipart HeadObject failed");
+
+        assert_eq!(multipart_head.checksum_crc32(), Some(expected_multipart));
+    }
+
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]

crates/e2e_test/src/common.rs

@@ -1469,31 +1469,18 @@ impl RustFSTestClusterEnvironment {
     ///   times out, or cluster service readiness times out.
     pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let binary_path = rustfs_binary_path();
+        self.start_with_binary(&binary_path).await
+    }
+
+    /// Start every cluster node with a specific RustFS binary.
+    ///
+    /// Upgrade compatibility tests use this to initialize a cluster with a
+    /// pinned previous release before replacing nodes with the workspace build.
+    pub async fn start_with_binary(&mut self, binary_path: &Path) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let volumes_arg = self.build_volumes_arg();
 
-        for (i, node) in self.nodes.iter_mut().enumerate() {
-            info!("Starting cluster node {} on {}", i, node.address);
-
-            let mut command = Command::new(&binary_path);
-            command
-                .env("RUSTFS_VOLUMES", &volumes_arg)
-                .env("RUSTFS_ADDRESS", &node.address)
-                .env("RUSTFS_ACCESS_KEY", &self.access_key)
-                .env("RUSTFS_SECRET_KEY", &self.secret_key)
-                .env("RUSTFS_CONSOLE_ENABLE", "false")
-                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
-
-            for (key, value) in &self.extra_env {
-                command.env(key, value);
-            }
-            for (key, value) in &self.node_extra_env[i] {
-                command.env(key, value);
-            }
-            capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
-
-            let process = command.current_dir(&node.data_dir).spawn()?;
-
-            node.process = Some(process);
+        for node_idx in 0..self.nodes.len() {
+            self.spawn_node(node_idx, binary_path, &volumes_arg)?;
         }
 
         for (i, node) in self.nodes.iter().enumerate() {
@@ -1509,20 +1496,46 @@ impl RustFSTestClusterEnvironment {
 
     /// Start one node process using the cluster's existing volume layout.
     pub async fn start_node(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        self.start_node_from_binary(node_idx, &binary_path).await
+    }
+
+    /// Start one stopped cluster node with a specific RustFS binary while
+    /// preserving the cluster's volume layout and that node's data directory.
+    pub async fn start_node_from_binary(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let volumes_arg = self.build_volumes_arg();
+        self.spawn_node(node_idx, binary_path, &volumes_arg)?;
+
+        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        self.wait_for_node_service_ready(node_idx).await?;
+        Ok(())
+    }
+
+    fn spawn_node(
+        &mut self,
+        node_idx: usize,
+        binary_path: &Path,
+        volumes_arg: &str,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         self.ensure_node_index(node_idx)?;
         if self.nodes[node_idx].process.is_some() {
             return Err(format!("cluster node {node_idx} is already running").into());
         }
+        if !binary_path.is_file() {
+            return Err(format!("RustFS binary does not exist: {}", binary_path.display()).into());
+        }
 
-        let binary_path = rustfs_binary_path();
-        let volumes_arg = self.build_volumes_arg();
         let log_path = self.node_capture_log_paths[node_idx].clone();
         let node = &mut self.nodes[node_idx];
-        info!("Starting cluster node {} on {}", node_idx, node.address);
+        info!("Starting cluster node {} on {} with {}", node_idx, node.address, binary_path.display());
 
-        let mut command = Command::new(&binary_path);
+        let mut command = Command::new(binary_path);
         command
-            .env("RUSTFS_VOLUMES", &volumes_arg)
+            .env("RUSTFS_VOLUMES", volumes_arg)
             .env("RUSTFS_ADDRESS", &node.address)
             .env("RUSTFS_ACCESS_KEY", &self.access_key)
             .env("RUSTFS_SECRET_KEY", &self.secret_key)
@@ -1539,9 +1552,6 @@ impl RustFSTestClusterEnvironment {
 
         let process = command.current_dir(&node.data_dir).spawn()?;
         node.process = Some(process);
-
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
-        self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
 
@@ -1689,6 +1699,51 @@ impl RustFSTestClusterEnvironment {
         process.wait()?;
         Ok(())
     }
+
+    /// Gracefully stop one cluster node and wait for its process to exit.
+    ///
+    /// This is intentionally separate from [`Self::stop_node`]: the latter is
+    /// a hard kill used by crash-recovery tests, while this path lets RustFS
+    /// complete its normal shutdown hooks before a test restarts the node.
+    pub async fn stop_node_gracefully(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.ensure_node_index(node_idx)?;
+
+        #[cfg(unix)]
+        {
+            let Some(process) = self.nodes[node_idx].process.as_ref() else {
+                return Ok(());
+            };
+            let pid = process.id().to_string();
+            let signal_status = Command::new("kill").args(["-TERM", &pid]).status()?;
+            if !signal_status.success() {
+                return Err(format!("failed to send SIGTERM to cluster node {node_idx} (pid {pid})").into());
+            }
+
+            let mut process = self.nodes[node_idx]
+                .process
+                .take()
+                .ok_or_else(|| format!("cluster node {node_idx} process disappeared while stopping"))?;
+            let deadline = std::time::Instant::now() + Duration::from_secs(45);
+            loop {
+                if let Some(status) = process.try_wait()? {
+                    info!("Cluster node {} stopped gracefully with {}", node_idx, status);
+                    return Ok(());
+                }
+                if std::time::Instant::now() >= deadline {
+                    let _ = process.kill();
+                    let _ = process.wait();
+                    return Err(format!("cluster node {node_idx} did not stop gracefully within 45 seconds").into());
+                }
+                sleep(Duration::from_millis(100)).await;
+            }
+        }
+
+        #[cfg(not(unix))]
+        {
+            let _ = node_idx;
+            Err("graceful cluster-node stop is only supported on Unix E2E hosts".into())
+        }
+    }
 }
 
 impl Drop for RustFSTestClusterEnvironment {

crates/e2e_test/src/degraded_listing_availability_test.rs

@@ -0,0 +1,174 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression: an object legally committed at degraded write quorum must stay
+//! listable while a *different* drive is offline.
+//!
+//! On a 4-drive EC 2+2 set, a PUT made while one drive is down persists
+//! `xl.meta` on 3 of 4 drives (write quorum). If a different drive later goes
+//! offline before heal converges, a strict latest-listing quorum of 3 can only
+//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
+//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
+//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
+//! product fix relaxes the listing's required object quorum by the number of
+//! set drives the listing could not consult (see
+//! `latest_listing_required_object_quorum` in
+//! `crates/ecstore/src/store/list_objects.rs`).
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestClusterEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use bytes::Bytes;
+    use std::collections::HashSet;
+    use std::error::Error;
+    use std::time::{Duration, Instant};
+    use tracing::info;
+
+    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+    const BUCKET: &str = "degraded-listing-availability";
+    const OBJECT_COUNT: usize = 8;
+    /// Well under the observed heal-convergence gap (~50s in the CI incident),
+    /// so a listing that only completes after heal restores the missing copy
+    /// still fails this deadline on a regressed build.
+    const LISTING_DEADLINE: Duration = Duration::from_secs(25);
+    const GET_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+    const PUT_RETRY_DEADLINE: Duration = Duration::from_secs(15);
+
+    fn object_key(idx: usize) -> String {
+        format!("degraded-object-{idx:02}")
+    }
+
+    async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
+        let mut keys = HashSet::new();
+        let mut continuation_token: Option<String> = None;
+        loop {
+            let response = client
+                .list_objects_v2()
+                .bucket(BUCKET)
+                .set_continuation_token(continuation_token.clone())
+                .send()
+                .await?;
+            keys.extend(
+                response
+                    .contents()
+                    .iter()
+                    .filter_map(|object| object.key().map(str::to_owned)),
+            );
+            match response.next_continuation_token() {
+                Some(token) => continuation_token = Some(token.to_owned()),
+                None => break,
+            }
+        }
+        Ok(keys)
+    }
+
+    /// 4-node single-drive cluster (EC 2+2, write quorum 3):
+    /// 1. Stop node 1 and PUT objects — each commits on nodes {0, 2, 3} only.
+    /// 2. Stop node 3 (a holder drive), then bring node 1 back before heal can
+    ///    recreate the missing copies there.
+    /// 3. Every object still satisfies read quorum (nodes 0 and 2), so GET
+    ///    must succeed AND ListObjectsV2 must report every key well before
+    ///    heal converges.
+    #[tokio::test]
+    async fn degraded_write_remains_listable_while_a_different_drive_is_offline() -> TestResult {
+        init_logging();
+
+        let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+        // Listing availability must not depend on heal convergence: disable
+        // the background healers so the degraded objects keep their metadata
+        // on exactly 3 of 4 drives for the whole test.
+        cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
+        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
+        cluster.start().await?;
+        cluster.create_test_bucket(BUCKET).await?;
+        let client = cluster.create_s3_client(0)?;
+
+        info!("stopping node 1 so the uploads commit at degraded write quorum (3 of 4)");
+        cluster.stop_node(1)?;
+        // The first writes after a node drops can see transient 503s while the
+        // survivors notice the dead peer; retry briefly (overwrites of the same
+        // unversioned key are idempotent).
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let body = format!("degraded listing payload {idx}");
+            let deadline = Instant::now() + PUT_RETRY_DEADLINE;
+            loop {
+                let request = client
+                    .put_object()
+                    .bucket(BUCKET)
+                    .key(&key)
+                    .body(Bytes::from(body.clone()).into());
+                match request.send().await {
+                    Ok(_) => break,
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded PUT for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded PUT for {key} failed: {error}").into()),
+                }
+            }
+        }
+
+        info!("stopping node 3 (holds a copy) and restoring node 1 (holds none)");
+        cluster.stop_node(3)?;
+        cluster.start_node(1).await?;
+
+        // The first requests after a node drops can see transient 503s while
+        // the survivors notice the dead peer; retry briefly before asserting.
+        for idx in 0..OBJECT_COUNT {
+            let key = object_key(idx);
+            let deadline = Instant::now() + GET_RETRY_DEADLINE;
+            let body = loop {
+                match client.get_object().bucket(BUCKET).key(&key).send().await {
+                    Ok(response) => break response.body.collect().await?.into_bytes(),
+                    Err(error) if Instant::now() < deadline => {
+                        info!("retrying degraded GET for {key}: {error}");
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    }
+                    Err(error) => return Err(format!("degraded object {key} failed read quorum GET: {error}").into()),
+                }
+            };
+            assert!(!body.is_empty(), "degraded object {key} should read back at read quorum");
+        }
+
+        let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
+        let deadline = Instant::now() + LISTING_DEADLINE;
+        let listed = loop {
+            let listed = match list_all_keys(&client).await {
+                Ok(keys) => keys,
+                Err(error) if Instant::now() < deadline => {
+                    info!("retrying degraded listing: {error}");
+                    tokio::time::sleep(Duration::from_millis(500)).await;
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            if expected.is_subset(&listed) {
+                break listed;
+            }
+            assert!(
+                Instant::now() < deadline,
+                "objects readable at read quorum stayed missing from ListObjectsV2 for {LISTING_DEADLINE:?}: \
+                 missing={:?} listed={listed:?}",
+                expected.difference(&listed).collect::<Vec<_>>(),
+            );
+            tokio::time::sleep(Duration::from_millis(500)).await;
+        };
+        info!(listed = listed.len(), "degraded objects are listable while node 3 is offline");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/fake_s3_target/README.md

@@ -1,11 +1,15 @@
 # Programmable fake S3 target
 
-This module is the shared failure-injection boundary for replication end-to-end tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
+This module is the shared failure-injection boundary for replication end-to-end tests and the programmable external source for on-demand-migration (ODM) tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
 
 `FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
 
-Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
+Supported data operations are HeadBucket, GetBucketVersioning, ListObjectsV2, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets created with `create_bucket` are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
 
-Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
+`create_bucket_with_mode(name, BucketMode::Unversioned)` models a plain migration source: PUT overwrites in place, DELETE removes the key without a delete marker, GetBucketVersioning reports no status, and no `x-amz-version-id` is returned by PUT, GET, HEAD, tagging, or multipart completion. The only `versionId` such a bucket accepts is `null`; any other value is rejected with `InvalidArgument`. The mode is fixed at creation.
 
-The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
+ListObjectsV2 lists current versions only (a key whose newest version is a delete marker is hidden) in byte order and supports `prefix`, `delimiter`, `max-keys` (clamped to 1000), `start-after`, and `continuation-token`; common prefixes count toward `max-keys`, `IsTruncated` / `NextContinuationToken` / `KeyCount` follow S3, and continuation tokens are opaque. `encoding-type` and `fetch-owner` are accepted but ignored, and ListObjects (v1) is not implemented. GET and HEAD honor `Range` in the `bytes=first-last`, `bytes=first-`, and `bytes=-suffix` forms with a 206 status, exact `Content-Range`, and `Accept-Ranges: bytes`; unsatisfiable ranges answer 416 `InvalidRange` with `Content-Range: bytes */<length>`. PUT and CreateMultipartUpload accept `Content-Type`, `Content-Encoding`, `Content-Disposition`, `Content-Language`, `Cache-Control`, `Expires`, and `x-amz-meta-*` (names stored lowercased), and HEAD/GET replay them verbatim together with `Last-Modified` and the ETag (hex MD5 for single PUTs, `<md5-of-part-md5s>-<parts>` for multipart objects). `put_seed_object` stores an object directly, bypassing the wire, the fault script, and the journal, so a source can be seeded without polluting the assertions a scenario later makes.
+
+Fault actions cover HTTP 401/403/503 responses (`Status`), any 4xx/5xx status paired with the matching S3 error code (`ResponseStatus`), pre-dispatch delay, holding a fully computed successful response before its first byte (`Stall`), connection abort when a logical request-body threshold is reached, GetObject bodies cut off after N bytes while `Content-Length` announces the full size (`TruncateBodyAt`), streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions and `count_requests(operation, key)` counts entries for one exact key. Each record journals the `Range` and `User-Agent` request headers, the ListObjectsV2 `prefix` and `continuation-token` query values, and a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
+
+The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type and each standard object header at 1 KiB. By default a PUT or uploaded part is capped at 64 MiB and a completed multipart object and all stored object/part data are capped at 128 MiB; `FakeS3Target::start_with_options(FakeS3TargetOptions { max_object_bytes })` raises the object cap up to 256 MiB, and the total budget then becomes twice the object cap (never below 128 MiB). Body drain, body-permit waits, delay, stall, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -17,15 +17,115 @@
 #[cfg(test)]
 mod tests {
     use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
-    use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging};
+    use crate::common::{
+        FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
+    };
+    use crate::storage_api::RUSTFS_META_BUCKET;
     use aws_sdk_s3::primitives::ByteStream;
     use http::Method;
     use std::collections::HashSet;
     use std::error::Error;
+    use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
+    use std::process::Command;
+    use tokio::net::TcpStream;
     use tokio::time::{Duration, Instant, sleep, timeout};
     use tracing::info;
 
+    const POOL_METADATA_OBJECT: &str = "pool.bin";
+
+    struct TcpPortBlackhole {
+        port: u16,
+        comment: String,
+        use_sudo: bool,
+        active: bool,
+    }
+
+    impl TcpPortBlackhole {
+        fn install(address: &str) -> Result<Self, Box<dyn Error + Send + Sync>> {
+            let address = address.parse::<SocketAddr>()?;
+            if !address.ip().is_loopback() {
+                return Err(format!("refusing to install a test firewall rule for non-loopback address {address}").into());
+            }
+
+            let id = Command::new("id").arg("-u").output()?;
+            if !id.status.success() {
+                return Err(format!("failed to determine the test process uid: {}", String::from_utf8_lossy(&id.stderr)).into());
+            }
+            let use_sudo = String::from_utf8_lossy(&id.stdout).trim() != "0";
+            let mut blackhole = Self {
+                port: address.port(),
+                comment: format!("rustfs-e2e-{}", uuid::Uuid::new_v4()),
+                use_sudo,
+                active: false,
+            };
+            blackhole.run_iptables(true)?;
+            blackhole.active = true;
+            Ok(blackhole)
+        }
+
+        fn restore(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            if !self.active {
+                return Ok(());
+            }
+            self.run_iptables(false)?;
+            self.active = false;
+            Ok(())
+        }
+
+        fn run_iptables(&self, insert: bool) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let mut command = if self.use_sudo {
+                let mut command = Command::new("sudo");
+                command.args(["-n", "iptables"]);
+                command
+            } else {
+                Command::new("iptables")
+            };
+            command.args(["-w", "5"]);
+            if insert {
+                command.args(["-I", "OUTPUT", "1"]);
+            } else {
+                command.args(["-D", "OUTPUT"]);
+            }
+            let port = self.port.to_string();
+            let output = command
+                .args([
+                    "-p",
+                    "tcp",
+                    "-d",
+                    "127.0.0.1/32",
+                    "--dport",
+                    &port,
+                    "-m",
+                    "comment",
+                    "--comment",
+                    &self.comment,
+                    "-j",
+                    "DROP",
+                ])
+                .output()?;
+            if !output.status.success() {
+                let action = if insert { "install" } else { "remove" };
+                return Err(format!(
+                    "failed to {action} endpoint blackhole rule for port {}: stdout={}, stderr={}",
+                    self.port,
+                    String::from_utf8_lossy(&output.stdout),
+                    String::from_utf8_lossy(&output.stderr)
+                )
+                .into());
+            }
+            Ok(())
+        }
+    }
+
+    impl Drop for TcpPortBlackhole {
+        fn drop(&mut self) {
+            if let Err(error) = self.restore() {
+                eprintln!("failed to remove {} firewall rule during test cleanup: {error}", self.comment);
+            }
+        }
+    }
+
     fn has_file_under(path: &Path) -> bool {
         let Ok(entries) = std::fs::read_dir(path) else {
             return false;
@@ -91,6 +191,62 @@ mod tests {
             .count()
     }
 
+    async fn assert_all_nodes_list_exact_keys(
+        clients: &[aws_sdk_s3::Client],
+        bucket: &str,
+        expected_keys: &HashSet<String>,
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        const PAGE_SIZE: i32 = 10;
+        for (node_index, client) in clients.iter().enumerate() {
+            let mut listed_keys = Vec::new();
+            let mut continuation_token = None;
+            let max_pages = expected_keys.len().div_ceil(PAGE_SIZE as usize) + 1;
+            let mut page_count = 0;
+            loop {
+                page_count += 1;
+                if page_count > max_pages {
+                    return Err(format!("node {node_index} listing exceeded the bounded {max_pages}-page budget").into());
+                }
+                let response = timeout(
+                    Duration::from_secs(15),
+                    client
+                        .list_objects_v2()
+                        .bucket(bucket)
+                        .max_keys(PAGE_SIZE)
+                        .set_continuation_token(continuation_token.clone())
+                        .send(),
+                )
+                .await??;
+                listed_keys.extend(
+                    response
+                        .contents()
+                        .iter()
+                        .filter_map(|object| object.key().map(str::to_owned)),
+                );
+                if !response.is_truncated().unwrap_or(false) {
+                    break;
+                }
+                let next_token = response
+                    .next_continuation_token()
+                    .filter(|token| Some(*token) != continuation_token.as_deref())
+                    .ok_or_else(|| format!("node {node_index} returned a truncated listing without a new continuation token"))?;
+                continuation_token = Some(next_token.to_owned());
+            }
+
+            let listed_key_set = listed_keys.iter().cloned().collect::<HashSet<_>>();
+            assert_eq!(
+                listed_keys.len(),
+                listed_key_set.len(),
+                "node {node_index} returned duplicate keys after recovery: {listed_keys:?}"
+            );
+            assert_eq!(
+                &listed_key_set, expected_keys,
+                "node {node_index} did not expose the complete recovered namespace"
+            );
+        }
+        Ok(())
+    }
+
     fn heal_task_status_diagnostic(body: &str) -> String {
         let Ok(status) = serde_json::from_str::<serde_json::Value>(body) else {
             return body.to_string();
@@ -125,11 +281,12 @@ mod tests {
             && operations["retryingTasks"].as_u64() == Some(0)
     }
 
+    // Queued low-priority repairs cannot execute while the single admin slot is
+    // occupied; ownership is determined by active and retrying tasks only.
     fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
         let operations = &status["healOperations"];
         status["clusterStatusComplete"] == serde_json::Value::Bool(true)
             && status["state"].as_str() == Some("active")
-            && operations["queueLength"].as_u64() == Some(0)
             && operations["activeTasks"].as_u64() == Some(1)
             && operations["retryingTasks"].as_u64() == Some(0)
             && operations["activeBySource"]["admin"].as_u64() == Some(1)
@@ -547,41 +704,144 @@ mod tests {
         .into())
     }
 
+    async fn wait_for_scanner_cycle_after(
+        cluster: &RustFSTestClusterEnvironment,
+        previous_cycle_end: u64,
+    ) -> Result<u64, Box<dyn Error + Send + Sync>> {
+        let deadline = Instant::now() + Duration::from_secs(60);
+        loop {
+            let mut latest_cycle_end = 0;
+            let mut versions_observed = false;
+            let mut observations = Vec::with_capacity(cluster.nodes.len());
+            for (node_index, node) in cluster.nodes.iter().enumerate() {
+                let (status, body) = timeout(
+                    Duration::from_secs(5),
+                    admin_request(
+                        &node.url,
+                        Method::GET,
+                        "/rustfs/admin/v3/scanner/status",
+                        None,
+                        &cluster.access_key,
+                        &cluster.secret_key,
+                    ),
+                )
+                .await??;
+                assert_eq!(status, 200, "scanner status must be available: {body}");
+                let status: serde_json::Value = serde_json::from_str(&body)?;
+                assert_eq!(status["enabled"].as_bool(), Some(true), "scanner must stay enabled: {status}");
+                let metrics = &status["metrics"];
+                let cycle_end = metrics["last_cycle_end_unix_secs"]
+                    .as_u64()
+                    .ok_or("scanner status is missing its completed-cycle timestamp")?;
+                let versions_scanned = metrics["versions_scanned"]
+                    .as_u64()
+                    .ok_or("scanner status is missing its version-coverage counter")?;
+                latest_cycle_end = latest_cycle_end.max(cycle_end);
+                versions_observed |= versions_scanned > 0;
+                observations.push(format!(
+                    "node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}",
+                    metrics["current_cycle"],
+                    metrics["current_cycle_active"],
+                    metrics["leader_lock_state"],
+                    metrics["last_cycle_result"],
+                ));
+            }
+            // The coordinator records cycle completion, but remote workers
+            // record scanned versions. Both witnesses need not share a node.
+            if latest_cycle_end > previous_cycle_end && versions_observed {
+                return Ok(latest_cycle_end);
+            }
+            if Instant::now() >= deadline {
+                return Err(format!(
+                    "enabled scanner did not complete an object-scanning cycle after {previous_cycle_end}: {observations:?}"
+                )
+                .into());
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+    }
+
     // Keep the original unformatted-disk scenario above. This case retains the
     // format identity so only the explicit admin task can rebuild missing data.
     #[tokio::test(flavor = "multi_thread")]
     async fn test_cluster_root_heal_resumes_missing_remote_shards_after_node_restart() -> Result<(), Box<dyn Error + Send + Sync>>
     {
+        run_cluster_root_heal_interruption(InterruptionScenario::IsolatedTargetRestart).await
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        timeout(
+            Duration::from_secs(420),
+            run_cluster_root_heal_interruption(InterruptionScenario::BackgroundCoordinatorRestart),
+        )
+        .await?
+    }
+
+    #[cfg(target_os = "linux")]
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_cluster_root_heal_recovers_after_target_endpoint_blackhole() -> Result<(), Box<dyn Error + Send + Sync>> {
+        timeout(
+            Duration::from_secs(420),
+            run_cluster_root_heal_interruption(InterruptionScenario::TargetEndpointBlackhole),
+        )
+        .await?
+    }
+
+    #[derive(Clone, Copy, PartialEq, Eq)]
+    enum InterruptionScenario {
+        IsolatedTargetRestart,
+        BackgroundCoordinatorRestart,
+        TargetEndpointBlackhole,
+    }
+
+    async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let (background_enabled, interruption_node, interruption_kind) = match scenario {
+            InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
+            InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
+            InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
+        };
         init_logging();
         info!(
-            event = "heal_restart_started",
+            event = "heal_interruption_started",
             component = "e2e_test",
             subsystem = "heal",
-            "Starting root-heal restart test"
+            background_enabled,
+            interruption_node,
+            interruption_kind,
+            "Starting root-heal interruption test"
         );
 
         let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
         cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
-        cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false");
-        cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", "false");
-        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
-        cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
-        cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
-        cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
-        cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
-        // Keep all storage nodes' Heal runtimes enabled so their disk services
-        // complete normal registration after restart. Scanner, auto-heal and
-        // MRF are disabled; the pre-root idle barrier below drains the direct
-        // outage-object repair before the explicit admin task starts.
+        // Heal control uses the first lexicographically sorted grid host.
+        // Keep that coordinator distinct from the remote target at index 1.
+        cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
+        cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", background_enabled.to_string());
+        cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", background_enabled.to_string());
+        cluster.set_env("RUSTFS_SCANNER_ENABLED", background_enabled.to_string());
+        if background_enabled {
+            // Only the scanner cadence is accelerated. Keep normal Heal
+            // concurrency and every automatic recovery owner enabled.
+            for &(key, value) in FAST_DATA_USAGE_SCANNER_ENV {
+                cluster.set_env(key, value);
+            }
+        } else {
+            cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
+            cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
+            cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
+            cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
+        }
+        // Keep every node's Heal runtime enabled for normal disk registration.
         let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
             .unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
         cluster.set_env("RUST_LOG", server_rust_log);
-        if let Ok(log_dir) = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
-            std::fs::create_dir_all(&log_dir)?;
-            for node_index in 0..cluster.nodes.len() {
-                cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
-            }
+        let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
+        std::fs::create_dir_all(&log_dir)?;
+        for node_index in 0..cluster.nodes.len() {
+            cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
         }
         cluster.start().await?;
         let clients = cluster.create_all_clients()?;
@@ -634,6 +894,18 @@ mod tests {
             });
         }
 
+        let expected_pool_metadata = if background_enabled {
+            let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
+            assert!(
+                census.is_complete(),
+                "target must hold complete pool metadata before the fault: {census:?}"
+            );
+            wait_for_scanner_cycle_after(&cluster, 0).await?;
+            Some(census)
+        } else {
+            None
+        };
+
         cluster.stop_node(1)?;
         std::fs::remove_dir_all(&replaced_disk)?;
         std::fs::create_dir_all(
@@ -691,25 +963,25 @@ mod tests {
             .find(|index| !outage_peer_erasure_indices.contains(index))
             .ok_or("online outage-object shards leave no erasure index for the replacement target")?;
 
-        // The PUT path may have admitted a direct Internal object repair while
-        // node 1 was offline. Cancel the isolated bucket path before the target
-        // returns; otherwise it could rebuild the outage object and invalidate
-        // the explicit-root ownership assertion below.
-        let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
-        let (cancel_status, cancel_body) = admin_request(
-            &cluster.nodes[0].url,
-            Method::POST,
-            &cancel_outage_heal_path,
-            Some(
-                r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#
-                    .to_string(),
-            ),
-            &cluster.access_key,
-            &cluster.secret_key,
-        )
-        .await?;
-        if !cancel_status.is_success() {
-            return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
+        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+        if !background_enabled {
+            // The PUT path may have admitted a direct Internal object repair while
+            // node 1 was offline. Cancel the isolated bucket path before the target
+            // returns; otherwise it could rebuild the outage object and invalidate
+            // the explicit-root ownership assertion below.
+            let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
+            let (cancel_status, cancel_body) = admin_request(
+                &cluster.nodes[0].url,
+                Method::POST,
+                &cancel_outage_heal_path,
+                Some(heal_body.to_string()),
+                &cluster.access_key,
+                &cluster.secret_key,
+            )
+            .await?;
+            if !cancel_status.is_success() {
+                return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
+            }
         }
 
         cluster.start_node(1).await?;
@@ -724,7 +996,12 @@ mod tests {
             );
             let recovered: serde_json::Value = serde_json::from_str(&status_body)
                 .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
-            if cluster_heal_is_idle(&recovered) {
+            let ready = if background_enabled {
+                recovered["clusterStatusComplete"] == serde_json::Value::Bool(true)
+            } else {
+                cluster_heal_is_idle(&recovered)
+            };
+            if ready {
                 break;
             }
             if Instant::now() >= recovery_deadline {
@@ -732,23 +1009,24 @@ mod tests {
             }
             sleep(Duration::from_millis(250)).await;
         }
-        assert_eq!(
-            matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
-            0,
-            "non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
-        );
-        assert!(
-            !census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
-            "the object written during the outage must be absent before the explicit root heal"
-        );
         let pre_heal_replacement = replacement_recovery_status(&cluster).await?;
-        assert_eq!(
-            pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
-            Some(0),
-            "isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
-        );
+        if !background_enabled {
+            assert_eq!(
+                matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
+                0,
+                "non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
+            );
+            assert!(
+                !census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
+                "the object written during the outage must be absent before the explicit root heal"
+            );
+            assert_eq!(
+                pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
+                Some(0),
+                "isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
+            );
+        }
 
-        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
         let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
         let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
         let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
@@ -764,24 +1042,39 @@ mod tests {
             .and_then(|value| value.parse::<u64>().ok())
             .unwrap_or(60);
         let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
-        let pre_interrupt_status = loop {
+        loop {
             let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
             let active_status: serde_json::Value = serde_json::from_str(&status_body)
                 .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
-            if only_admin_heal_is_active(&active_status) {
-                break active_status;
+            let active = if background_enabled {
+                active_status["state"].as_str() == Some("active")
+                    && active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
+            } else {
+                only_admin_heal_is_active(&active_status)
+            };
+            if active {
+                break;
             }
             if Instant::now() >= partial_deadline {
                 return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
             }
             sleep(Duration::from_millis(50)).await;
-        };
-        let partial_count = loop {
-            let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
-            if matching > 0 && matching < expected_manifests.len() {
-                break matching;
+        }
+        let (partial_count, partial_manifest) = loop {
+            // Hash one committed shard to prove progress without letting a
+            // full-corpus hash pass consume the interruption window.
+            let materialized = metadata_count(&replaced_disk, bucket, &expected_manifests);
+            if materialized > 0
+                && materialized < expected_manifests.len()
+                && let Some(expected) = expected_manifests
+                    .iter()
+                    .find(|expected| object_metadata_exists_on_disk(&replaced_disk, bucket, &expected.key))
+                && census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?
+                    .matches_manifest(&expected.shard_census)
+            {
+                break (materialized, expected);
             }
-            if matching == expected_manifests.len() {
+            if materialized == expected_manifests.len() {
                 return Err(format!(
                     "root heal rebuilt all {} baseline objects before the target could be interrupted",
                     expected_manifests.len()
@@ -796,30 +1089,195 @@ mod tests {
             }
             sleep(Duration::from_millis(10)).await;
         };
+
+        let pre_interrupt_status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+        let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
+            .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
+        let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
+        let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
+        assert!(
+            coordinator_log
+                .lines()
+                .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+                .any(|event| {
+                    event["event"] == "heal_task_state"
+                        && event["task_id"] == client_token
+                        && event["heal_type"] == "cluster"
+                        && event["state"] == "started"
+                }),
+            "node 0 must have started the exact admin task before interruption"
+        );
+        let pre_interrupt_operations = &pre_interrupt_status["healOperations"];
+        assert_eq!(
+            pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
+            Some(1),
+            "interruption must occur while the single admin task is active: {pre_interrupt_status}"
+        );
+        if !background_enabled {
+            assert!(
+                only_admin_heal_is_active(&pre_interrupt_status),
+                "isolated interruption must retain only the admin task: {pre_interrupt_status}"
+            );
+            assert_eq!(
+                pre_interrupt_replacement["cluster"]["records"].as_array().map(Vec::len),
+                Some(0),
+                "root-heal interruption point must not retain an automatic replacement generation: {pre_interrupt_replacement}"
+            );
+        }
         info!(
-            event = "heal_restart_checkpoint",
+            event = "heal_interruption_checkpoint",
             component = "e2e_test",
             subsystem = "heal",
-            partial_count,
-            "Verified unique admin owner before target interruption"
+            background_enabled,
+            interruption_node,
+            interruption_kind,
+            partial_metadata_count = partial_count,
+            verified_key = partial_manifest.key,
+            "Observed partial rebuild before interruption"
         );
 
-        cluster.stop_node(1)?;
-        let stopped_count = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
-        assert!(
-            stopped_count > 0 && stopped_count < expected_manifests.len(),
-            "the target must stop after a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
-            expected_manifests.len()
-        );
-        let unclean_shutdown_marker = replaced_disk.join(".rustfs.sys").join("unclean-shutdown");
-        match std::fs::remove_file(&unclean_shutdown_marker) {
-            Ok(()) => {}
-            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
-            Err(error) => {
-                return Err(format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into());
+        let target_pid = cluster.nodes[1].process.as_ref().ok_or("target process is not running")?.id();
+        if scenario == InterruptionScenario::TargetEndpointBlackhole {
+            let node_pids = cluster
+                .nodes
+                .iter()
+                .map(|node| {
+                    node.process
+                        .as_ref()
+                        .ok_or("cluster process is not running")
+                        .map(std::process::Child::id)
+                })
+                .collect::<Result<Vec<_>, _>>()?;
+
+            timeout(Duration::from_secs(2), TcpStream::connect(&cluster.nodes[1].address)).await??;
+            let mut blackhole = TcpPortBlackhole::install(&cluster.nodes[1].address)?;
+            let blocked_connect = timeout(Duration::from_millis(500), TcpStream::connect(&cluster.nodes[1].address)).await;
+            assert!(
+                blocked_connect.is_err(),
+                "target endpoint connection must time out while the OUTPUT DROP rule is active: {blocked_connect:?}"
+            );
+
+            let stable_window_secs = std::env::var("RUSTFS_HEAL_CHAOS_BLACKHOLE_STABLE_SECS")
+                .ok()
+                .and_then(|value| value.parse::<u64>().ok())
+                .unwrap_or(2)
+                .clamp(1, 5);
+            let blackhole_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_BLACKHOLE_TIMEOUT_SECS")
+                .ok()
+                .and_then(|value| value.parse::<u64>().ok())
+                .unwrap_or(20)
+                .clamp(stable_window_secs + 1, 60);
+            let blackhole_deadline = Instant::now() + Duration::from_secs(blackhole_timeout_secs);
+            let mut stable_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
+            let mut stable_since = Instant::now();
+            loop {
+                for (node_index, (node, expected_pid)) in cluster.nodes.iter_mut().zip(&node_pids).enumerate() {
+                    let process = node
+                        .process
+                        .as_mut()
+                        .ok_or_else(|| format!("node {node_index} process disappeared"))?;
+                    assert_eq!(process.id(), *expected_pid, "node {node_index} PID changed during endpoint blackhole");
+                    assert!(process.try_wait()?.is_none(), "node {node_index} exited during endpoint blackhole");
+                }
+
+                let current_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
+                if current_count == expected_manifests.len() {
+                    return Err("root heal completed before the endpoint blackhole became observable".into());
+                }
+                if current_count != stable_count {
+                    stable_count = current_count;
+                    stable_since = Instant::now();
+                }
+                if stable_since.elapsed() >= Duration::from_secs(stable_window_secs) {
+                    break;
+                }
+                if Instant::now() >= blackhole_deadline {
+                    return Err(format!(
+                        "target rebuild never remained stable for {stable_window_secs}s during the endpoint blackhole: last_count={stable_count}, total={}",
+                        expected_manifests.len()
+                    )
+                    .into());
+                }
+                sleep(Duration::from_millis(100)).await;
+            }
+
+            let blocked_task_body = timeout(
+                Duration::from_secs(5),
+                signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
+            )
+            .await??;
+            let blocked_task: serde_json::Value = serde_json::from_str(&blocked_task_body)
+                .map_err(|err| format!("blackholed task status is not JSON ({err}): {blocked_task_body}"))?;
+            assert_eq!(
+                blocked_task["summary"].as_str(),
+                Some("running"),
+                "the original admin task must remain resumable during the endpoint blackhole: {blocked_task}"
+            );
+            assert!(
+                census_object_version_on_disk(&replaced_disk, bucket, &partial_manifest.key, None)?
+                    .matches_manifest(&partial_manifest.shard_census),
+                "the witnessed complete shard must survive the endpoint blackhole"
+            );
+
+            blackhole.restore()?;
+            timeout(Duration::from_secs(2), TcpStream::connect(&cluster.nodes[1].address)).await??;
+            info!(
+                event = "heal_endpoint_blackhole_restored",
+                component = "e2e_test",
+                subsystem = "heal",
+                interruption_kind,
+                stable_metadata_count = stable_count,
+                stable_window_secs,
+                "Restored target endpoint forwarding"
+            );
+        } else {
+            cluster.stop_node(interruption_node)?;
+            let stopped_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
+            assert!(
+                stopped_count > 0 && stopped_count < expected_manifests.len(),
+                "node {interruption_node} must stop during a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
+                expected_manifests.len()
+            );
+            assert!(
+                census_object_version_on_disk(&replaced_disk, bucket, &partial_manifest.key, None)?
+                    .matches_manifest(&partial_manifest.shard_census),
+                "the witnessed complete shard must survive interruption"
+            );
+            let unclean_shutdown_marker = Path::new(&cluster.nodes[interruption_node].data_dir)
+                .join(".rustfs.sys")
+                .join("unclean-shutdown");
+            if background_enabled {
+                assert!(
+                    unclean_shutdown_marker.is_file(),
+                    "background restart must retain the real unclean-shutdown marker"
+                );
+            } else {
+                match std::fs::remove_file(&unclean_shutdown_marker) {
+                    Ok(()) => {}
+                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+                    Err(error) => {
+                        return Err(
+                            format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into()
+                        );
+                    }
+                }
+            }
+            cluster.start_node(interruption_node).await?;
+            if interruption_node == 0 {
+                let target = cluster.nodes[1]
+                    .process
+                    .as_mut()
+                    .ok_or("target process disappeared during coordinator restart")?;
+                assert_eq!(target.id(), target_pid, "coordinator restart must not replace the target process");
+                assert!(target.try_wait()?.is_none(), "the target must remain alive during coordinator restart");
             }
         }
-        cluster.start_node(1).await?;
+
+        let scanner_cycle_floor = if background_enabled {
+            Some(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs())
+        } else {
+            None
+        };
 
         let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REPLACED_DISK_TIMEOUT_SECS")
             .ok()
@@ -832,13 +1290,22 @@ mod tests {
             {
                 let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
                 let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
-                if matching == expected_manifests.len() && outage_census.is_complete() {
+                let pool_metadata_matches = match &expected_pool_metadata {
+                    Some(expected) => {
+                        census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?
+                            .matches_manifest(expected)
+                    }
+                    None => true,
+                };
+                if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches {
                     break;
                 }
             }
             if Instant::now() >= heal_deadline {
                 let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
                 let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
+                let pool_metadata =
+                    census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
                 let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
                     .await
                     .unwrap_or_else(|err| format!("status request failed: {err}"));
@@ -858,7 +1325,7 @@ mod tests {
                     Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
                 };
                 return Err(format!(
-                    "root heal did not resume after target restart within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, replacement_status={replacement_status}",
+                    "root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
                     expected_manifests.len()
                 )
                 .into());
@@ -885,6 +1352,17 @@ mod tests {
             "the outage object must be rebuilt into its own missing erasure slot"
         );
 
+        if let Some(cycle_end) = scanner_cycle_floor {
+            wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
+        }
+
+        let mut expected_keys = expected_manifests
+            .iter()
+            .map(|manifest| manifest.key.clone())
+            .collect::<HashSet<_>>();
+        assert!(expected_keys.insert(outage_key.to_string()));
+        assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
+
         let target_client = cluster.create_s3_client(1)?;
         for expected in &expected_manifests {
             let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
@@ -914,6 +1392,30 @@ mod tests {
         let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
         let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
             .map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
+        if interruption_node == 0 {
+            // Admin tasks are process-local. Physical and queue convergence
+            // above establish recovery; a lost task must not report success.
+            assert_eq!(
+                task_status["summary"].as_str(),
+                Some("notFound"),
+                "interrupted task status: {task_status}"
+            );
+            assert_eq!(
+                task_status["detail"].as_str(),
+                Some("heal task not found or expired"),
+                "interrupted admin task must be explicitly unavailable: {task_status}"
+            );
+            info!(
+                event = "heal_interruption_recovered",
+                component = "e2e_test",
+                subsystem = "heal",
+                interruption_node,
+                interruption_kind,
+                task_state = "not_found",
+                "Physical recovery completed after coordinator restart"
+            );
+            return Ok(());
+        }
         if task_status["summary"].as_str() != Some("finished") {
             return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
         }

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -560,18 +560,12 @@ async fn test_multipart_encryption_type(
         .set_parts(Some(completed_parts))
         .build();
 
-    let mut complete_request = s3_client
+    let complete_request = s3_client
         .complete_multipart_upload()
         .bucket(bucket)
         .key(object_key)
         .upload_id(upload_id)
         .multipart_upload(completed_multipart_upload);
-    if matches!(encryption_type, EncryptionType::SSEC) {
-        complete_request = complete_request
-            .sse_customer_algorithm("AES256")
-            .sse_customer_key(sse_c_key.as_ref().unwrap())
-            .sse_customer_key_md5(sse_c_md5.as_ref().unwrap());
-    }
     let _complete_output = complete_request.send().await?;
 
     // Download and verify

crates/e2e_test/src/lib.rs

@@ -23,10 +23,17 @@ pub mod common;
 #[cfg(test)]
 pub mod chaos;
 
-// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8).
+// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
+// and on-demand-migration source scenarios (backlog#2151).
 #[cfg(test)]
 pub mod fake_s3_target;
 
+// On-demand migration (backlog#2147): shared two-server environment, admin
+// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
+// added by later ODM tasks.
+#[cfg(test)]
+pub mod on_demand_migration;
+
 // Socket-level network fault-injection proxy for black-box cluster tests
 // (backlog#1325 network fault-injection block): latency / blackhole / one-way
 // partition on the wire between nodes. Serves #1312/#1319 (lock-plane one-way
@@ -348,6 +355,11 @@ mod delete_regression_test;
 #[cfg(test)]
 mod listing_regression_test;
 
+// Cluster regression: objects committed at degraded write quorum must stay
+// listable while a different drive is offline (CI run 33478999853).
+#[cfg(test)]
+mod degraded_listing_availability_test;
+
 // P1 regression: bucket statistics accuracy (rustfs#5615, #5008, #5116, #5055, #3898, #1012)
 #[cfg(test)]
 mod bucket_stats_regression_test;

crates/e2e_test/src/negative_sigv4_test.rs

@@ -225,8 +225,8 @@ fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
     let checksum = sha256_base64(decoded);
     let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
     encoded.extend_from_slice(decoded);
-    encoded.extend_from_slice(b"\r\n0\r\n\r\n");
-    encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
+    encoded.extend_from_slice(b"\r\n0\r\n");
+    encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}\r\n\r\n").as_bytes());
     encoded
 }
 
@@ -549,6 +549,68 @@ async fn tampered_upload_part_payload_is_rejected() -> Result<(), Box<dyn std::e
     Ok(())
 }
 
+/// s3s v0.16 validates the aws-chunked decoded length while RustFS consumes the
+/// body stream. Mismatches are client body errors and must not leak as 500s.
+#[tokio::test]
+async fn aws_chunked_decoded_length_mismatch_returns_incomplete_body() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+
+    for (key, declared_len) in [
+        ("decoded-length-overrun.bin", 3_usize),
+        ("decoded-length-shortfall.bin", 9_usize),
+    ] {
+        let decoded = b"decoded";
+        assert_ne!(declared_len, decoded.len(), "test case must exercise a mismatch");
+        let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(decoded);
+        let decoded_content_length = declared_len.to_string();
+        let path = format!("/{BUCKET}/{key}");
+        let signer = SigV4::new(&env);
+        let extra_signed_headers = [
+            ("content-encoding", "aws-chunked"),
+            ("x-amz-decoded-content-length", decoded_content_length.as_str()),
+            ("x-amz-trailer", "x-amz-checksum-sha256"),
+        ];
+        let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
+
+        let response = local_http_client()
+            .put(format!("{}{}", env.url, path))
+            .header("authorization", &headers.authorization)
+            .header("content-encoding", "aws-chunked")
+            .header("x-amz-content-sha256", &headers.content_sha256)
+            .header("x-amz-date", &headers.amz_date)
+            .header("x-amz-decoded-content-length", &decoded_content_length)
+            .header("x-amz-trailer", "x-amz-checksum-sha256")
+            .body(encoded_body)
+            .timeout(std::time::Duration::from_secs(10))
+            .send()
+            .await?;
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        assert_eq!(
+            status,
+            reqwest::StatusCode::BAD_REQUEST,
+            "decoded length mismatch must be a client error, body:\n{body}"
+        );
+        assert_error_code(&body, "IncompleteBody");
+
+        let absent = env
+            .create_s3_client()
+            .get_object()
+            .bucket(BUCKET)
+            .key(key)
+            .send()
+            .await
+            .expect_err("decoded length mismatch must not publish an object");
+        assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
+        assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
+    }
+
+    env.stop_server();
+    Ok(())
+}
+
 /// (e) A request whose `x-amz-date` is skewed beyond the server's tolerance
 /// (s3s default 900s / 15 min) must be rejected with RequestTimeTooSkewed /
 /// 403. The signature is otherwise valid: the credential-scope date and

crates/e2e_test/src/on_demand_migration/common.rs

@@ -0,0 +1,452 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared environment for on-demand migration (ODM) end-to-end tests.
+//!
+//! [`OdmTestEnv`] pairs one RustFS server under test with one in-process
+//! programmable S3 source ([`FakeS3Target`]). Admin calls target the route
+//! convention fixed by the tracking plan
+//! (`/rustfs/admin/v3/on-demand-migration/{bucket}`, JSON bodies); the
+//! server side lands with ODM-07, so until then the wrappers compile but are
+//! not exercised by the harness self-test.
+
+use crate::common::{RustFSTestEnvironment, signed_request};
+use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FakeS3TargetOptions, SeedMetadata};
+use aws_config::retry::RetryConfig;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use bytes::Bytes;
+use serde::Serialize;
+use std::fmt;
+
+pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
+
+/// Module switch the server reads at startup (`false` before GA). The harness
+/// turns it on so scenario tests exercise the feature without repeating it.
+pub const ODM_MODULE_SWITCH_ENV: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED";
+/// Admin route prefix; the bucket name is appended as one path segment.
+pub const ODM_ADMIN_ROUTE: &str = "/rustfs/admin/v3/on-demand-migration";
+/// Region the fake source is addressed with (it accepts any SigV4 region).
+pub const FAKE_SOURCE_REGION: &str = "us-east-1";
+
+/// Wire form of the bucket-level ODM configuration (ODM-01 model). Every
+/// field is public so a scenario can tweak one knob and serialize the rest
+/// with the documented defaults.
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSourceSpec {
+    pub version: u32,
+    pub enabled: bool,
+    pub source: OdmSource,
+    pub filter: OdmFilter,
+    pub policy: OdmPolicy,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSource {
+    pub provider: String,
+    pub endpoint: String,
+    pub region: String,
+    pub bucket: String,
+    pub path_style: String,
+    pub credentials: Option<OdmCredentials>,
+    pub tls: OdmTls,
+}
+
+#[derive(Clone, Serialize)]
+pub struct OdmCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+    pub session_token: Option<String>,
+}
+
+impl fmt::Debug for OdmCredentials {
+    /// Test logs are captured into CI artifacts; keep the secret out of them.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("OdmCredentials")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &"REDACTED")
+            .field("session_token", &self.session_token.as_ref().map(|_| "REDACTED"))
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct OdmTls {
+    pub skip_verify: bool,
+    pub ca_cert_pem: Option<String>,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct OdmFilter {
+    pub prefix: Option<String>,
+    pub source_prefix: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmPolicy {
+    pub head: String,
+    pub range_get: String,
+    pub source_error: String,
+    pub respect_local_delete_marker: bool,
+    pub preserve_etag: bool,
+    pub copy_tags: bool,
+    pub emit_events: bool,
+    pub negative_cache_ttl_secs: u64,
+    pub inline_max_bytes: u64,
+    pub multipart_part_size_bytes: u64,
+    pub max_concurrent_pulls: u32,
+    pub pull_queue_capacity: u32,
+    pub source_timeout: OdmSourceTimeout,
+    pub bandwidth_limit_bytes_per_sec: Option<u64>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSourceTimeout {
+    pub connect_ms: u64,
+    pub first_byte_ms: u64,
+    pub idle_ms: u64,
+}
+
+impl Default for OdmPolicy {
+    /// The ODM-01 defaults verbatim.
+    fn default() -> Self {
+        Self {
+            head: "proxy".to_string(),
+            range_get: "serve_and_backfill".to_string(),
+            source_error: "propagate".to_string(),
+            respect_local_delete_marker: true,
+            preserve_etag: true,
+            copy_tags: false,
+            emit_events: true,
+            negative_cache_ttl_secs: 30,
+            inline_max_bytes: 16 * 1024 * 1024,
+            multipart_part_size_bytes: 64 * 1024 * 1024,
+            max_concurrent_pulls: 8,
+            pull_queue_capacity: 1024,
+            source_timeout: OdmSourceTimeout {
+                connect_ms: 5_000,
+                first_byte_ms: 15_000,
+                idle_ms: 30_000,
+            },
+            bandwidth_limit_bytes_per_sec: None,
+        }
+    }
+}
+
+impl OdmSourceSpec {
+    /// Enabled configuration pointing at a bucket on the fake source with the
+    /// fixture credentials, path-style addressing, and default policy.
+    pub fn for_fake_source(source: &FakeS3Target, source_bucket: impl Into<String>) -> Self {
+        Self::new(
+            "s3",
+            source.endpoint(),
+            FAKE_SOURCE_REGION,
+            source_bucket,
+            FAKE_ACCESS_KEY,
+            FAKE_SECRET_KEY,
+        )
+    }
+
+    /// Enabled configuration pointing at a bucket on a second RustFS server
+    /// (see [`start_source_rustfs`]).
+    pub fn for_rustfs_source(source: &RustFSTestEnvironment, source_bucket: impl Into<String>) -> Self {
+        Self::new(
+            "rustfs",
+            &source.url,
+            FAKE_SOURCE_REGION,
+            source_bucket,
+            &source.access_key,
+            &source.secret_key,
+        )
+    }
+
+    fn new(
+        provider: &str,
+        endpoint: &str,
+        region: &str,
+        source_bucket: impl Into<String>,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Self {
+        Self {
+            version: 1,
+            enabled: true,
+            source: OdmSource {
+                provider: provider.to_string(),
+                endpoint: endpoint.to_string(),
+                region: region.to_string(),
+                bucket: source_bucket.into(),
+                path_style: "path".to_string(),
+                credentials: Some(OdmCredentials {
+                    access_key: access_key.to_string(),
+                    secret_key: secret_key.to_string(),
+                    session_token: None,
+                }),
+                tls: OdmTls::default(),
+            },
+            filter: OdmFilter::default(),
+            policy: OdmPolicy::default(),
+        }
+    }
+
+    pub fn to_json(&self) -> serde_json::Value {
+        serde_json::to_value(self).expect("ODM source spec serializes")
+    }
+}
+
+/// Backfill job control (ODM-12 route shape).
+#[derive(Debug, Clone)]
+pub enum BackfillOp {
+    Start(BackfillRequest),
+    Cancel,
+    Status,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct BackfillRequest {
+    pub prefix: Option<String>,
+    pub skip_existing: Option<String>,
+    pub dry_run: bool,
+}
+
+/// Status plus raw body of an admin call, so a scenario can assert on the
+/// HTTP status first and only then parse the JSON.
+#[derive(Debug, Clone)]
+pub struct AdminResponse {
+    pub status: u16,
+    pub body: String,
+}
+
+impl AdminResponse {
+    pub fn json(&self) -> Result<serde_json::Value, BoxError> {
+        Ok(serde_json::from_str(&self.body)?)
+    }
+}
+
+/// One object to seed into the source.
+#[derive(Clone)]
+pub struct SeedObject {
+    pub key: String,
+    pub body: Bytes,
+    pub metadata: SeedMetadata,
+}
+
+impl SeedObject {
+    pub fn new(key: impl Into<String>, body: impl Into<Bytes>) -> Self {
+        Self {
+            key: key.into(),
+            body: body.into(),
+            metadata: SeedMetadata::new(),
+        }
+    }
+
+    pub fn with_metadata(mut self, metadata: SeedMetadata) -> Self {
+        self.metadata = metadata;
+        self
+    }
+}
+
+/// RustFS under test plus its fake S3 source.
+pub struct OdmTestEnv {
+    pub rustfs: RustFSTestEnvironment,
+    pub source: FakeS3Target,
+    /// S3 client for the RustFS under test.
+    pub client: Client,
+}
+
+impl OdmTestEnv {
+    /// Start a fake source with default limits and a RustFS server with the
+    /// ODM module switch enabled.
+    pub async fn start() -> Result<Self, BoxError> {
+        Self::start_with_options(FakeS3TargetOptions::default()).await
+    }
+
+    pub async fn start_with_options(options: FakeS3TargetOptions) -> Result<Self, BoxError> {
+        let source = FakeS3Target::start_with_options(options).await?;
+        let mut rustfs = RustFSTestEnvironment::new().await?;
+        rustfs
+            .start_rustfs_server_with_env(vec![], &[(ODM_MODULE_SWITCH_ENV, "true")])
+            .await?;
+        let client = rustfs.create_s3_client();
+        Ok(Self { rustfs, source, client })
+    }
+
+    /// S3 client addressing the fake source directly, for assertions on the
+    /// source's own state. Retries are off so a scripted fault is consumed by
+    /// exactly the request the test issued.
+    pub fn source_client(&self) -> Client {
+        fake_source_client(&self.source)
+    }
+
+    /// Enabled ODM configuration for `source_bucket` on the fake source.
+    pub fn fake_source_spec(&self, source_bucket: impl Into<String>) -> OdmSourceSpec {
+        OdmSourceSpec::for_fake_source(&self.source, source_bucket)
+    }
+
+    /// `PUT /rustfs/admin/v3/on-demand-migration/{bucket}` with the JSON spec.
+    pub async fn configure_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::PUT, &format!("/{bucket}"), Some(spec.to_json()))
+            .await
+    }
+
+    /// Same as [`Self::configure_source`] with `dry-run=true`: validate and
+    /// probe without persisting.
+    pub async fn validate_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::PUT, &format!("/{bucket}?dry-run=true"), Some(spec.to_json()))
+            .await
+    }
+
+    /// `GET .../{bucket}`: redacted configuration, 404 when unconfigured.
+    pub async fn get_config(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::GET, &format!("/{bucket}"), None).await
+    }
+
+    /// `DELETE .../{bucket}`: remove the configuration (idempotent).
+    pub async fn disable(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::DELETE, &format!("/{bucket}"), None).await
+    }
+
+    /// `GET .../{bucket}/status`: runtime snapshot.
+    pub async fn status(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::GET, &format!("/{bucket}/status"), None).await
+    }
+
+    /// Backfill control: `POST .../{bucket}/backfill?op=start|cancel` or
+    /// `GET .../{bucket}/backfill` for the checkpoint.
+    pub async fn backfill(&self, bucket: &str, op: BackfillOp) -> Result<AdminResponse, BoxError> {
+        match op {
+            BackfillOp::Start(request) => {
+                self.admin(
+                    http::Method::POST,
+                    &format!("/{bucket}/backfill?op=start"),
+                    Some(serde_json::to_value(request)?),
+                )
+                .await
+            }
+            BackfillOp::Cancel => {
+                self.admin(http::Method::POST, &format!("/{bucket}/backfill?op=cancel"), None)
+                    .await
+            }
+            BackfillOp::Status => self.admin(http::Method::GET, &format!("/{bucket}/backfill"), None).await,
+        }
+    }
+
+    async fn admin(
+        &self,
+        method: http::Method,
+        path_and_query: &str,
+        body: Option<serde_json::Value>,
+    ) -> Result<AdminResponse, BoxError> {
+        let url = format!("{}{ODM_ADMIN_ROUTE}{path_and_query}", self.rustfs.url);
+        let body = body.map(|value| serde_json::to_vec(&value)).transpose()?;
+        let content_type = body.is_some().then_some("application/json");
+        let response = signed_request(method, &url, &self.rustfs.access_key, &self.rustfs.secret_key, body, content_type).await?;
+        Ok(AdminResponse {
+            status: response.status().as_u16(),
+            body: response.text().await?,
+        })
+    }
+
+    /// Store objects directly in the fake source (no wire traffic, no journal
+    /// entries). Returns the ETags in input order.
+    pub fn seed_source(&self, source_bucket: &str, objects: &[SeedObject]) -> Vec<String> {
+        objects
+            .iter()
+            .map(|object| {
+                self.source
+                    .put_seed_object(source_bucket, object.key.clone(), object.body.clone(), &object.metadata)
+            })
+            .collect()
+    }
+
+    /// Whether `key` is listed by the RustFS under test. Listing is served from
+    /// local state only, so this does not trigger a migration the way GET or
+    /// HEAD would.
+    pub async fn local_key_listed(&self, bucket: &str, key: &str) -> Result<bool, BoxError> {
+        let listed = self
+            .client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(key)
+            .max_keys(1)
+            .send()
+            .await?;
+        Ok(listed.contents().iter().any(|object| object.key() == Some(key)))
+    }
+
+    /// Panics unless `key` is stored locally with exactly `expected` bytes.
+    /// Presence is checked through listing first so a missing object fails
+    /// here instead of being pulled from the source by the GET.
+    pub async fn assert_local_present(&self, bucket: &str, key: &str, expected: &[u8]) {
+        assert!(
+            self.local_key_listed(bucket, key)
+                .await
+                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
+            "{bucket}/{key} must be present locally"
+        );
+        let body = self
+            .client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .unwrap_or_else(|error| panic!("GET {bucket}/{key} failed: {error}"))
+            .body
+            .collect()
+            .await
+            .unwrap_or_else(|error| panic!("reading {bucket}/{key} failed: {error}"))
+            .into_bytes();
+        assert_eq!(body.as_ref(), expected, "{bucket}/{key} local content mismatch");
+    }
+
+    /// Panics if `key` is listed locally.
+    pub async fn assert_local_absent(&self, bucket: &str, key: &str) {
+        assert!(
+            !self
+                .local_key_listed(bucket, key)
+                .await
+                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
+            "{bucket}/{key} must be absent locally"
+        );
+    }
+}
+
+/// S3 client for the fake source with retries disabled (see
+/// [`OdmTestEnv::source_client`]).
+pub fn fake_source_client(source: &FakeS3Target) -> Client {
+    let credentials = Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "odm-fake-source");
+    Client::from_conf(
+        aws_sdk_s3::Config::builder()
+            .credentials_provider(credentials)
+            .region(Region::new(FAKE_SOURCE_REGION))
+            .endpoint_url(source.endpoint())
+            .force_path_style(true)
+            .behavior_version_latest()
+            .retry_config(RetryConfig::standard().with_max_attempts(1))
+            .http_client(SmithyHttpClientBuilder::new().build_http())
+            .build(),
+    )
+}
+
+/// Start a second, fully independent RustFS process (own port, data
+/// directory, and default credentials) to act as a real S3 source. It is
+/// spawned the same way `reliant::tiering` starts its cold tier; the process
+/// is stopped and its directory removed when the returned environment drops.
+pub async fn start_source_rustfs() -> Result<RustFSTestEnvironment, BoxError> {
+    let mut source = RustFSTestEnvironment::new().await?;
+    source.start_rustfs_server_without_cleanup(vec![]).await?;
+    Ok(source)
+}

crates/e2e_test/src/on_demand_migration/harness_self_test.rs

@@ -0,0 +1,606 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Self-test of the ODM harness (rustfs/backlog#2151): the fake source's
+//! migration-facing surface (ListObjectsV2 paging, `Range`, unversioned
+//! buckets, metadata replay, fault actions) and the two-server environment.
+//! No ODM behavior is exercised here.
+
+use super::common::{OdmTestEnv, SeedObject, fake_source_client, start_source_rustfs};
+use crate::fake_s3_target::{BucketMode, FakeS3Target, FakeS3TargetOptions, FaultAction, Operation, SeedMetadata};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::{ByteStream, DateTime};
+use bytes::Bytes;
+use std::collections::BTreeSet;
+use std::time::{Duration, Instant};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const SOURCE_BUCKET: &str = "odm-source";
+
+/// Position-dependent payload so a misaligned range read is caught.
+fn payload(len: usize) -> Bytes {
+    (0..len).map(|index| (index % 251) as u8).collect::<Vec<u8>>().into()
+}
+
+async fn fake_source() -> Result<(FakeS3Target, Client), Box<dyn std::error::Error + Send + Sync>> {
+    let source = FakeS3Target::start().await?;
+    source.create_bucket(SOURCE_BUCKET);
+    let client = fake_source_client(&source);
+    Ok((source, client))
+}
+
+/// Full ListObjectsV2 traversal. Returns `(keys, common prefixes, pages)` and
+/// checks the page shape on the way: every page except the last is full and
+/// truncated, the last carries no continuation token.
+async fn list_all(
+    client: &Client,
+    prefix: Option<&str>,
+    delimiter: Option<&str>,
+    start_after: Option<&str>,
+    max_keys: i32,
+) -> Result<(Vec<String>, Vec<String>, usize), Box<dyn std::error::Error + Send + Sync>> {
+    let mut keys = Vec::new();
+    let mut prefixes = Vec::new();
+    let mut pages = 0usize;
+    let mut token: Option<String> = None;
+    loop {
+        let page = client
+            .list_objects_v2()
+            .bucket(SOURCE_BUCKET)
+            .set_prefix(prefix.map(str::to_string))
+            .set_delimiter(delimiter.map(str::to_string))
+            .set_start_after(start_after.map(str::to_string))
+            .max_keys(max_keys)
+            .set_continuation_token(token.clone())
+            .send()
+            .await?;
+        pages += 1;
+        let page_keys: Vec<String> = page
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        let page_prefixes: Vec<String> = page
+            .common_prefixes()
+            .iter()
+            .filter_map(|common| common.prefix().map(str::to_string))
+            .collect();
+        let entries = page_keys.len() + page_prefixes.len();
+        assert_eq!(page.key_count(), Some(entries as i32), "KeyCount must count keys and prefixes");
+        assert_eq!(page.continuation_token(), token.as_deref(), "the request token must be echoed");
+        keys.extend(page_keys);
+        prefixes.extend(page_prefixes);
+        if page.is_truncated() == Some(true) {
+            assert_eq!(entries as i32, max_keys, "every truncated page must be full");
+            token = Some(
+                page.next_continuation_token()
+                    .expect("truncated page must carry a continuation token")
+                    .to_string(),
+            );
+        } else {
+            assert!(page.next_continuation_token().is_none(), "final page must not carry a token");
+            return Ok((keys, prefixes, pages));
+        }
+    }
+}
+
+#[tokio::test]
+async fn fake_source_list_objects_v2_paginates_with_delimiter() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let mut expected_keys = BTreeSet::new();
+    for directory in 0..30 {
+        for file in 0..30 {
+            expected_keys.insert(format!("d{directory:02}/k{file:03}"));
+        }
+    }
+    for index in 0..100 {
+        expected_keys.insert(format!("top-{index:03}"));
+    }
+    assert_eq!(expected_keys.len(), 1000);
+    for key in &expected_keys {
+        source.put_seed_object(SOURCE_BUCKET, key.clone(), Bytes::from(key.clone()), &SeedMetadata::new());
+    }
+    // A key whose current version is a delete marker must stay hidden.
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("hidden/marker")
+        .body(ByteStream::from_static(b"gone"))
+        .send()
+        .await?;
+    client
+        .delete_object()
+        .bucket(SOURCE_BUCKET)
+        .key("hidden/marker")
+        .send()
+        .await?;
+    let expected_sorted: Vec<String> = expected_keys.iter().cloned().collect();
+    let expected_prefixes: Vec<String> = (0..30).map(|directory| format!("d{directory:02}/")).collect();
+    let expected_top: Vec<String> = (0..100).map(|index| format!("top-{index:03}")).collect();
+
+    // Flat traversal in byte order, 1000 keys in pages of 7.
+    let (keys, prefixes, pages) = list_all(&client, None, None, None, 7).await?;
+    assert_eq!(keys, expected_sorted);
+    assert!(prefixes.is_empty());
+    assert_eq!(pages, 143);
+
+    // Delimiter folding: 30 common prefixes then 100 top-level keys, pages of 7.
+    let (keys, prefixes, pages) = list_all(&client, None, Some("/"), None, 7).await?;
+    assert_eq!(prefixes, expected_prefixes);
+    assert_eq!(keys, expected_top);
+    assert_eq!(pages, 19);
+
+    // Empty prefix equals no prefix.
+    let (keys, _, _) = list_all(&client, Some(""), None, None, 1000).await?;
+    assert_eq!(keys, expected_sorted);
+
+    // No match: empty, not truncated, no token.
+    let (keys, prefixes, pages) = list_all(&client, Some("zzz/"), Some("/"), None, 7).await?;
+    assert!(keys.is_empty() && prefixes.is_empty());
+    assert_eq!(pages, 1);
+    let (keys, _, _) = list_all(&client, Some("hidden/"), None, None, 7).await?;
+    assert!(keys.is_empty(), "a current delete marker must hide its key");
+
+    // Exact page boundary: 30 keys under one directory, max-keys=30 -> one
+    // untruncated page.
+    let (keys, prefixes, pages) = list_all(&client, Some("d05/"), Some("/"), None, 30).await?;
+    assert_eq!(keys.len(), 30);
+    assert!(prefixes.is_empty());
+    assert_eq!(pages, 1);
+
+    // start-after skips keys at or before the marker.
+    let (keys, _, _) = list_all(&client, None, None, Some("top-097"), 1000).await?;
+    assert_eq!(keys, ["top-098", "top-099"]);
+
+    // max-keys is clamped to 1000; exactly 1000 keys fit in one page.
+    let (keys, _, pages) = list_all(&client, None, None, None, 5000).await?;
+    assert_eq!(keys.len(), 1000);
+    assert_eq!(pages, 1);
+
+    let listings: Vec<_> = source
+        .requests()
+        .into_iter()
+        .filter(|record| record.operation == Operation::ListObjectsV2)
+        .collect();
+    assert!(listings.len() >= 143 + 19);
+    assert!(listings.iter().any(|record| record.prefix.as_deref() == Some("d05/")));
+    assert!(
+        listings.iter().any(|record| record.continuation_token.is_some()),
+        "resumed pages must journal their continuation token"
+    );
+    assert!(listings.iter().all(|record| record.user_agent.is_some()));
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_range_get_variants_and_416() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(1000);
+    source.put_seed_object(SOURCE_BUCKET, "ranged", body.clone(), &SeedMetadata::new());
+
+    for (range, expected_range, expected_slice) in [
+        ("bytes=10-19", "bytes 10-19/1000", &body[10..20]),
+        ("bytes=990-", "bytes 990-999/1000", &body[990..]),
+        ("bytes=-5", "bytes 995-999/1000", &body[995..]),
+        ("bytes=0-5000", "bytes 0-999/1000", &body[..]),
+    ] {
+        let output = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("ranged")
+            .range(range)
+            .send()
+            .await?;
+        assert_eq!(output.content_range(), Some(expected_range), "{range}");
+        assert_eq!(output.accept_ranges(), Some("bytes"), "{range}");
+        assert_eq!(output.content_length(), Some(expected_slice.len() as i64), "{range}");
+        let collected = output.body.collect().await?.into_bytes();
+        assert_eq!(collected.as_ref(), expected_slice, "{range}");
+    }
+    let head = client
+        .head_object()
+        .bucket(SOURCE_BUCKET)
+        .key("ranged")
+        .range("bytes=10-19")
+        .send()
+        .await?;
+    assert_eq!(head.content_range(), Some("bytes 10-19/1000"));
+    assert_eq!(head.content_length(), Some(10));
+
+    for range in ["bytes=1000-", "bytes=-0"] {
+        let error = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("ranged")
+            .range(range)
+            .send()
+            .await
+            .expect_err("unsatisfiable range must fail");
+        let response = error.raw_response().expect("416 must retain the raw response");
+        assert_eq!(response.status().as_u16(), 416, "{range}");
+        assert_eq!(response.headers().get("content-range"), Some("bytes */1000"), "{range}");
+        assert_eq!(error.code(), Some("InvalidRange"), "{range}");
+    }
+
+    let ranged = source
+        .requests()
+        .into_iter()
+        .find(|record| record.operation == Operation::GetObject && record.range.as_deref() == Some("bytes=10-19"))
+        .expect("the Range header must be journaled verbatim");
+    assert_eq!(ranged.key.as_deref(), Some("ranged"));
+    assert!(source.count_requests(Operation::GetObject, "ranged") >= 6);
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_unversioned_bucket_overwrites_and_deletes() -> TestResult {
+    let (source, client) = fake_source().await?;
+    source.create_bucket_with_mode("plain-source", BucketMode::Unversioned);
+    let versioning = client.get_bucket_versioning().bucket("plain-source").send().await?;
+    assert!(versioning.status().is_none(), "unversioned bucket must report no versioning status");
+
+    let first = client
+        .put_object()
+        .bucket("plain-source")
+        .key("doc")
+        .body(ByteStream::from_static(b"first"))
+        .send()
+        .await?;
+    assert!(first.version_id().is_none());
+    let second = client
+        .put_object()
+        .bucket("plain-source")
+        .key("doc")
+        .body(ByteStream::from_static(b"second"))
+        .send()
+        .await?;
+    assert!(second.version_id().is_none());
+    let get = client.get_object().bucket("plain-source").key("doc").send().await?;
+    assert!(get.version_id().is_none(), "GET must not return x-amz-version-id");
+    assert_eq!(get.body.collect().await?.into_bytes().as_ref(), b"second");
+    let head = client.head_object().bucket("plain-source").key("doc").send().await?;
+    assert!(head.version_id().is_none(), "HEAD must not return x-amz-version-id");
+    assert_eq!(source.stored_versions("plain-source", "doc").len(), 1, "overwrite must replace in place");
+
+    let deleted = client.delete_object().bucket("plain-source").key("doc").send().await?;
+    assert!(deleted.delete_marker().is_none() && deleted.version_id().is_none());
+    let missing = client
+        .get_object()
+        .bucket("plain-source")
+        .key("doc")
+        .send()
+        .await
+        .expect_err("deleted object must be gone");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert_eq!(missing.code(), Some("NoSuchKey"));
+    let missing_head = client
+        .head_object()
+        .bucket("plain-source")
+        .key("doc")
+        .send()
+        .await
+        .expect_err("deleted object must fail HEAD");
+    assert_eq!(missing_head.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert!(source.stored_versions("plain-source", "doc").is_empty(), "DELETE must not leave a marker");
+
+    // The versioned bucket on the same target keeps its version ids.
+    let versioned = client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("doc")
+        .body(ByteStream::from_static(b"versioned"))
+        .send()
+        .await?;
+    assert!(versioned.version_id().is_some());
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_replays_standard_and_user_metadata() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(4096);
+    let expected_etag = format!("\"{}\"", {
+        use md5::Digest as _;
+        hex_simd::encode_to_string(md5::Md5::digest(&body), hex_simd::AsciiCase::Lower)
+    });
+    // 2026-01-01T00:00:00Z rendered as an HTTP date by the SDK.
+    let expires = DateTime::from_secs(1_767_225_600);
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("meta")
+        .body(ByteStream::from(body.clone()))
+        .content_type("application/x-odm")
+        .content_encoding("gzip")
+        .content_disposition("attachment; filename=\"meta.bin\"")
+        .content_language("en-US")
+        .cache_control("max-age=60")
+        .expires(expires)
+        .metadata("Foo-Bar", "mixed case name")
+        .metadata("UPPER", "upper name")
+        .metadata("already-lower", "lower name")
+        .send()
+        .await?;
+
+    let head = client.head_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
+    let get = client.get_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
+    for (label, content_type, content_encoding, content_disposition, content_language, cache_control, expires_string, e_tag) in [
+        (
+            "HEAD",
+            head.content_type(),
+            head.content_encoding(),
+            head.content_disposition(),
+            head.content_language(),
+            head.cache_control(),
+            head.expires_string(),
+            head.e_tag(),
+        ),
+        (
+            "GET",
+            get.content_type(),
+            get.content_encoding(),
+            get.content_disposition(),
+            get.content_language(),
+            get.cache_control(),
+            get.expires_string(),
+            get.e_tag(),
+        ),
+    ] {
+        assert_eq!(content_type, Some("application/x-odm"), "{label}");
+        assert_eq!(content_encoding, Some("gzip"), "{label}");
+        assert_eq!(content_disposition, Some("attachment; filename=\"meta.bin\""), "{label}");
+        assert_eq!(content_language, Some("en-US"), "{label}");
+        assert_eq!(cache_control, Some("max-age=60"), "{label}");
+        assert_eq!(expires_string, Some("Thu, 01 Jan 2026 00:00:00 GMT"), "{label}");
+        assert_eq!(e_tag, Some(expected_etag.as_str()), "{label}");
+    }
+    for metadata in [head.metadata(), get.metadata()] {
+        let metadata = metadata.expect("user metadata must be replayed");
+        assert_eq!(metadata.get("foo-bar").map(String::as_str), Some("mixed case name"));
+        assert_eq!(metadata.get("upper").map(String::as_str), Some("upper name"));
+        assert_eq!(metadata.get("already-lower").map(String::as_str), Some("lower name"));
+        assert!(!metadata.contains_key("Foo-Bar") && !metadata.contains_key("UPPER"));
+    }
+    assert!(head.last_modified().is_some());
+    assert_eq!(head.last_modified(), get.last_modified());
+    assert_eq!(head.content_length(), Some(4096));
+    assert_eq!(get.body.collect().await?.into_bytes(), body);
+
+    // Seeded objects replay the same way.
+    let seeded_etag = source.put_seed_object(
+        SOURCE_BUCKET,
+        "seeded",
+        Bytes::from_static(b"seeded"),
+        &SeedMetadata::new()
+            .content_type("text/plain")
+            .content_encoding("identity")
+            .cache_control("no-store")
+            .user_metadata("Origin", "seed"),
+    );
+    let seeded = client.head_object().bucket(SOURCE_BUCKET).key("seeded").send().await?;
+    assert_eq!(seeded.e_tag(), Some(format!("\"{seeded_etag}\"").as_str()));
+    assert_eq!(seeded.content_type(), Some("text/plain"));
+    assert_eq!(seeded.content_encoding(), Some("identity"));
+    assert_eq!(seeded.cache_control(), Some("no-store"));
+    assert_eq!(
+        seeded
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("seed")
+    );
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_fault_actions_truncate_stall_and_status() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(4096);
+    source.put_seed_object(SOURCE_BUCKET, "faulty", body.clone(), &SeedMetadata::new());
+
+    // TruncateBodyAt: headers promise 4096 bytes, the body ends after 100.
+    source.inject_for_key(Operation::GetObject, "faulty", FaultAction::TruncateBodyAt(100), 1);
+    let truncated = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert_eq!(truncated.content_length(), Some(4096));
+    let short_read = truncated
+        .body
+        .collect()
+        .await
+        .expect_err("a truncated body must fail to collect");
+    let short_read = short_read.to_string();
+    assert!(!short_read.is_empty());
+
+    // ResponseStatus: arbitrary status with the matching S3 error code.
+    for (code, expected_code) in [
+        (429u16, "SlowDown"),
+        (404, "NoSuchKey"),
+        (500, "InternalError"),
+        (503, "ServiceUnavailable"),
+    ] {
+        source.inject(Operation::GetObject, FaultAction::ResponseStatus(code), 1);
+        let error = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("faulty")
+            .send()
+            .await
+            .expect_err("scripted status must fail");
+        assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(code));
+        assert_eq!(error.code(), Some(expected_code));
+    }
+
+    // Stall: the fully computed response is held before its first byte.
+    source.inject(Operation::HeadObject, FaultAction::Stall(Duration::from_millis(400)), 1);
+    let started = Instant::now();
+    let stalled = client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert!(started.elapsed() >= Duration::from_millis(350), "stall must delay the first byte");
+    assert_eq!(stalled.content_length(), Some(4096));
+    let post_stall_started = Instant::now();
+    client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert!(post_stall_started.elapsed() < Duration::from_millis(350), "stall is consumed once");
+
+    // The object is intact once the script is drained.
+    let intact = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert_eq!(intact.body.collect().await?.into_bytes(), body);
+
+    assert_eq!(source.count_requests(Operation::GetObject, "faulty"), 6);
+    assert_eq!(source.count_requests(Operation::HeadObject, "faulty"), 2);
+    assert_eq!(source.count_requests(Operation::GetObject, "other"), 0);
+    let records = source.requests();
+    assert!(
+        records.iter().all(|record| record
+            .user_agent
+            .as_deref()
+            .is_some_and(|agent| agent.contains("aws-sdk-rust"))),
+        "the SDK user agent must be journaled"
+    );
+    assert!(
+        records
+            .iter()
+            .any(|record| record.fault == Some(FaultAction::TruncateBodyAt(100)))
+    );
+    assert!(
+        records
+            .iter()
+            .any(|record| record.fault == Some(FaultAction::Stall(Duration::from_millis(400))))
+    );
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_raised_object_cap_accepts_large_put() -> TestResult {
+    let source = FakeS3Target::start_with_options(FakeS3TargetOptions {
+        max_object_bytes: 96 * 1024 * 1024,
+    })
+    .await?;
+    source.create_bucket(SOURCE_BUCKET);
+    let client = fake_source_client(&source);
+    let len = 64 * 1024 * 1024 + 1;
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("large")
+        .body(ByteStream::from(vec![7u8; len]))
+        .send()
+        .await?;
+    let head = client.head_object().bucket(SOURCE_BUCKET).key("large").send().await?;
+    assert_eq!(head.content_length(), Some(len as i64));
+    let tail = client
+        .get_object()
+        .bucket(SOURCE_BUCKET)
+        .key("large")
+        .range("bytes=-1")
+        .send()
+        .await?;
+    assert_eq!(tail.content_range(), Some(format!("bytes {}-{}/{len}", len - 1, len - 1).as_str()));
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn odm_env_starts_rustfs_and_fake_source() -> TestResult {
+    let env = OdmTestEnv::start().await?;
+    env.source.create_bucket(SOURCE_BUCKET);
+    let local_bucket = "odm-local";
+    env.rustfs.create_test_bucket(local_bucket).await?;
+
+    let etags = env.seed_source(
+        SOURCE_BUCKET,
+        &[
+            SeedObject::new("seed/a", Bytes::from_static(b"alpha")),
+            SeedObject::new("seed/b", Bytes::from_static(b"beta"))
+                .with_metadata(SeedMetadata::new().content_type("text/plain").user_metadata("Kind", "seed")),
+        ],
+    );
+    assert_eq!(etags.len(), 2);
+    assert!(env.source.requests().is_empty(), "seeding must not touch the journal");
+    let source_client = env.source_client();
+    let seeded = source_client.head_object().bucket(SOURCE_BUCKET).key("seed/b").send().await?;
+    assert_eq!(seeded.content_type(), Some("text/plain"));
+    assert_eq!(seeded.e_tag(), Some(format!("\"{}\"", etags[1]).as_str()));
+    assert_eq!(env.source.count_requests(Operation::HeadObject, "seed/b"), 1);
+
+    env.assert_local_absent(local_bucket, "seed/a").await;
+    env.client
+        .put_object()
+        .bucket(local_bucket)
+        .key("seed/a")
+        .body(ByteStream::from_static(b"alpha"))
+        .send()
+        .await?;
+    env.assert_local_present(local_bucket, "seed/a", b"alpha").await;
+    env.assert_local_absent(local_bucket, "seed/b").await;
+
+    let spec = env.fake_source_spec(SOURCE_BUCKET).to_json();
+    assert_eq!(spec["version"], 1);
+    assert_eq!(spec["enabled"], true);
+    assert_eq!(spec["source"]["provider"], "s3");
+    assert_eq!(spec["source"]["endpoint"], env.source.endpoint());
+    assert_eq!(spec["source"]["bucket"], SOURCE_BUCKET);
+    assert_eq!(spec["source"]["credentials"]["secret_key"], "fake-secret");
+    assert_eq!(spec["policy"]["source_timeout"]["first_byte_ms"], 15_000);
+    assert!(spec["policy"]["bandwidth_limit_bytes_per_sec"].is_null());
+    let debug = format!("{:?}", env.fake_source_spec(SOURCE_BUCKET));
+    assert!(!debug.contains("fake-secret"), "Debug output must redact the secret");
+    Ok(())
+}
+
+#[tokio::test]
+async fn start_source_rustfs_round_trips_put_get() -> TestResult {
+    let env = OdmTestEnv::start().await?;
+    let source = start_source_rustfs().await?;
+    assert_ne!(source.url, env.rustfs.url, "the source must be a separate instance");
+
+    source.create_test_bucket(SOURCE_BUCKET).await?;
+    let source_client = source.create_s3_client();
+    let body = payload(70_000);
+    let put = source_client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("real/object")
+        .body(ByteStream::from(body.clone()))
+        .content_type("application/octet-stream")
+        .send()
+        .await?;
+    assert!(put.e_tag().is_some());
+    let get = source_client
+        .get_object()
+        .bucket(SOURCE_BUCKET)
+        .key("real/object")
+        .send()
+        .await?;
+    assert_eq!(get.content_type(), Some("application/octet-stream"));
+    assert_eq!(get.body.collect().await?.into_bytes(), body);
+
+    let visible_to_primary = env
+        .client
+        .list_buckets()
+        .send()
+        .await?
+        .buckets()
+        .iter()
+        .any(|bucket| bucket.name() == Some(SOURCE_BUCKET));
+    assert!(!visible_to_primary, "the two servers must not share state");
+    let spec = super::common::OdmSourceSpec::for_rustfs_source(&source, SOURCE_BUCKET).to_json();
+    assert_eq!(spec["source"]["provider"], "rustfs");
+    assert_eq!(spec["source"]["endpoint"], source.url);
+    Ok(())
+}

crates/e2e_test/src/on_demand_migration/mod.rs

@@ -0,0 +1,24 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! On-demand migration (ODM) end-to-end suite (rustfs/backlog#2147).
+//!
+//! `common` is the shared environment: one RustFS under test, one programmable
+//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
+//! `harness_self_test` proves the harness itself; ODM behavior scenarios are
+//! separate modules wired by later tasks.
+
+pub mod common;
+
+mod harness_self_test;

crates/e2e_test/src/replacement_privileged_e2e_test.rs

@@ -40,10 +40,10 @@ mod tests {
 
     const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E";
     const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E_IN_NAMESPACE";
+    const LOG_DIR_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_LOG_DIR";
     const TARGET_NODE: usize = 1;
     const TARGET_DRIVE: usize = 0;
     const MOUNT_SIZE: &str = "size=128m,mode=0700";
-    const ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS: u64 = 180;
     const REPLACEMENT_RECOVERY_DIR: &str = ".rustfs.sys/buckets/ahm-replacement";
     const REPLACEMENT_INTENT_SUFFIX: &str = "_ahm_replacement_intent.json";
     const REPLACEMENT_COMPLETION_PROOF_SUFFIX: &str = "_ahm_replacement_completion_proof.json";
@@ -142,6 +142,23 @@ mod tests {
             run_command("dmsetup", &["resume", &self.dm_name])
         }
 
+        fn verify_raw_io_is_unavailable(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let mapper = format!("/dev/mapper/{}", self.dm_name);
+            let output = Command::new("dd")
+                .env("LC_ALL", "C")
+                .arg(format!("if={mapper}"))
+                .args(["of=/dev/null", "bs=4096", "count=1", "iflag=direct", "status=none"])
+                .output()?;
+            if output.status.success() {
+                return Err(format!("dm-error target unexpectedly allowed a raw read from {mapper}").into());
+            }
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            if !stderr.contains("Input/output error") {
+                return Err(format!("raw read from dm-error target failed unexpectedly: {stderr}").into());
+            }
+            Ok(())
+        }
+
         fn restore_available(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
             let sectors = run_command_stdout("blockdev", &["--getsz", &self.loop_device])?;
             let linear_table = format!("0 {sectors} linear {} 0", self.loop_device);
@@ -194,6 +211,72 @@ mod tests {
         }
     }
 
+    struct ZramBlockMount {
+        target: PathBuf,
+        device: String,
+        mounted: bool,
+    }
+
+    impl ZramBlockMount {
+        fn reserve(target: &Path) -> Result<Self, Box<dyn Error + Send + Sync>> {
+            if !Path::new("/dev/zram-control").exists() {
+                run_command("modprobe", &["zram"])?;
+            }
+            let device = run_command_stdout("zramctl", &["--find", "--size", "256M"])?;
+            if device.is_empty() {
+                return Err("zramctl --find --size returned an empty device".into());
+            }
+
+            Ok(Self {
+                target: target.to_path_buf(),
+                device,
+                mounted: false,
+            })
+        }
+
+        fn mount_target(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let result = (|| {
+                run_command("mkfs.ext4", &["-F", &self.device])?;
+                let target_arg = path_to_string(&self.target, "zram replacement mount target")?;
+                run_command("mount", &[&self.device, &target_arg])
+            })();
+            if let Err(error) = result {
+                let _ = self.cleanup();
+                return Err(error);
+            }
+            self.mounted = true;
+            Ok(())
+        }
+
+        fn cleanup(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let mut first_error: Option<Box<dyn Error + Send + Sync>> = None;
+            if self.mounted {
+                if let Err(error) = detach_mount(&self.target) {
+                    first_error.get_or_insert(error);
+                } else {
+                    self.mounted = false;
+                }
+            }
+            if !self.device.is_empty() {
+                if let Err(error) = run_command("zramctl", &["--reset", &self.device]) {
+                    first_error.get_or_insert(error);
+                } else {
+                    self.device.clear();
+                }
+            }
+            if let Some(error) = first_error {
+                return Err(error);
+            }
+            Ok(())
+        }
+    }
+
+    impl Drop for ZramBlockMount {
+        fn drop(&mut self) {
+            let _ = self.cleanup();
+        }
+    }
+
     fn checked_command_output(program: &str, args: &[&str]) -> Result<std::process::Output, Box<dyn Error + Send + Sync>> {
         let output = Command::new(program).args(args).output()?;
         if output.status.success() {
@@ -298,6 +381,18 @@ mod tests {
         Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN; unshare exited with status {status}").into())
     }
 
+    fn replacement_node_log_path(
+        cluster_temp_dir: &str,
+        parity: usize,
+        node_index: usize,
+    ) -> Result<PathBuf, Box<dyn Error + Send + Sync>> {
+        let log_dir = std::env::var_os(LOG_DIR_ENV)
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from(cluster_temp_dir));
+        fs::create_dir_all(&log_dir)?;
+        Ok(log_dir.join(format!("replacement-ec{parity}-node{node_index}-{}.log", std::process::id())))
+    }
+
     fn payload(len: usize, seed: u8) -> Vec<u8> {
         let mut next = seed;
         (0..len)
@@ -472,8 +567,20 @@ mod tests {
             if let Some(version_id) = &version.version_id {
                 request = request.version_id(version_id);
             }
-            let response = request.send().await?;
-            let body = response.body.collect().await?.into_bytes();
+            let response = request.send().await.map_err(|error| {
+                format!("body GET failed for {}/{}@{:?}: {error}", version.bucket, version.key, version.version_id)
+            })?;
+            let body = response
+                .body
+                .collect()
+                .await
+                .map_err(|error| {
+                    format!(
+                        "body stream failed for {}/{}@{:?}: {error}",
+                        version.bucket, version.key, version.version_id
+                    )
+                })?
+                .into_bytes();
             assert_eq!(
                 sha256_hex(&body),
                 *expected_sha256,
@@ -582,81 +689,6 @@ mod tests {
         Ok(())
     }
 
-    fn log_tail(log: &str) -> String {
-        let mut lines = log.lines().rev().take(80).collect::<Vec<_>>();
-        lines.reverse();
-        lines.join("\n")
-    }
-
-    fn log_len(path: &Path) -> Result<u64, Box<dyn Error + Send + Sync>> {
-        match fs::metadata(path) {
-            Ok(metadata) => Ok(metadata.len()),
-            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0),
-            Err(error) => Err(format!("failed to stat target node log {path:?}: {error}").into()),
-        }
-    }
-
-    fn log_from_offset(path: &Path, offset: u64) -> Result<String, Box<dyn Error + Send + Sync>> {
-        let log = match fs::read(path) {
-            Ok(log) => log,
-            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
-            Err(error) => return Err(format!("failed to read target node log {path:?}: {error}").into()),
-        };
-        let start = usize::try_from(offset).unwrap_or(usize::MAX).min(log.len());
-        Ok(String::from_utf8_lossy(&log[start..]).into_owned())
-    }
-
-    fn live_disk_loss_scan_completed(log: &str, target_disk: &Path) -> bool {
-        let target = target_disk.to_string_lossy();
-        let mut saw_live_loss = false;
-        for line in log.lines() {
-            if line.contains("Heal auto-scan disk inspection failed")
-                && line.contains("check_failed")
-                && line.contains(target.as_ref())
-            {
-                saw_live_loss = true;
-                continue;
-            }
-            if saw_live_loss && (line.contains("Heal auto disk scanner idle") || line.contains("Heal auto-scan cycle completed"))
-            {
-                return true;
-            }
-        }
-        false
-    }
-
-    fn live_disk_loss_scan_completed_from_path(
-        log_path: &Path,
-        start_offset: u64,
-        target_disk: &Path,
-    ) -> Result<bool, Box<dyn Error + Send + Sync>> {
-        Ok(live_disk_loss_scan_completed(&log_from_offset(log_path, start_offset)?, target_disk))
-    }
-
-    async fn wait_for_live_disk_loss_observation(
-        log_path: &Path,
-        target_disk: &Path,
-        start_offset: u64,
-        timeout_secs: u64,
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let deadline = Instant::now() + Duration::from_secs(timeout_secs);
-        let mut tick = interval(Duration::from_secs(1));
-        loop {
-            if live_disk_loss_scan_completed_from_path(log_path, start_offset, target_disk)? {
-                return Ok(());
-            }
-            if Instant::now() >= deadline {
-                let log = log_from_offset(log_path, start_offset)?;
-                return Err(format!(
-                    "scanner did not finish a live target-loss scan for {target_disk:?} within {timeout_secs}s; log tail:\n{}",
-                    log_tail(&log)
-                )
-                .into());
-            }
-            tick.tick().await;
-        }
-    }
-
     fn cluster_status_is_definitive(status: &serde_json::Value) -> Result<bool, Box<dyn Error + Send + Sync>> {
         status["cluster"]["definitive"]
             .as_bool()
@@ -707,6 +739,13 @@ mod tests {
             .collect()
     }
 
+    fn is_transient_recovery_version_absence(error: &(dyn Error + 'static)) -> bool {
+        matches!(
+            error.downcast_ref::<rustfs_filemeta::Error>(),
+            Some(rustfs_filemeta::Error::FileVersionNotFound)
+        )
+    }
+
     fn incomplete_versions(
         target_disk: &Path,
         versions: &[BaselineVersion],
@@ -714,7 +753,21 @@ mod tests {
         let mut missing = BTreeSet::new();
         for version in versions {
             let actual =
-                census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref())?;
+                match census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref()) {
+                    Ok(actual) => actual,
+                    // During replacement recovery, xl.meta may arrive before this
+                    // particular historical version. The generic census helper
+                    // correctly reports that as an error; this progress poll must
+                    // instead wait for the version to be restored.
+                    Err(error) if is_transient_recovery_version_absence(error.as_ref()) => {
+                        missing.insert(format!(
+                            "{}/{}@{:?}: version metadata not yet present on replacement",
+                            version.bucket, version.key, version.version_id
+                        ));
+                        continue;
+                    }
+                    Err(error) => return Err(error),
+                };
             if !actual.matches_manifest(&version.expected) {
                 missing.insert(format!("{}/{}@{:?}: {actual:?}", version.bucket, version.key, version.version_id));
             }
@@ -822,13 +875,15 @@ mod tests {
 
         let mut mount_ns = MountNamespaceGuard::new()?;
         let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(3, 4)).await?;
-        let target_log_path = PathBuf::from(&cluster.temp_dir).join(format!("replacement-node{TARGET_NODE}.log"));
-        cluster.set_node_capture_log_path(TARGET_NODE, target_log_path.to_string_lossy())?;
+        for node_index in 0..cluster.nodes.len() {
+            let node_log_path = replacement_node_log_path(&cluster.temp_dir, parity, node_index)?;
+            cluster.set_node_capture_log_path(node_index, node_log_path.to_string_lossy())?;
+        }
         let target_disk = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dirs[TARGET_DRIVE]);
-        // Each drive below is an independent tmpfs mount, so this privileged
-        // path must exercise the production distinct-device/readiness fences.
+        // The blank target uses a temporary zram block device, so the
+        // replacement readiness fence sees no root or sibling alias.
         cluster.extra_env.retain(|(key, _)| key != "RUSTFS_UNSAFE_BYPASS_DISK_CHECK");
-        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-faultable-images");
+        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-block-images");
         let mut target_mount = None;
         for (node_index, node) in cluster.nodes.iter().enumerate() {
             for (drive_index, drive) in node.data_dirs.iter().enumerate() {
@@ -845,6 +900,7 @@ mod tests {
             }
         }
         let mut target_mount = target_mount.ok_or("target drive was not mounted with the faultable block fixture")?;
+        let mut replacement_mount = ZramBlockMount::reserve(&target_disk)?;
 
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
         cluster.set_env("RUSTFS_SCANNER_ENABLED", "true");
@@ -852,28 +908,34 @@ mod tests {
         cluster.set_env("RUSTFS_SCANNER_CYCLE", "1");
         cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
         cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
-        cluster.set_node_env(TARGET_NODE, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
+        for node_index in 0..cluster.nodes.len() {
+            cluster.set_node_env(node_index, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
+        }
         cluster.start().await?;
 
         let clients = cluster.create_all_clients()?;
-        let versions = seed_baseline(&clients[0], &target_disk).await?;
-        verify_bodies(&clients[0], &versions).await?;
+        let versions = seed_baseline(&clients[0], &target_disk)
+            .await
+            .map_err(|error| format!("pre-fault baseline seeding failed: {error}"))?;
+        verify_bodies(&clients[0], &versions)
+            .await
+            .map_err(|error| format!("pre-fault body verification failed: {error}"))?;
 
-        let live_loss_log_offset = log_len(&target_log_path)?;
-        target_mount.make_unavailable()?;
-        wait_for_live_disk_loss_observation(
-            &target_log_path,
-            &target_disk,
-            live_loss_log_offset,
-            ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS,
-        )
-        .await?;
-        assert_no_replacement_status_records(&cluster, &target_disk).await?;
-        assert_no_replacement_admission_artifacts(&cluster, &target_disk)?;
+        target_mount
+            .make_unavailable()
+            .map_err(|error| format!("failed to install the dm-error target: {error}"))?;
+        target_mount
+            .verify_raw_io_is_unavailable()
+            .map_err(|error| format!("dm-error target was not proven by a direct raw read: {error}"))?;
+        assert_no_replacement_status_records(&cluster, &target_disk)
+            .await
+            .map_err(|error| format!("live-fault replacement status check failed: {error}"))?;
+        assert_no_replacement_admission_artifacts(&cluster, &target_disk)
+            .map_err(|error| format!("live-fault replacement artifact check failed: {error}"))?;
 
-        cluster.stop_node(TARGET_NODE)?;
+        cluster.stop_node_gracefully(TARGET_NODE).await?;
         target_mount.cleanup()?;
-        mount_ns.mount_tmpfs(&target_disk, &format!("rustfs-e2e-p{parity}-replacement"))?;
+        replacement_mount.mount_target()?;
         let missing_before_restart = incomplete_versions(&target_disk, &versions)?;
         assert_eq!(
             missing_before_restart.len(),
@@ -882,46 +944,26 @@ mod tests {
         );
         cluster.start_node(TARGET_NODE).await?;
 
-        wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
-        verify_bodies(&clients[0], &versions).await?;
+        let recovery_result = async {
+            wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
+            verify_bodies(&clients[0], &versions).await
+        }
+        .await;
+        let stop_result = cluster.stop_node_gracefully(TARGET_NODE).await;
+        let replacement_cleanup_result = replacement_mount.cleanup();
 
-        Ok(())
-    }
+        if let Err(error) = recovery_result {
+            if let Err(stop_error) = stop_result {
+                info!(%stop_error, "replacement target stop failed while preserving recovery failure");
+            }
+            if let Err(cleanup_error) = replacement_cleanup_result {
+                info!(%cleanup_error, "replacement zram cleanup failed while preserving recovery failure");
+            }
+            return Err(error);
+        }
+        stop_result?;
+        replacement_cleanup_result?;
 
-    #[test]
-    fn live_loss_barrier_requires_scanner_failure_after_log_offset() -> Result<(), Box<dyn Error + Send + Sync>> {
-        let target = Path::new("/mnt/target");
-        assert!(live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto-scan cycle completed",
-            target
-        ));
-        assert!(live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "Heal auto disk scanner idle\nHeal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "event=disk_health_check_failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        assert!(!live_disk_loss_scan_completed(
-            "Heal auto-scan disk inspection failed endpoint=/mnt/other disk_state=check_failed\nHeal auto disk scanner idle",
-            target
-        ));
-        let path = std::env::temp_dir().join(format!("rustfs-replacement-scan-{}.log", std::process::id()));
-        let stale =
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
-        fs::write(&path, stale)?;
-        let offset = log_len(&path)?;
-        assert!(!live_disk_loss_scan_completed_from_path(&path, offset, target)?);
-        let fresh =
-            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
-        fs::write(&path, format!("{stale}{fresh}"))?;
-        assert!(live_disk_loss_scan_completed_from_path(&path, offset, target)?);
-        fs::remove_file(path)?;
         Ok(())
     }
 
@@ -954,6 +996,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn recovery_census_only_treats_missing_version_as_transient() {
+        let missing_version: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileVersionNotFound);
+        let missing_file: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileNotFound);
+
+        assert!(is_transient_recovery_version_absence(missing_version.as_ref()));
+        assert!(!is_transient_recovery_version_absence(missing_file.as_ref()));
+    }
+
     #[tokio::test]
     async fn completion_poll_samples_census_before_status() {
         let order = std::rc::Rc::new(std::cell::RefCell::new(Vec::new()));

crates/e2e_test/src/storage_api.rs

@@ -15,7 +15,7 @@
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
 #[cfg(test)]
-pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
+pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
 pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,14 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
 };
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::task::JoinSet;
+use tokio::time::{Instant, sleep};
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
@@ -28,6 +31,14 @@ const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
 const VERSIONED_BUCKET: &str = "upgrade-versioned-data";
+const MIXED_BUCKET: &str = "upgrade-mixed-version-data";
+const MIXED_NODE_COUNT: usize = 4;
+const MULTIPART_WORKERS: usize = 16;
+const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
+// Peers keep a restarted node's drive in Suspect/Returning for roughly
+// probe_interval (2s) x success_threshold (3) after it comes back; 30s
+// comfortably covers that window plus CI scheduling jitter.
+const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
@@ -103,6 +114,132 @@ async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<
     Ok(())
 }
 
+fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("mixed-upgrade-node-{node_idx}.log"));
+        cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
+    let mut tasks = JoinSet::new();
+    for worker in 0..MULTIPART_WORKERS {
+        let client = clients[worker % clients.len()].clone();
+        let phase = phase.to_string();
+        tasks.spawn(async move {
+            let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
+            for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
+                let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
+                let part = vec![u8::try_from(worker)?; 64 * 1024];
+                write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
+                keys.push(key);
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
+        });
+    }
+
+    let mut keys = Vec::with_capacity(MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER);
+    while let Some(result) = tasks.join_next().await {
+        keys.extend(result??);
+    }
+    Ok(keys)
+}
+
+/// Assert that `client` eventually lists exactly `expected` objects under
+/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
+///
+/// A single-snapshot assertion here is racy by construction: each phase both
+/// writes and lists within seconds of a node restart. While a peer still holds
+/// the restarted node's drive in Suspect/Returning, strict-quorum listing
+/// consults only the remaining three drives and drops any object that was
+/// itself legally written at write quorum (3/4 drives) during an earlier
+/// node's identical post-restart window — its xl.meta is then visible on only
+/// two of the three consulted drives, below the required object quorum of
+/// three. GET still succeeds for such objects; only the listing under-counts
+/// until drive health converges. A genuine upgrade data-loss regression still
+/// fails after the deadline.
+async fn wait_for_phase_listing(client: &Client, phase: &str, expected: usize, context: &str) -> TestResult {
+    let deadline = Instant::now() + LISTING_CONVERGENCE_TIMEOUT;
+    loop {
+        let listed = client
+            .list_objects_v2()
+            .bucket(MIXED_BUCKET)
+            .prefix(format!("{phase}/"))
+            .send()
+            .await?;
+        let count = listed.contents().len();
+        if count == expected {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(format!(
+                "{context}: listing under {phase}/ returned {count} of {expected} objects even after {}s of post-restart convergence",
+                LISTING_CONVERGENCE_TIMEOUT.as_secs()
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn exercise_mixed_cluster(
+    cluster: &RustFSTestClusterEnvironment,
+    phase: &str,
+    current_node: usize,
+    previous_node: usize,
+) -> TestResult {
+    let clients = cluster.create_all_clients()?;
+    let current_client = &clients[current_node];
+    let previous_client = &clients[previous_node];
+
+    let current_key = format!("{phase}/written-by-current");
+    let current_body = format!("{phase}: current RustFS build").into_bytes();
+    current_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&current_key)
+        .body(ByteStream::from(current_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(previous_client, MIXED_BUCKET, &current_key, None).await?.1, current_body);
+
+    let previous_key = format!("{phase}/written-by-previous");
+    let previous_body = format!("{phase}: previous RustFS release").into_bytes();
+    previous_client
+        .put_object()
+        .bucket(MIXED_BUCKET)
+        .key(&previous_key)
+        .body(ByteStream::from(previous_body.clone()))
+        .send()
+        .await?;
+    assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
+
+    let multipart_keys = write_multipart_load(&clients, phase).await?;
+    let expected_count = multipart_keys.len() + 2;
+    for (label, client) in [("current", current_client), ("previous", previous_client)] {
+        wait_for_phase_listing(
+            client,
+            phase,
+            expected_count,
+            &format!("the {label} RustFS version must stream the complete mixed-version listing"),
+        )
+        .await?;
+    }
+
+    let last_multipart_key = format!("{phase}/multipart/{:02}/{:02}", MULTIPART_WORKERS - 1, MULTIPART_UPLOADS_PER_WORKER - 1);
+    assert_eq!(
+        read_object(previous_client, MIXED_BUCKET, &last_multipart_key, None).await?.1,
+        vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
+    );
+
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -252,3 +389,43 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
 
     Ok(())
 }
+
+#[tokio::test]
+#[ignore = "requires a pinned previous RustFS release binary"]
+async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let current_binary = rustfs_binary_path();
+    let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
+    cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn");
+    configure_cluster_logs(&mut cluster)?;
+    cluster.start_with_binary(&previous_binary).await?;
+    cluster.create_test_bucket(MIXED_BUCKET).await?;
+
+    cluster.stop_node(0)?;
+    cluster.start_node_from_binary(0, &current_binary).await?;
+    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+
+    for node_idx in [1, 2] {
+        cluster.stop_node(node_idx)?;
+        cluster.start_node_from_binary(node_idx, &current_binary).await?;
+    }
+    exercise_mixed_cluster(&cluster, "one-previous-node", 0, 3).await?;
+
+    cluster.stop_node(3)?;
+    cluster.start_node_from_binary(3, &current_binary).await?;
+
+    for (node_idx, client) in cluster.create_all_clients()?.iter().enumerate() {
+        for phase in ["one-current-node", "one-previous-node"] {
+            wait_for_phase_listing(
+                client,
+                phase,
+                MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2,
+                &format!("node {node_idx}: the homogeneous current cluster must preserve every object"),
+            )
+            .await?;
+        }
+    }
+
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -75,6 +75,10 @@ pub mod bucket {
                 delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
                 inspect_transition_transaction_for_operator,
             };
+            #[cfg(feature = "test-util")]
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionTransactionRecoveryStats, recover_transition_transaction_records,
+            };
         }
 
         pub mod evaluator {
@@ -99,6 +103,17 @@ pub mod bucket {
             pub use crate::bucket::lifecycle::tier_delete_journal::{
                 persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
             };
+
+            #[cfg(feature = "test-util")]
+            pub mod test_util {
+                /// Model a single-node, all-v6 fleet after its capability probe has completed.
+                ///
+                /// Call this only once while constructing an isolated test store, before any
+                /// tier-delete journal permit or background worker can be active.
+                pub fn install_all_v6_fleet_capability_proof() {
+                    crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+                }
+            }
         }
 
         pub mod tier_last_day_stats {
@@ -425,6 +440,11 @@ pub mod object {
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
         SnapshotConsistencyError,
     };
+
+    #[cfg(feature = "test-util")]
+    pub mod test_util {
+        pub use crate::store::DeleteAfterObjectLockSnapshotBarrier;
+    }
 }
 
 pub mod rebalance {

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -485,6 +485,19 @@ impl BucketTargetSys {
         mutex
     }
 
+    /// Snapshot the heartbeat-tracked health of `url`'s endpoint.
+    ///
+    /// Returns `None` when the heartbeat has never seen the endpoint. Unlike
+    /// [`Self::is_offline`] this deliberately does not call `init_hc`: a caller
+    /// that only reports metrics must not create health entries as a side
+    /// effect, or merely rendering a status page would mark an unknown peer
+    /// online.
+    pub async fn endpoint_health(&self, url: &Url) -> Option<EpHealth> {
+        let key = endpoint_health_key(url);
+        let health_map = self.h_mutex.read().await;
+        health_map.get(&key).cloned()
+    }
+
     pub async fn is_offline(&self, url: &Url) -> bool {
         let key = endpoint_health_key(url);
         {

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
     DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
 };
 use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
-use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
+use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
 use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
 use crate::bucket::object_lock::ObjectLockApi;
 use crate::bucket::versioning::VersioningApi as _;
@@ -50,7 +50,10 @@ use crate::disk::error::DiskError;
 use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
 use crate::error::Error;
 use crate::error::StorageError;
-use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
+use crate::error::{
+    is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
+    is_network_or_host_down,
+};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
 use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
 use crate::services::tier::{
@@ -586,25 +589,217 @@ impl ExpiryOp for FreeVersionTask {
     }
 }
 
-async fn delete_free_version_remote_object(
+async fn acquire_free_version_tier_lease(
     oi: &ObjectInfo,
     tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
-) -> Result<(), std::io::Error> {
+) -> Result<(TierOperationLease, bool), std::io::Error> {
     let version_id_exact = validate_transition_remote_version(oi)?;
     let identity = tier_destination_id_from_metadata(&oi.user_defined)?
         .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
-    delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+    let lease =
+        TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
+            .await
+            .map_err(std::io::Error::other)?;
+    Ok((lease, version_id_exact))
+}
+
+async fn delete_free_version_remote_object_with_lease(
+    oi: &ObjectInfo,
+    lease: &TierOperationLease,
+    version_id_exact: bool,
+) -> Result<(), std::io::Error> {
+    delete_object_from_remote_tier_with_lease_idempotent(
         &oi.transitioned_object.name,
         &oi.transitioned_object.version_id,
-        &oi.transitioned_object.tier,
-        identity,
-        tier_config_mgr,
+        lease,
         version_id_exact,
     )
     .await?;
     Ok(())
 }
 
+fn free_version_physical_topology_generation(api: &ECStore) -> String {
+    let mut hasher = Sha256::new();
+    for pool in &api.pools {
+        hasher.update(pool.pool_idx.to_be_bytes());
+        hasher.update(pool.disk_set.len().to_be_bytes());
+        for set in &pool.disk_set {
+            hasher.update(set.set_index.to_be_bytes());
+        }
+    }
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
+    if candidate.transitioned_object.tier != expected.transitioned_object.tier
+        || candidate.transitioned_object.name != expected.transitioned_object.name
+    {
+        return Ok(false);
+    }
+    let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
+    let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
+    if candidate_identity != expected_identity {
+        return Ok(false);
+    }
+    if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+        || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
+    {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version remote version state is unknown",
+        ));
+    }
+    Ok(candidate.transition_version_state == expected.transition_version_state
+        && candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
+}
+
+async fn scan_exact_free_version_targets(
+    api: &ECStore,
+    oi: &ObjectInfo,
+    local_object: &str,
+) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
+    let mut targets = Vec::new();
+    for pool in &api.pools {
+        for set in &pool.disk_set {
+            let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
+                Ok(Some(versions)) => versions,
+                Ok(None) => continue,
+                Err(err) if is_err_strict_volume_not_found(&err) => continue,
+                Err(err) => return Err(std::io::Error::other(err)),
+            };
+            for version in versions.versions.iter().chain(versions.free_versions.iter()) {
+                let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
+                if free_version_remote_tuple_matches(&candidate, oi)? {
+                    if candidate.transitioned_object.free_version {
+                        // Data movement can leave the same remote tuple in
+                        // several physical pools. Ordinary deletion assigns a
+                        // fresh local free-version UUID to each copy, but all
+                        // of those markers own the same idempotent remote
+                        // DELETE. Consume them together while holding every
+                        // physical object lock; treating their local UUIDs as
+                        // conflicting would strand cleanup forever.
+                        let mut actual = version.clone();
+                        actual.name = local_object.to_string();
+                        targets.push((Arc::clone(set), actual));
+                    } else {
+                        return Err(std::io::Error::new(
+                            std::io::ErrorKind::WouldBlock,
+                            "a live transitioned source still references the free-version remote tuple",
+                        ));
+                    }
+                }
+            }
+        }
+    }
+    Ok(targets)
+}
+
+fn free_version_cleanup_fences_current(
+    topology_generation: &str,
+    api: &ECStore,
+    bucket_guard: &rustfs_lock::NamespaceLockGuard,
+    object_guards: &[crate::store::ObjectLockDiagGuard],
+    lease: &TierOperationLease,
+    cancel: &CancellationToken,
+    deadline: tokio::time::Instant,
+) -> bool {
+    !cancel.is_cancelled()
+        && tokio::time::Instant::now() < deadline
+        && !bucket_guard.is_lock_lost()
+        && object_guards.iter().all(|guard| !guard.is_lock_lost())
+        && lease.is_current_generation()
+        && free_version_physical_topology_generation(api) == topology_generation
+}
+
+async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
+    const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
+
+    let topology_generation = free_version_physical_topology_generation(&api);
+    let bucket_guard = api
+        .acquire_bucket_lifecycle_read_lock(&oi.bucket)
+        .await
+        .map_err(std::io::Error::other)?;
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
+    let local_object = encode_dir_object(&oi.name);
+    let object_guards = api
+        .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
+        .await
+        .map_err(std::io::Error::other)?;
+    let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if targets.is_empty() {
+        return Ok(false);
+    }
+
+    let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence is invalid before remote delete",
+        ));
+    }
+    tokio::select! {
+        _ = cancel.cancelled() => {
+            return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
+        }
+        result = tokio::time::timeout_at(
+            deadline,
+            delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
+        ) => {
+            result
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
+        }
+    }
+    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
+        // Remote DELETE is idempotent, but a changed fence makes the local
+        // outcome ambiguous. Keep every marker for a fully fenced retry.
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::WouldBlock,
+            "tier free-version cleanup fence changed after remote delete",
+        ));
+    }
+
+    let mut first_error = None;
+    for (set, actual) in &targets {
+        let mut delete_request = FileInfo {
+            name: local_object.clone(),
+            version_id: actual.version_id,
+            ..Default::default()
+        };
+        delete_request.set_tier_free_version();
+        if let Err(err) = set
+            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
+            .await
+            && first_error.is_none()
+        {
+            first_error = Some(std::io::Error::other(err));
+        }
+    }
+    let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
+    if !remaining.is_empty() {
+        return Err(first_error.unwrap_or_else(|| {
+            std::io::Error::new(
+                std::io::ErrorKind::WouldBlock,
+                "tier free-version cleanup remained on at least one physical set",
+            )
+        }));
+    }
+    if let Some(err) = first_error {
+        return Err(err);
+    }
+    Ok(true)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn delete_free_version_remote_object(
+    oi: &ObjectInfo,
+    tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
+) -> Result<(), std::io::Error> {
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
+}
+
 #[allow(
     dead_code,
     reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -618,8 +813,11 @@ where
     F: FnOnce() -> Fut,
     Fut: std::future::Future<Output = T>,
 {
-    delete_free_version_remote_object(oi, tier_config_mgr).await?;
-    Ok(delete_local().await)
+    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
+    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
+    let result = delete_local().await;
+    drop(lease);
+    Ok(result)
 }
 
 struct NewerNoncurrentTask {
@@ -690,6 +888,10 @@ impl ExpiryState {
         usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
     }
 
+    pub fn active_tasks(&self) -> usize {
+        usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
+    }
+
     fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
         let queued = wrkr.try_send(Some(task)).is_ok();
         if queued {
@@ -719,7 +921,7 @@ impl ExpiryState {
         Ok(())
     }
 
-    pub fn enqueue_free_version(&mut self, oi: ObjectInfo) -> bool {
+    pub fn enqueue_free_version(&self, oi: ObjectInfo) -> bool {
         let task = FreeVersionTask(oi);
         let wrkr = self.get_worker_ch(task.op_hash());
         if wrkr.is_none() {
@@ -826,7 +1028,7 @@ impl ExpiryState {
     }
 
     pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let expiry_state = api.ctx.expiry_state();
         if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
             return;
         }
@@ -867,7 +1069,7 @@ impl ExpiryState {
         stats: Arc<ExpiryStats>,
         recovery_notify: Arc<Notify>,
     ) {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
             static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
             FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
         });
@@ -968,119 +1170,33 @@ impl ExpiryState {
                     else if v.as_any().is::<FreeVersionTask>() {
                         let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
                         let oi = v.0.clone();
-                        if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
-                            recovery_notify.notify_one();
-                            debug!(
-                                bucket = %oi.bucket,
-                                object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
-                                error = ?err,
-                                event = EVENT_LIFECYCLE_WORKER_STATE,
-                                component = LOG_COMPONENT_ECSTORE,
-                                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                reason = "remote_tier_delete_failed",
-                                "Lifecycle worker skipped remote tier delete"
-                            );
-                            continue;
-                        }
-
-                        let local_object = encode_dir_object(&oi.name);
-                        let mut fi = FileInfo {
-                            name: local_object.clone(),
-                            version_id: oi.version_id,
-                            ..Default::default()
-                        };
-                        // This removes an existing internal cleanup marker. Keeping
-                        // `deleted` false makes duplicate tasks return not-found
-                        // instead of creating an ordinary delete marker.
-                        fi.set_tier_free_version();
-
-                        let mut deleted_locally = false;
-                        for pool in &api.pools {
-                            let set = pool.get_disks_by_key(&local_object);
-                            let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
-                                Ok(lock) => lock,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        pool_index = pool.pool_idx,
-                                        set_index = set.set_index,
-                                        error = ?err,
-                                        reason = "local_free_version_lock_failed",
-                                        "Lifecycle worker failed to create local free-version cleanup lock"
-                                    );
-                                    continue;
-                                }
-                            };
-                            let _object_lock_guard =
-                                match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
-                                    Ok(guard) => guard,
-                                    Err(err) => {
-                                        recovery_notify.notify_one();
-                                        debug!(
-                                            event = EVENT_LIFECYCLE_WORKER_STATE,
-                                            component = LOG_COMPONENT_ECSTORE,
-                                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                            bucket = %oi.bucket,
-                                            object = %oi.name,
-                                            pool_index = pool.pool_idx,
-                                            set_index = set.set_index,
-                                            error = ?err,
-                                            reason = "local_free_version_lock_failed",
-                                            "Lifecycle worker failed to acquire local free-version cleanup lock"
-                                        );
-                                        continue;
-                                    }
-                                };
-                            match set
-                                .delete_object_version(&oi.bucket, &local_object, &fi, false)
-                                .await
-                            {
-                                Ok(()) => {
-                                    deleted_locally = true;
-                                    break;
-                                }
-                                Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
-                                Err(err) => {
-                                    recovery_notify.notify_one();
-                                    debug!(
-                                        event = EVENT_LIFECYCLE_WORKER_STATE,
-                                        component = LOG_COMPONENT_ECSTORE,
-                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                        bucket = %oi.bucket,
-                                        object = %oi.name,
-                                        remote_object = %oi.transitioned_object.name,
-                                        remote_version_id = %oi.transitioned_object.version_id,
-                                        tier = %oi.transitioned_object.tier,
-                                        error = ?err,
-                                        reason = "local_free_version_delete_failed",
-                                        "Lifecycle worker failed local free-version cleanup"
-                                    );
-                                    break;
-                                }
-                            }
-                        }
-
-                        if !deleted_locally {
-                            debug!(
+                        match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
+                            Ok(true) => {}
+                            Ok(false) => debug!(
                                 event = EVENT_LIFECYCLE_WORKER_STATE,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
                                 bucket = %oi.bucket,
                                 object = %oi.name,
-                                remote_object = %oi.transitioned_object.name,
-                                remote_version_id = %oi.transitioned_object.version_id,
-                                tier = %oi.transitioned_object.tier,
                                 reason = "local_free_version_missing",
-                                "Lifecycle worker could not find transitioned free version locally"
-                            );
+                                "Lifecycle worker found that the exact free-version was already absent"
+                            ),
+                            Err(err) => {
+                                recovery_notify.notify_one();
+                                debug!(
+                                    event = EVENT_LIFECYCLE_WORKER_STATE,
+                                    component = LOG_COMPONENT_ECSTORE,
+                                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                    bucket = %oi.bucket,
+                                    object = %oi.name,
+                                    remote_object = %oi.transitioned_object.name,
+                                    remote_version_id = %oi.transitioned_object.version_id,
+                                    tier = %oi.transitioned_object.tier,
+                                    error = ?err,
+                                    reason = "free_version_exact_cleanup_deferred",
+                                    "Lifecycle worker retained the exact free-version for a fenced retry"
+                                );
+                            }
                         }
                     }
                     else {
@@ -1099,6 +1215,22 @@ impl ExpiryState {
     }
 }
 
+pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions: Vec<ObjectInfo>) -> usize {
+    if free_versions.is_empty() {
+        return 0;
+    }
+
+    let expiry_state = api.ctx.expiry_state();
+    let state = expiry_state.read().await;
+    let mut queued = 0;
+    for free_version in free_versions {
+        if state.enqueue_free_version(free_version) {
+            queued += 1;
+        }
+    }
+    queued
+}
+
 async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
     let task = FreeVersionTask(oi);
     let hash = task.op_hash();
@@ -1152,8 +1284,8 @@ fn set_recovered_free_version_enqueue_observer(
     RecoveredFreeVersionEnqueueObserverGuard
 }
 
-pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
-    let expiry_state = runtime_sources::expiry_state_handle();
+pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
+    let expiry_state = api.ctx.expiry_state();
     let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
 
     #[cfg(test)]
@@ -2580,8 +2712,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
     }
 
     Some(tokio::spawn(async move {
-        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
-        let expiry_state = runtime_sources::expiry_state_handle();
+        let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
+        let expiry_state = api.ctx.expiry_state();
         run_tier_free_version_recovery_loop(
             cancel_token,
             expiry_state,
@@ -6229,9 +6361,18 @@ mod tests {
             rustfs_utils::crypto::hex(old_identity),
         );
         oi.user_defined = Arc::new(metadata.clone());
+        let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
         delete_free_version_remote_object_then(&oi, &manager, {
             let local_delete_calls = Arc::clone(&local_delete_calls);
+            let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
+            let manager = manager.clone();
             move || async move {
+                assert_eq!(
+                    crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+                    1,
+                    "the identity-bound tier lease must span the exact local marker delete"
+                );
+                lease_observed_during_local_delete.store(true, Ordering::Relaxed);
                 local_delete_calls.fetch_add(1, Ordering::Relaxed);
             }
         })
@@ -6239,6 +6380,12 @@ mod tests {
         .expect("matching destination identity should allow idempotent remote cleanup");
         assert_eq!(old_backend.remove_count().await, 1);
         assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
+        assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
+        assert_eq!(
+            crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
+            0,
+            "the tier lease should be released after the local marker delete completes"
+        );
 
         let mut single_prefix_metadata = HashMap::new();
         single_prefix_metadata.insert(
@@ -6472,6 +6619,7 @@ mod tests {
         let state = ExpiryState::new();
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6480,6 +6628,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         let err = state
@@ -6494,7 +6643,7 @@ mod tests {
     async fn enqueue_free_version_reports_false_without_worker_channel() {
         let state = ExpiryState::new();
         let recovery_notify = Arc::clone(&state.read().await.recovery_notify);
-        let mut state = state.write().await;
+        let state = state.write().await;
         let oi = ObjectInfo {
             bucket: "bucket".to_string(),
             name: "object".to_string(),
@@ -6620,6 +6769,7 @@ mod tests {
         let state = ExpiryState::new_with_unconsumed_worker_channel(1);
         let mut state = state.write().await;
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6628,6 +6778,7 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
         state
@@ -6684,7 +6835,7 @@ mod tests {
             },
             ..Default::default()
         };
-        let mut state = state.write().await;
+        let state = state.write().await;
 
         assert!(state.enqueue_free_version(oi.clone()));
         assert!(recovery_notify.notified().now_or_never().is_none());
@@ -6759,7 +6910,7 @@ mod tests {
         };
 
         assert!(
-            super::enqueue_recovered_free_version(oi).await,
+            super::enqueue_recovered_free_version(&ecstore, oi).await,
             "the resized production worker queue should accept the task"
         );
         stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -6875,12 +7026,12 @@ mod tests {
             .await
             .expect("free-version task should reach the worker");
         tokio::time::timeout(StdDuration::from_secs(30), async {
-            while remote_backend.remove_count().await == 0 {
+            while stats.active_tasks() == 0 {
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("worker should complete remote cleanup before taking the local lock");
+        .expect("worker should mark the cleanup task active before the lock assertion");
         let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
             while stats.active_tasks() != 0 {
                 tokio::task::yield_now().await;
@@ -6889,7 +7040,12 @@ mod tests {
         .await;
         assert!(
             completed_while_locked.is_err(),
-            "local cleanup must wait while a competing object writer owns the namespace lock"
+            "the cleanup task must wait while a competing object writer owns the namespace lock"
+        );
+        assert_eq!(
+            remote_backend.remove_count().await,
+            0,
+            "the remote tuple must not be deleted before the all-physical namespace fence is acquired"
         );
         for disk_path in &disk_paths {
             assert!(
@@ -6900,6 +7056,13 @@ mod tests {
         }
 
         drop(object_lock_guard);
+        tokio::time::timeout(StdDuration::from_secs(30), async {
+            while remote_backend.remove_count().await == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("worker should delete the remote tuple after acquiring the released namespace fence");
         tx.send(None).await.expect("worker stop signal should be delivered");
         worker.await.expect("free-version worker should stop cleanly");
 
@@ -6995,6 +7158,7 @@ mod tests {
             .next()
             .expect("seeded free version should be recoverable");
         let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
+        let ordinary_marker_mod_time = OffsetDateTime::now_utc();
 
         for disk_path in &disk_paths {
             let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7017,7 +7181,7 @@ mod tests {
                     name: object.to_string(),
                     version_id: Some(stale_version_id),
                     deleted: true,
-                    mod_time: Some(OffsetDateTime::now_utc()),
+                    mod_time: Some(ordinary_marker_mod_time),
                     ..Default::default()
                 })
                 .expect("same-ID ordinary marker should replace the stale free version");
@@ -7031,6 +7195,13 @@ mod tests {
             .expect("same-ID ordinary marker metadata should be written");
         }
 
+        assert!(
+            !super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
+                .await
+                .expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
+            "the stale free-version task must not report local cleanup"
+        );
+
         let state = ExpiryState::new();
         let (stats, recovery_notify) = {
             let state = state.read().await;
@@ -11522,7 +11693,7 @@ mod tests {
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
+    async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11530,6 +11701,7 @@ mod tests {
             .expect("mock tier lease should be available")
             .backend_identity();
         let je = Jentry {
+            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "legacy-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -11538,25 +11710,28 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Unknown,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
+            dispatch: None,
         };
 
+        crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
+            .await
+            .expect("legacy unknown journal should remain byte-compatible and persistable");
         let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
             .await
-            .expect_err("unknown journal state must fail before backend IO");
+            .expect_err("legacy unknown journal must be quarantined before backend IO");
 
-        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
         assert_eq!(backend.remove_count().await, 0);
     }
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_deletes_confirmed_exact_provider_token() {
+    async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
             .await
             .expect("mock tier lease should be available");
-        let identity = lease.backend_identity();
         backend
             .set_put_remote_version(Some("provider-version-token".to_string()))
             .await;
@@ -11570,34 +11745,30 @@ mod tests {
             .expect("confirmed remote candidate should be seeded");
         backend.set_remove_failure(true);
         backend.set_reject_non_empty_remote_versions(true);
-        let je = Jentry {
-            obj_name: "remote/object".to_string(),
-            version_id: "provider-version-token".to_string(),
-            tier_name: "WARM".to_string(),
-            backend_identity: Some(identity),
-            version_id_exact: true,
-            version_state: rustfs_filemeta::TransitionVersionState::Exact,
-            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
-            source: None,
-        };
-
-        crate::set_disk::cleanup_rejected_transition_upload_durably(
+        let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
             &lease,
-            &je.obj_name,
-            &je.version_id,
+            "remote/object",
+            "provider-version-token",
             true,
             Some(ecstore.clone()),
         )
         .await
-        .expect("failed immediate cleanup should remain durable in the journal");
-        assert!(backend.contains(&je.obj_name).await);
+        .expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
+        assert_eq!(err.kind(), std::io::ErrorKind::Other);
+        assert!(backend.contains("remote/object").await);
 
         backend.set_remove_failure(false);
-        crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
-            .await
-            .expect("identity-bound exact journal must retry confirmed candidate cleanup");
+        crate::set_disk::cleanup_rejected_transition_upload_durably(
+            &lease,
+            "remote/object",
+            "provider-version-token",
+            true,
+            Some(ecstore),
+        )
+        .await
+        .expect("the transaction retry must delete the same confirmed candidate");
 
-        assert!(!backend.contains(&je.obj_name).await);
+        assert!(!backend.contains("remote/object").await);
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(
             backend.remove_versions().await,
@@ -11760,11 +11931,14 @@ mod tests {
         };
         let mut recovery_rx = recovery_rx.lock().await;
         assert!(
-            super::enqueue_recovered_free_version(ObjectInfo {
-                bucket: "prefill".to_string(),
-                name: "prefill".to_string(),
-                ..Default::default()
-            })
+            super::enqueue_recovered_free_version(
+                &ecstore,
+                ObjectInfo {
+                    bucket: "prefill".to_string(),
+                    name: "prefill".to_string(),
+                    ..Default::default()
+                },
+            )
             .await,
             "the production recovery queue should accept its first task"
         );

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -33,6 +33,7 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum DurableIlmRecordKind {
     TierDeleteJournal,
+    TierDeleteDispatchManifest,
     TransitionTransaction,
     ManualTransitionJob,
     ManualTransitionScope,
@@ -54,6 +55,18 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
     max_record_size: 64 * 1024,
     kind: DurableIlmRecordKind::TierDeleteJournal,
 };
+pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-journal-v6",
+    prefix: "ilm/tier-delete-journal-v6/",
+    max_record_size: 64 * 1024,
+    kind: DurableIlmRecordKind::TierDeleteJournal,
+};
+pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
+    name: "tier-delete-dispatch-manifest",
+    prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
+    max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
+    kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
+};
 pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
     name: "transition-transaction",
     prefix: "ilm/transition-transactions/records",
@@ -85,8 +98,10 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
+    TIER_DELETE_JOURNAL_V6_NAMESPACE,
+    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
     TRANSITION_TRANSACTION_NAMESPACE,
     MANUAL_TRANSITION_JOB_NAMESPACE,
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -157,6 +172,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         content_sha256: String,
         identity_sha256: String,
         committed: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        dispatch_identity_sha256: Option<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        state: Option<super::tier_sweeper::TierDeleteJournalState>,
+    },
+    TierDeleteDispatchManifest {
+        content_sha256: String,
+        identity_sha256: String,
+        state: tier_delete_journal::TierDeleteDispatchManifestState,
     },
     TransitionTransaction {
         content_sha256: String,
@@ -195,6 +219,7 @@ impl DurableIlmRecordCheckpoint {
     pub(crate) fn content_sha256(&self) -> &str {
         match self {
             Self::TierDeleteJournal { content_sha256, .. }
+            | Self::TierDeleteDispatchManifest { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
@@ -228,6 +253,19 @@ impl DurableIlmRecordCheckpoint {
     }
 
     pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
+        for checkpoint in [self, next] {
+            if let Self::TierDeleteJournal {
+                committed,
+                dispatch_identity_sha256,
+                state,
+                ..
+            } = checkpoint
+                && (state.is_some() != dispatch_identity_sha256.is_some()
+                    || state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
+            {
+                return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
+            }
+        }
         if self == next {
             if let Self::ManualTransitionJob {
                 progress,
@@ -244,18 +282,64 @@ impl DurableIlmRecordCheckpoint {
         let valid = match (self, next) {
             (
                 Self::TierDeleteJournal {
+                    content_sha256: previous_content,
                     identity_sha256: previous_identity,
                     committed: previous_committed,
+                    dispatch_identity_sha256: previous_dispatch_identity,
+                    state: previous_state,
                     ..
                 },
                 Self::TierDeleteJournal {
+                    content_sha256: next_content,
                     identity_sha256: next_identity,
                     committed: next_committed,
+                    dispatch_identity_sha256: next_dispatch_identity,
+                    state: next_state,
                     ..
                 },
             ) => {
+                use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+                let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
+                    (Some(previous), Some(next)) => previous == next,
+                    (None, None) => true,
+                    // Old receipts did not record the v6 dispatch binding. A
+                    // byte-identical observation may adopt the stronger proof,
+                    // but an in-flight mutation must fail closed instead of
+                    // guessing which operation owned the journal.
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                let state_is_monotonic = match (previous_state, next_state) {
+                    (Some(previous), Some(next)) => {
+                        previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
+                    }
+                    (None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
+                    (None, Some(_)) => previous_content == next_content,
+                    (Some(_), None) => false,
+                };
+                previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
+            }
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: previous_state,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: next_identity,
+                    state: next_state,
+                    ..
+                },
+            ) => {
+                use tier_delete_journal::TierDeleteDispatchManifestState::{
+                    Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
+                };
                 previous_identity == next_identity
-                    && (previous_committed == next_committed || (!previous_committed && *next_committed))
+                    && matches!(
+                        (previous_state, next_state),
+                        (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
+                    )
             }
             (
                 Self::TransitionTransaction {
@@ -351,6 +435,49 @@ impl DurableIlmRecordCheckpoint {
             Err(Error::other("durable ILM record generation is not a monotonic successor"))
         }
     }
+
+    /// Whether `self` is an older generation of the same immutable record
+    /// that can reach `terminal` through one or more valid state transitions.
+    /// This is deliberately broader than `validate_successor`, which remains
+    /// adjacent-only for receipt advancement. Terminal cleanup uses this only
+    /// after the exact terminal ETag and terminal receipt were committed, to
+    /// purge older object versions exposed by that deletion.
+    pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
+        if self == terminal || self.validate_successor(terminal).is_ok() {
+            return true;
+        }
+        match (self, terminal) {
+            (
+                Self::TierDeleteJournal {
+                    identity_sha256: previous_identity,
+                    dispatch_identity_sha256: previous_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
+                    ..
+                },
+                Self::TierDeleteJournal {
+                    identity_sha256: terminal_identity,
+                    dispatch_identity_sha256: terminal_dispatch,
+                    state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
+                    ..
+                },
+            ) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
+            (
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: previous_identity,
+                    state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
+                    ..
+                },
+                Self::TierDeleteDispatchManifest {
+                    identity_sha256: terminal_identity,
+                    state:
+                        tier_delete_journal::TierDeleteDispatchManifestState::Aborted
+                        | tier_delete_journal::TierDeleteDispatchManifestState::Completed,
+                    ..
+                },
+            ) => previous_identity == terminal_identity,
+            _ => false,
+        }
+    }
 }
 
 fn transition_state_distance(
@@ -750,10 +877,19 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
             if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
                 return Err(Error::other("tier delete journal content does not match its path"));
             }
-            let operation_id = path
+            let legacy_operation_id = path
                 .strip_prefix(namespace.prefix)
                 .and_then(|suffix| suffix.strip_suffix(".json"))
                 .ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
+            // Legacy v1-v5 paths already expose a 64-hex operation id and
+            // must remain receipt-compatible. V6 uses an operation-scoped
+            // nested path, so derive a fixed, path-unique receipt id instead
+            // of embedding slashes in the receipt locator.
+            let operation_id = if entry.persisted_version == 6 {
+                hex_sha256(path.as_bytes(), ToOwned::to_owned)
+            } else {
+                legacy_operation_id.to_string()
+            };
             let identity_sha256 = checkpoint_hash(&(
                 &entry.obj_name,
                 &entry.version_id,
@@ -763,13 +899,29 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 entry.version_state,
                 &entry.source,
             ))?;
+            let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
             (
                 "operation_id",
-                operation_id.to_string(),
+                operation_id,
                 DurableIlmRecordCheckpoint::TierDeleteJournal {
                     content_sha256,
                     identity_sha256,
                     committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
+                    dispatch_identity_sha256,
+                    state: (entry.persisted_version == 6).then_some(entry.state),
+                },
+            )
+        }
+        DurableIlmRecordKind::TierDeleteDispatchManifest => {
+            let (operation_id, identity_sha256, state) =
+                tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
+            (
+                "operation_id",
+                hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
+                DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
+                    content_sha256,
+                    identity_sha256,
+                    state,
                 },
             )
         }
@@ -956,6 +1108,87 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
+        use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
+
+        let operation_id = Uuid::new_v4();
+        let checkpoint = |state| {
+            let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
+            let namespace = classify_durable_ilm_record(&path)
+                .expect("dispatch manifest namespace should classify")
+                .expect("dispatch manifest should be durable");
+            assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
+            validate_durable_ilm_record(&path, &data)
+                .expect("dispatch manifest should validate")
+                .checkpoint
+        };
+
+        let preparing = checkpoint(Preparing);
+        let authorized = checkpoint(DispatchAuthorized);
+        let completed = checkpoint(Completed);
+        let aborting = checkpoint(Aborting);
+        let aborted = checkpoint(Aborted);
+
+        preparing
+            .validate_successor(&authorized)
+            .expect("Preparing may become DispatchAuthorized");
+        authorized
+            .validate_successor(&completed)
+            .expect("DispatchAuthorized may become Completed");
+        preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
+        aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
+        assert!(authorized.validate_successor(&aborting).is_err());
+        assert!(completed.validate_successor(&authorized).is_err());
+        assert!(aborted.validate_successor(&preparing).is_err());
+    }
+
+    #[test]
+    fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
+        use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
+
+        let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: content.repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: state == Some(Committed),
+            dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
+            state,
+        };
+        let prepared = checkpoint("a", Some("d"), Some(Prepared));
+        let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
+        let committed = checkpoint("c", Some("d"), Some(Committed));
+        prepared
+            .validate_successor(&dispatched)
+            .expect("Prepared may advance to Dispatched");
+        dispatched
+            .validate_successor(&committed)
+            .expect("Dispatched may advance to Committed");
+        assert!(prepared.validate_successor(&committed).is_err());
+        assert!(dispatched.validate_successor(&prepared).is_err());
+
+        let rebound = checkpoint("b", Some("e"), Some(Dispatched));
+        assert!(dispatched.validate_successor(&rebound).is_err());
+
+        let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
+            "kind": "tier_delete_journal",
+            "content_sha256": "a".repeat(64),
+            "identity_sha256": "i".repeat(64),
+            "committed": false
+        }))
+        .expect("legacy tier-delete checkpoint should remain decodable");
+        legacy
+            .validate_successor(&prepared)
+            .expect("byte-identical legacy receipt may adopt the stronger v6 proof");
+        let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
+            content_sha256: "z".repeat(64),
+            identity_sha256: "i".repeat(64),
+            committed: false,
+            dispatch_identity_sha256: None,
+            state: None,
+        };
+        assert!(changed_legacy.validate_successor(&prepared).is_err());
+    }
+
     #[test]
     fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
         let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -34,6 +34,6 @@ pub mod tier_sweeper;
 pub mod transition_transaction;
 
 pub(crate) use durable_namespace::{
-    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
-    validate_durable_ilm_record,
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
+    classify_durable_ilm_record, validate_durable_ilm_record,
 };

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -172,7 +172,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
         return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
     }
 
-    let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
+    let page =
+        list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
     let mut stats = FreeVersionRecoveryStats {
         scanned: 0,
         enqueued: 0,
@@ -190,7 +191,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
             return Err(tier_free_version_recovery_cancelled());
         }
         retry_cursor.visit(&oi);
-        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
+        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
             let (bucket_marker, object_marker) = retry_cursor.retry_markers();
             stats.truncated = true;
             stats.next_bucket_marker = bucket_marker;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -255,6 +255,7 @@ impl ObjSweeper {
         }
         if del_tier {
             return Some(Jentry {
+                persisted_version: 0,
                 obj_name: self.remote_object.clone(),
                 version_id: self.transition_version_id.clone(),
                 tier_name: self.transition_tier.clone(),
@@ -266,6 +267,7 @@ impl ObjSweeper {
                 version_state: self.transition_version_state,
                 state: TierDeleteJournalState::Committed,
                 source: None,
+                dispatch: None,
             });
         }
         None
@@ -298,9 +300,19 @@ impl ObjSweeper {
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 pub(crate) enum TierDeleteJournalState {
     Prepared,
+    Dispatched,
     Committed,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct TierDeleteDispatchBinding {
+    pub(crate) operation_id: Uuid,
+    pub(crate) manifest_object: String,
+    pub(crate) journal_set_sha256: String,
+    pub(crate) topology_generation: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TierDeleteSourceIdentity {
@@ -342,6 +354,10 @@ impl TierDeleteSourceIdentity {
 #[derive(Debug, Clone)]
 #[allow(unused_assignments)]
 pub struct Jentry {
+    /// On-disk format version when decoded. Newly constructed entries use 0;
+    /// the encoder chooses their format from the durable ownership fields.
+    /// Recovery uses this value to quarantine v1-v5 without rewriting them.
+    pub(crate) persisted_version: u8,
     pub(crate) obj_name: String,
     pub(crate) version_id: String,
     pub(crate) tier_name: String,
@@ -350,6 +366,23 @@ pub struct Jentry {
     pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
     pub(crate) state: TierDeleteJournalState,
     pub(crate) source: Option<TierDeleteSourceIdentity>,
+    pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
+}
+
+impl Jentry {
+    /// Whether this prepared transaction is eligible to become the sole
+    /// cleanup owner for its transitioned source. The caller may use this to
+    /// decide whether to persist it, but must not set `skip_free_version`
+    /// until persistence succeeds.
+    pub(crate) fn can_replace_tier_free_version(&self) -> bool {
+        self.state == TierDeleteJournalState::Prepared
+            && self.backend_identity.is_some()
+            && self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
+            && self
+                .source
+                .as_ref()
+                .is_some_and(TierDeleteSourceIdentity::has_stable_identity)
+    }
 }
 
 impl ExpiryOp for Jentry {
@@ -617,6 +650,7 @@ pub fn transitioned_force_delete_journal_entry(
     }
 
     Some(Jentry {
+        persisted_version: 0,
         obj_name: transitioned.name.clone(),
         version_id: transitioned.version_id.clone(),
         tier_name: transitioned.tier.clone(),
@@ -628,6 +662,7 @@ pub fn transitioned_force_delete_journal_entry(
         version_state: transition_version_state,
         state: TierDeleteJournalState::Committed,
         source: None,
+        dispatch: None,
     })
 }
 
@@ -673,17 +708,73 @@ mod test {
     use rustfs_s3_client::signer_error::invalid_utf8_header_error;
 
     use super::{
-        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
-        RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
-        delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
-        is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
-        should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
+        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
+        RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
+        delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
+        delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
+        is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
+        transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
     };
     use crate::storage_api_contracts::lifecycle::TransitionedObject;
     use rustfs_filemeta::TransitionVersionState;
     use std::io::{Error, ErrorKind};
     use std::time::{Duration, Instant};
 
+    fn stable_prepared_journal() -> Jentry {
+        Jentry {
+            persisted_version: 0,
+            obj_name: "remote/object".to_string(),
+            version_id: "remote-version".to_string(),
+            tier_name: "WARM".to_string(),
+            backend_identity: Some([7; 32]),
+            version_id_exact: true,
+            version_state: TransitionVersionState::Exact,
+            state: TierDeleteJournalState::Prepared,
+            source: Some(TierDeleteSourceIdentity {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: Some(uuid::Uuid::new_v4().to_string()),
+                versioned: true,
+                version_suspended: false,
+                data_dir: None,
+                etag: None,
+                mod_time: None,
+            }),
+            dispatch: None,
+        }
+    }
+
+    #[test]
+    fn only_stable_prepared_journal_can_replace_tier_free_version() {
+        let stable = stable_prepared_journal();
+        assert!(stable.can_replace_tier_free_version());
+
+        let mut committed = stable.clone();
+        committed.state = TierDeleteJournalState::Committed;
+        assert!(!committed.can_replace_tier_free_version());
+
+        let mut unbound = stable.clone();
+        unbound.backend_identity = None;
+        assert!(!unbound.can_replace_tier_free_version());
+
+        let mut unknown = stable.clone();
+        unknown.version_state = TransitionVersionState::Unknown;
+        assert!(!unknown.can_replace_tier_free_version());
+
+        let mut unstable = stable;
+        unstable.source = Some(TierDeleteSourceIdentity {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+            versioned: false,
+            version_suspended: false,
+            data_dir: None,
+            etag: Some("etag-only".to_string()),
+            mod_time: None,
+        });
+        assert!(!unstable.can_replace_tier_free_version());
+    }
+
     #[test]
     fn signer_header_error_detection_matches_utf8_failures() {
         let err = Error::new(

crates/ecstore/src/bucket/metadata_sys.rs

@@ -21,7 +21,7 @@ use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
-use crate::error::{Error, Result, is_err_bucket_not_found};
+use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
 use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::heal::HealOperations as _;
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
@@ -659,13 +659,12 @@ async fn acquire_config_write_guard_for_incarnation(
             async {
                 match metadata_sys
                     .api
-                    .peer_sys
-                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
                 {
                     Ok(_) => Ok(()),
-                    Err(crate::disk::error::Error::VolumeNotFound) => Err(Error::BucketNotFound(bucket.to_string())),
-                    Err(err) => Err(err.into()),
+                    Err(err) if is_err_strict_volume_not_found(&err) => Err(Error::BucketNotFound(bucket.to_string())),
+                    Err(err) => Err(err),
                 }
             },
         ),
@@ -1136,6 +1135,16 @@ pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Res
     bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
 }
 
+pub(crate) async fn has_authoritative_never_versioned_state_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<bool> {
+    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
+
+    bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
+}
+
 pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1264,7 +1273,6 @@ pub struct BucketMetadataSys {
     /// name floods while avoiding repeated namespace and erasure reads.
     missing_buckets: moka::future::Cache<String, ()>,
     api: Arc<ECStore>,
-    initialized: Arc<RwLock<bool>>,
 }
 
 impl BucketMetadataSys {
@@ -1293,7 +1301,6 @@ impl BucketMetadataSys {
                 .time_to_live(MISSING_BUCKET_TTL)
                 .build(),
             api,
-            initialized: Arc::new(RwLock::new(false)),
         }
     }
 
@@ -1354,13 +1361,12 @@ impl BucketMetadataSys {
         await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
             match self
                 .api
-                .peer_sys
-                .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                 .await
             {
                 Ok(_) => Ok(true),
-                Err(crate::disk::error::Error::VolumeNotFound) => Ok(false),
-                Err(err) => Err(err.into()),
+                Err(Error::VolumeNotFound) => Ok(false),
+                Err(err) => Err(err),
             }
         })
         .await
@@ -1370,9 +1376,15 @@ impl BucketMetadataSys {
         let _ = self.init_internal(buckets).await;
     }
     async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
-        let count = runtime_sources::endpoint_erasure_set_count()
-            .map(|count| count * 10)
-            .ok_or_else(|| Error::other("endpoint pools not initialized"))?;
+        let count = self
+            .api
+            .pools
+            .iter()
+            .map(|pool| pool.disk_set.len())
+            .sum::<usize>()
+            .checked_mul(10)
+            .filter(|count| *count != 0)
+            .ok_or_else(|| Error::other("bucket metadata store has no erasure sets"))?;
 
         let mut failed_buckets: HashSet<String> = HashSet::new();
         let mut buckets = buckets.as_slice();
@@ -1390,9 +1402,6 @@ impl BucketMetadataSys {
             buckets = &buckets[count..]
         }
 
-        let mut initialized = self.initialized.write().await;
-        *initialized = true;
-
         Ok(())
     }
 
@@ -1469,14 +1478,6 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
-        await_bucket_namespace_operation(
-            Some(namespace_guard),
-            bucket,
-            "bucket metadata heal",
-            self.api.heal_bucket(bucket, &HealOpts::default()),
-        )
-        .await?;
-
         if !self
             .bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
             .await?
@@ -1496,6 +1497,20 @@ impl BucketMetadataSys {
             return Ok(());
         }
 
+        await_bucket_namespace_operation(
+            Some(namespace_guard),
+            bucket,
+            "bucket metadata heal",
+            self.api.heal_bucket(
+                bucket,
+                &HealOpts {
+                    recreate: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await?;
+
         let (bm, persisted) = await_bucket_namespace_operation(
             Some(namespace_guard),
             bucket,
@@ -1877,23 +1892,13 @@ impl BucketMetadataSys {
                     "lazy metadata IO must start while the bucket namespace read lock is held"
                 );
             }
-            let (bm, persisted) = match await_bucket_namespace_operation(
+            let (bm, persisted) = await_bucket_namespace_operation(
                 Some(&guard),
                 bucket,
                 "lazy bucket metadata load",
                 Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
             )
-            .await
-            {
-                Ok(res) => res,
-                Err(err) => {
-                    return if *self.initialized.read().await {
-                        Err(Error::other("errBucketMetadataNotInitialized"))
-                    } else {
-                        Err(err)
-                    };
-                }
-            };
+            .await?;
 
             let bm = Arc::new(bm);
 
@@ -1904,11 +1909,9 @@ impl BucketMetadataSys {
                     "lazy bucket metadata existence check",
                     Box::pin(async {
                         self.api
-                            .peer_sys
-                            .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                            .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                             .await
                             .map(|_| ())
-                            .map_err(Into::into)
                     }),
                 )
                 .await?;
@@ -2187,10 +2190,8 @@ impl BucketMetadataSys {
             "legacy bucket metadata existence check",
             async {
                 self.api
-                    .peer_sys
-                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
-                    .map_err(crate::error::StorageError::from)
             },
         )
         .await
@@ -2292,10 +2293,8 @@ impl BucketMetadataSys {
             "bucket metadata snapshot existence check",
             async {
                 self.api
-                    .peer_sys
-                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
-                    .map_err(crate::error::StorageError::from)
             },
         )
         .await

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -22,6 +22,7 @@ pub(crate) use rustfs_replication::{
     delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
     delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
     is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
-    resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
+    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
+    single_part_replica_etag_mismatch, target_delete_version_id,
 };

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -32,8 +32,9 @@ use super::replication_object_decision_boundary::{
     MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
     delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
     is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
-    should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
+    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
+    resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
+    target_delete_version_id,
 };
 use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
 use super::replication_resync_boundary::ResyncStatusType;
@@ -88,6 +89,7 @@ use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, LazyLock, Mutex as StdMutex};
+use std::time::Instant;
 use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 use tokio::io::AsyncRead;
@@ -118,6 +120,7 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_
 const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
 const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
 const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
+const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
 const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
 
 #[allow(
@@ -190,11 +193,19 @@ fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &Obj
 
 const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
 
-/// Targets that already produced a version-identity-drift warning this
-/// process lifetime, by ARN. Deduping is advisory only (the metric still
-/// counts every drifting PUT), so a reconfigured target re-warning only
-/// after a restart is acceptable.
-static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
+/// How long a target stays quiet after reporting version-identity drift.
+///
+/// This used to be a plain "once per ARN per process": one line ever, which on
+/// a long-lived server meant the single most important diagnostic for a
+/// non-converging generic S3 target scrolled away hours before anyone looked
+/// (rustfs#6822). Re-arming on an interval keeps the log bounded while leaving
+/// the condition discoverable in any recent window.
+const VERSION_IDENTITY_DRIFT_LOG_INTERVAL: TokioDuration = TokioDuration::from_secs(600);
+
+/// When each target last reported version-identity drift, by ARN. Throttling is
+/// advisory only — the metric still counts every drifting PUT.
+static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashMap<String, Instant>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
 
 /// Version purges the peer denied under object lock (#6850). A RustFS peer
 /// with the replicated-purge GOVERNANCE exemption
@@ -322,20 +333,39 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
         return;
     }
     counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
+    if !version_identity_drift_log_due(&tgt_client.arn, Instant::now()) {
+        return;
+    }
+    // `error`, not `warn`: the target silently refuses the addressing scheme
+    // every version-addressed delete and heal on it depends on, so replication
+    // to it can never converge. At `warn` this sat below `DEFAULT_LOG_LEVEL`
+    // and no default deployment ever saw the one line that explains why a
+    // purged version is still on the target (rustfs#6822).
+    error!(
+        event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+        arn = %tgt_client.arn,
+        endpoint = %tgt_client.endpoint,
+        sent_version_id = %source_version_id,
+        assigned_version_id = assigned_version_id.unwrap_or("<none>"),
+        "Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
+    );
+}
+
+/// Whether this ARN's version-identity drift is due to be logged again at
+/// `now`, re-arming the throttle when it is. Split out from the audit so the
+/// interval policy is testable without a target client.
+fn version_identity_drift_log_due(arn: &str, now: Instant) -> bool {
     let mut warned = VERSION_IDENTITY_WARNED_ARNS
         .lock()
         .unwrap_or_else(|poisoned| poisoned.into_inner());
-    if warned.insert(tgt_client.arn.clone()) {
-        warn!(
-            event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
-            component = LOG_COMPONENT_ECSTORE,
-            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-            arn = %tgt_client.arn,
-            endpoint = %tgt_client.endpoint,
-            sent_version_id = %source_version_id,
-            assigned_version_id = assigned_version_id.unwrap_or("<none>"),
-            "Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
-        );
+    match warned.get(arn) {
+        Some(last) if now.duration_since(*last) < VERSION_IDENTITY_DRIFT_LOG_INTERVAL => false,
+        _ => {
+            warned.insert(arn.to_string(), now);
+            true
+        }
     }
 }
 
@@ -2050,10 +2080,13 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
         }
     }
 
+    let delete_version_id = dobj.delete_object.version_id.map(|v| v.to_string());
+    note_replication_terminal_failure(&bucket, &dobj.delete_object.object_name, delete_version_id.as_deref(), &rinfos);
+
     let mut drs = get_replication_state(
         &rinfos,
         &dobj.delete_object.replication_state.clone().unwrap_or_default(),
-        dobj.delete_object.version_id.map(|v| v.to_string()),
+        delete_version_id,
     );
     if replication_status != prev_status {
         drs.replication_timestamp = Some(OffsetDateTime::now_utc());
@@ -3023,8 +3056,11 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
         }
     }
 
+    let version_id = roi.version_id.map(|v| v.to_string());
+    note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos);
+
     let previous_state = roi.replication_state.clone().unwrap_or_default();
-    let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
+    let merged_state = get_replication_state(&rinfos, &previous_state, version_id);
     let replication_status = merged_state.composite_replication_status();
     let new_replication_internal = merged_state.replication_status_internal.clone();
     let mut object_info = roi.to_object_info();
@@ -3101,6 +3137,61 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
     (merged_state, state_persisted)
 }
 
+/// Emit the operator-visible record of a replication attempt that ended FAILED.
+///
+/// Every per-branch failure log in this module is deliberately quieter than
+/// `error`: most of them sit on the replication hot path and fire once per
+/// object *per ARN*, so a target that stays unreachable would flood the log
+/// from inside the transfer loop. That left a hole customers fell into
+/// (rustfs#6825): `DEFAULT_LOG_LEVEL` is `error`, so on a stock deployment a
+/// failed object produced no line at all, and an operator staring at a replica
+/// that never arrived had nothing to correlate — the same trap already
+/// documented for the GET path in
+/// `crates/e2e_test/src/get_stream_failure_observability_test.rs`.
+///
+/// This is the one place that knows an object reached a *terminal* FAILED state
+/// for a target, so this is where the guaranteed-visible line belongs. It is
+/// bounded by the number of objects that actually fail rather than by attempts
+/// inside a transfer, and it carries the target's own error so a remote
+/// rejection is diagnosable without the operator first having to lower the
+/// global log level and reproduce.
+fn note_replication_terminal_failure(bucket: &str, object: &str, version_id: Option<&str>, rinfos: &ReplicatedInfos) {
+    for target in rinfos.targets.iter() {
+        if target.is_empty() {
+            continue;
+        }
+        let replication_failed = target.replication_status == ReplicationStatusType::Failed;
+        let purge_failed = target.version_purge_status == VersionPurgeStatusType::Failed;
+        if !replication_failed && !purge_failed {
+            continue;
+        }
+
+        error!(
+            event = EVENT_REPLICATION_OBJECT_FAILED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            bucket = %bucket,
+            object = %object,
+            version_id = version_id.unwrap_or("-"),
+            arn = %target.arn,
+            endpoint = %target.endpoint,
+            op_type = %target.op_type,
+            size = target.size,
+            replication_status = %target.replication_status.as_str(),
+            version_purge_status = %target.version_purge_status.as_str(),
+            // The target's error can carry a signed URL or an echoed auth
+            // header, so it goes through the same redaction as the persisted
+            // resync detail rather than straight into the log.
+            error = %target
+                .error
+                .as_deref()
+                .and_then(sanitize_resync_error_detail)
+                .unwrap_or_else(|| "<none>".to_string()),
+            "Replication failed for object"
+        );
+    }
+}
+
 fn unavailable_object_target_info(roi: &ReplicateObjectInfo, arn: &str) -> ReplicatedTargetInfo {
     ReplicatedTargetInfo {
         arn: arn.to_string(),
@@ -3400,6 +3491,33 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
+        if let Some(reason) = replication_single_put_size_error(is_multipart, transfer_size) {
+            drop(gr);
+            rinfo.replication_status = ReplicationStatusType::Failed;
+            rinfo.error = Some(reason.clone());
+            warn!(
+                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                target_bucket = %tgt_client.bucket,
+                arn = %tgt_client.arn,
+                object = %object,
+                operation = "put_object",
+                transfer_size = transfer_size,
+                error = %reason,
+                "Replication target operation failed"
+            );
+            send_local_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.to_string(),
+                bucket_name: bucket.clone(),
+                object: object_info,
+                user_agent: "Internal: [Replication]".to_string(),
+                ..Default::default()
+            });
+            return rinfo;
+        }
+
         if let Some(err) = if is_multipart {
             drop(gr);
             let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -4068,6 +4186,14 @@ async fn replicate_all_payload_to_target<S: ReplicationObjectIO>(
     ctx: ReplicateAllPayloadContext<'_, S>,
     mut gr: GetObjectReader,
 ) -> Option<std::io::Error> {
+    // Fail before streaming a body the target is required to reject: an S3
+    // PutObject caps at 5 GiB, and this route is chosen by the source object's
+    // storage shape rather than its size (rustfs#6825).
+    if let Some(reason) = replication_single_put_size_error(ctx.is_multipart, ctx.transfer_size) {
+        drop(gr);
+        return Some(std::io::Error::other(reason));
+    }
+
     if ctx.is_multipart {
         drop(gr);
         let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -5716,4 +5842,207 @@ mod tests {
         assert!(!retry_scheduled.load(Ordering::SeqCst));
         assert_eq!(result.unwrap_err().to_string(), "transfer failed");
     }
+
+    /// A replication target's terminal outcome, as the operator sees it.
+    fn failed_target(arn: &str, error: &str) -> ReplicatedTargetInfo {
+        ReplicatedTargetInfo {
+            arn: arn.to_string(),
+            size: 6 * 1024 * 1024 * 1024,
+            op_type: ReplicationType::Object,
+            replication_status: ReplicationStatusType::Failed,
+            endpoint: "s3.wasabisys.com".to_string(),
+            error: Some(error.to_string()),
+            ..Default::default()
+        }
+    }
+
+    /// Capture the log this module writes, filtered exactly the way a stock
+    /// deployment filters it.
+    fn logs_at_default_level(emit: impl FnOnce()) -> String {
+        use std::sync::{Arc, Mutex};
+        use tracing_subscriber::EnvFilter;
+        use tracing_subscriber::fmt::MakeWriter;
+        use tracing_subscriber::layer::SubscriberExt;
+
+        #[derive(Clone, Default)]
+        struct CapturedLogs {
+            buffer: Arc<Mutex<Vec<u8>>>,
+        }
+        struct CapturedLogWriter {
+            buffer: Arc<Mutex<Vec<u8>>>,
+        }
+        impl std::io::Write for CapturedLogWriter {
+            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+                self.buffer
+                    .lock()
+                    .expect("captured logs mutex should not be poisoned")
+                    .extend_from_slice(buf);
+                Ok(buf.len())
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        impl<'a> MakeWriter<'a> for CapturedLogs {
+            type Writer = CapturedLogWriter;
+            fn make_writer(&'a self) -> Self::Writer {
+                CapturedLogWriter {
+                    buffer: Arc::clone(&self.buffer),
+                }
+            }
+        }
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::registry()
+            // Not a hand-picked level: this is the filter an operator who has
+            // changed nothing is actually running.
+            .with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_writer(logs.clone())
+                    .with_ansi(false)
+                    .without_time(),
+            );
+        let _guard = tracing::subscriber::set_default(subscriber);
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
+
+        emit();
+
+        let buffer = logs
+            .buffer
+            .lock()
+            .expect("captured logs mutex should not be poisoned")
+            .clone();
+        String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
+    }
+
+    /// rustfs#6825: a 6 GiB object never reached the target and the server said
+    /// nothing an operator could act on, because every failure line in this
+    /// module sat below `DEFAULT_LOG_LEVEL`. The object key, the target, and
+    /// the target's own error have to survive the default filter.
+    #[test]
+    fn failed_replication_names_the_object_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![failed_target("arn:replication::wasabi", "put_object failed: EntityTooLarge")],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/vm-image.qcow2", Some("v-9"), &rinfos);
+        });
+
+        assert!(logs.contains("backups/vm-image.qcow2"), "the failed object must be named: {logs}");
+        assert!(logs.contains("arn:replication::wasabi"), "the target must be named: {logs}");
+        assert!(logs.contains("EntityTooLarge"), "the target's own error must survive: {logs}");
+        assert!(logs.contains("v-9"), "the version must be named: {logs}");
+        assert!(logs.contains(EVENT_REPLICATION_OBJECT_FAILED), "the event must be structured: {logs}");
+    }
+
+    #[test]
+    fn successful_replication_stays_quiet_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo {
+                arn: "arn:replication::wasabi".to_string(),
+                replication_status: ReplicationStatusType::Completed,
+                ..Default::default()
+            }],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/ok.bin", None, &rinfos);
+        });
+
+        assert!(logs.is_empty(), "a completed replication must not log an error: {logs}");
+    }
+
+    /// A failed version purge is the 6822 symptom (the version stays on the
+    /// target); it must be as visible as a failed transfer even though the
+    /// replication status itself is not FAILED.
+    #[test]
+    fn failed_version_purge_is_reported_at_the_default_log_level() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo {
+                arn: "arn:replication::wasabi".to_string(),
+                op_type: ReplicationType::Delete,
+                replication_status: ReplicationStatusType::Empty,
+                version_purge_status: VersionPurgeStatusType::Failed,
+                error: Some("remove_object failed: NoSuchVersion".to_string()),
+                ..Default::default()
+            }],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/purged.bin", Some("v-1"), &rinfos);
+        });
+
+        assert!(logs.contains("backups/purged.bin"), "the purged object must be named: {logs}");
+        assert!(logs.contains("NoSuchVersion"), "the target's own error must survive: {logs}");
+    }
+
+    /// The target's error is echoed remote text and can carry a signed URL or
+    /// an auth header, so it goes through the persisted-detail redaction rather
+    /// than straight into the log.
+    #[test]
+    fn failed_replication_redacts_a_sensitive_target_error() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![failed_target(
+                "arn:replication::wasabi",
+                "put_object failed: rejected Authorization: Bearer super-secret",
+            )],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/vm-image.qcow2", None, &rinfos);
+        });
+
+        assert!(logs.contains("backups/vm-image.qcow2"), "the object must still be named: {logs}");
+        assert!(!logs.contains("super-secret"), "the credential must not reach the log: {logs}");
+    }
+
+    /// An empty target slot carries no outcome; reporting it would invent a
+    /// failure for a target that was never attempted.
+    #[test]
+    fn empty_target_slots_are_not_reported_as_failures() {
+        let rinfos = ReplicatedInfos {
+            replication_timestamp: Some(OffsetDateTime::now_utc()),
+            targets: vec![ReplicatedTargetInfo::default()],
+        };
+
+        let logs = logs_at_default_level(|| {
+            note_replication_terminal_failure("photos", "backups/unattempted.bin", None, &rinfos);
+        });
+
+        assert!(logs.is_empty(), "an empty target slot must not be reported: {logs}");
+    }
+
+    #[test]
+    fn version_identity_drift_re_arms_after_the_throttle_interval() {
+        let arn = "arn:replication::drift-throttle-test";
+        let start = Instant::now();
+
+        assert!(version_identity_drift_log_due(arn, start), "first drift must be reported");
+        assert!(
+            !version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
+            "a second drift inside the interval must stay throttled"
+        );
+        assert!(
+            version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
+            "drift must become visible again once the interval elapses, instead of \
+             going silent for the rest of the process lifetime"
+        );
+    }
+
+    #[test]
+    fn version_identity_drift_throttles_each_target_independently() {
+        let now = Instant::now();
+
+        assert!(version_identity_drift_log_due("arn:replication::drift-a", now));
+        assert!(
+            version_identity_drift_log_due("arn:replication::drift-b", now),
+            "one target's report must not silence another's"
+        );
+    }
 }

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -260,8 +260,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
             // the response path relies on). Keep the object's own multipart
             // flag so encrypted objects stay on the multipart route.
         } else {
-            let (checksum_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
-            is_multipart = is_mp;
+            let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
+            // The checksum record describes how the *checksum* is composed,
+            // not how the object is stored. A full-object checksum carries no
+            // MULTIPART flag even on a multipart upload, so trusting it here
+            // routed a 768-part object through a single PutObject and the
+            // target rejected the 6 GiB body with EntityTooLarge
+            // (rustfs#6825). The object's own shape is the authority: the
+            // record may only add multipart-ness, never take it away.
+            is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
 
             for (key, value) in checksum_meta.iter() {
                 if key != AMZ_CHECKSUM_TYPE {
@@ -518,6 +525,109 @@ mod tests {
     use time::Duration;
     use uuid::Uuid;
 
+    /// Serialize an object-level checksum record the way
+    /// `complete_multipart_upload` persists it for a **full-object** checksum:
+    /// the record carries the plain algorithm type, without the MULTIPART
+    /// flags that a composite record gets.
+    fn full_object_multipart_checksum_record() -> bytes::Bytes {
+        let checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type("crc32", "FULL_OBJECT");
+        assert!(checksum_type.is_set(), "crc32 FULL_OBJECT must be a valid checksum type");
+        assert!(checksum_type.full_object_requested());
+
+        let mut combined = Vec::new();
+        let mut checksum = rustfs_rio::Checksum {
+            checksum_type,
+            ..Default::default()
+        };
+        for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
+            let part_checksum = rustfs_rio::Checksum::new_from_data(checksum_type, part).expect("part checksum");
+            combined.extend_from_slice(part_checksum.raw.as_slice());
+            checksum.add_part(&part_checksum, part.len() as i64).expect("add part");
+        }
+
+        checksum.to_bytes(&combined)
+    }
+
+    #[test]
+    fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
+        // rustfs#6825: a 768-part upload was replicated with a single
+        // PutObject and rejected by the target with EntityTooLarge. The
+        // object's storage shape says multipart; only the checksum record
+        // looked single-part, and the checksum record must not decide the
+        // transport.
+        let object_info = ObjectInfo {
+            etag: Some("0123456789abcdef0123456789abcdef-768".to_string()),
+            checksum: Some(full_object_multipart_checksum_record()),
+            size: 6 * 1024 * 1024 * 1024,
+            ..Default::default()
+        };
+
+        assert!(object_info.is_multipart(), "the fixture must be a multipart object");
+        let (_, checksum_says_multipart) = object_info
+            .decrypt_checksums(0, &HeaderMap::new())
+            .expect("checksum record must decode");
+        assert!(
+            !checksum_says_multipart,
+            "fixture precondition: a full-object record carries no MULTIPART flag, which is what used to \
+             downgrade the transport"
+        );
+
+        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
+
+        assert!(
+            is_multipart,
+            "a multipart object must replicate over multipart whatever its checksum record looks like"
+        );
+    }
+
+    #[test]
+    fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
+        // The mirror of the rustfs#6825 guard: an object stored as one PUT
+        // must keep the single-PUT transport, or its replica's ETag would
+        // change shape and every ETag-based convergence check would re-copy it.
+        let checksum =
+            rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"whole-object").expect("checksum fixture");
+        let object_info = ObjectInfo {
+            etag: Some("0123456789abcdef0123456789abcdef".to_string()),
+            checksum: Some(checksum.to_bytes(&[])),
+            ..Default::default()
+        };
+
+        assert!(!object_info.is_multipart(), "the fixture must be a single-part object");
+
+        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
+
+        assert!(!is_multipart, "a single-part object must not be promoted onto the multipart transport");
+    }
+
+    #[test]
+    fn composite_checksum_multipart_object_keeps_the_multipart_route() {
+        // The checksum shape that already worked before rustfs#6825, pinned so
+        // the fix cannot regress it.
+        let mut checksum_type = rustfs_rio::ChecksumType::from_string("crc32");
+        checksum_type
+            .merge(rustfs_rio::ChecksumType::MULTIPART)
+            .merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
+
+        let mut combined = Vec::new();
+        for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
+            let part_checksum =
+                rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
+            combined.extend_from_slice(part_checksum.raw.as_slice());
+        }
+        let checksum = rustfs_rio::Checksum::new_from_data(checksum_type, &combined).expect("composite checksum");
+
+        let object_info = ObjectInfo {
+            etag: Some("0123456789abcdef0123456789abcdef-2".to_string()),
+            checksum: Some(checksum.to_bytes(&combined)),
+            ..Default::default()
+        };
+
+        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
+
+        assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
+    }
+
     #[test]
     fn replication_action_for_target_head_existing_object_source_newer_null_version_requires_replication() {
         let source = ObjectInfo {

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
     ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
     ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
     StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
-    tier_mutation_control_service_client::TierMutationControlServiceClient,
+    TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
+    node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -462,6 +462,31 @@ pub struct PeerTierMutationOutcome {
     pub applied: bool,
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("{message}")]
+struct TierMutationDefinitelyRejected {
+    message: String,
+}
+
+fn tier_mutation_definitely_rejected_error(message: String) -> Error {
+    Error::other(TierMutationDefinitelyRejected { message })
+}
+
+#[cfg(test)]
+pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
+    tier_mutation_definitely_rejected_error(message.to_string())
+}
+
+pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
+    matches!(
+        error,
+        Error::Io(io_error)
+            if io_error
+                .get_ref()
+                .is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
+    )
+}
+
 fn validate_tier_mutation_response_proof(
     version: u32,
     phase: TierMutationRpcPhase,
@@ -469,6 +494,16 @@ fn validate_tier_mutation_response_proof(
     canonical_payload: &[u8],
     response: &TierMutationControlResponse,
 ) -> Result<()> {
+    if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
+        return Err(Error::other("peer tier mutation response proof exceeds size limit"));
+    }
+    if response
+        .error_info
+        .as_ref()
+        .is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
+    {
+        return Err(Error::other("peer tier mutation error response exceeds size limit"));
+    }
     let canonical_response =
         rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
             version,
@@ -479,6 +514,7 @@ fn validate_tier_mutation_response_proof(
             state: response.state,
             applied: response.applied,
             error_info: response.error_info.as_deref(),
+            failure_class: response.failure_class,
         })
         .map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
     verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -500,9 +536,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
         TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
         TierMutationRpcPhase::Abort => {
             if payload_len == 0 {
-                return Ok(());
+                return Err(Error::other("tier mutation abort payload is empty"));
             }
-            return Err(Error::other("tier mutation abort payload must be empty"));
+            rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
         }
         _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
     };
@@ -521,8 +557,29 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
     }
 }
 
-fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
-    Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
+fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
+    let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
+    let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
+    // RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
+    if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && status.code() == tonic::Code::FailedPrecondition
+        && status.message().as_bytes() == legacy_rejection.as_bytes()
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    Error::other(message)
+}
+
+fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
+    let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
+    if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+        && TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
+    {
+        return tier_mutation_definitely_rejected_error(message);
+    }
+    // Missing/zero, unknown, and explicit Ambiguous are deliberately the same
+    // fail-closed result: the coordinator must include this peer in Abort.
+    Error::other(message)
 }
 
 impl PeerRestClient {
@@ -1315,8 +1372,12 @@ impl PeerRestClient {
             .await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
-        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
+    pub async fn abort_tier_mutation(
+        &self,
+        mutation_id: Uuid,
+        canonical_prepare_payload: Bytes,
+    ) -> Result<PeerTierMutationOutcome> {
+        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
             .await
     }
 
@@ -1350,7 +1411,7 @@ impl PeerRestClient {
                         client
                             .prepare_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Commit => {
@@ -1363,7 +1424,7 @@ impl PeerRestClient {
                         client
                             .commit_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Abort => {
@@ -1376,18 +1437,19 @@ impl PeerRestClient {
                         client
                             .abort_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
                             .into_inner()
                     }
                     _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
                 };
                 validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
                 if !response.success {
-                    return Err(Error::other(
-                        response
-                            .error_info
-                            .unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
-                    ));
+                    return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
+                }
+                if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
+                    && response.failure_class != TierMutationFailureClass::Unspecified as i32
+                {
+                    return Err(Error::other("successful peer tier mutation response carried a failure class"));
                 }
                 let state = decode_tier_mutation_peer_state(response.state)?;
                 Ok(PeerTierMutationOutcome {
@@ -3332,6 +3394,7 @@ mod tests {
         state: i32,
         applied: bool,
         error_info: Option<&'a str>,
+        failure_class: i32,
     }
 
     fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3345,6 +3408,7 @@ mod tests {
                 state: input.state,
                 applied: input.applied,
                 error_info: input.error_info,
+                failure_class: input.failure_class,
             })
             .expect("small tier mutation response should encode");
         let response_proof =
@@ -3355,6 +3419,7 @@ mod tests {
             applied: input.applied,
             error_info: input.error_info.map(str::to_string),
             response_proof: response_proof.into(),
+            failure_class: input.failure_class,
         }
     }
 
@@ -3372,6 +3437,7 @@ mod tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
+            failure_class: TierMutationFailureClass::Unspecified as i32,
         });
         validate_tier_mutation_response_proof(
             rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3396,6 +3462,10 @@ mod tests {
                 applied: false,
                 ..response.clone()
             },
+            TierMutationControlResponse {
+                failure_class: TierMutationFailureClass::Ambiguous as i32,
+                ..response.clone()
+            },
         ] {
             let err = validate_tier_mutation_response_proof(
                 rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3419,6 +3489,44 @@ mod tests {
         assert!(err.to_string().contains("invalid tier mutation response proof"));
     }
 
+    #[test]
+    fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
+        let mutation_id = Uuid::new_v4();
+        let payload = b"tier-mutation-prepare";
+        let oversized_proof = TierMutationControlResponse {
+            success: false,
+            state: TierMutationPeerState::Unspecified as i32,
+            applied: false,
+            error_info: None,
+            response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
+            failure_class: TierMutationFailureClass::Ambiguous as i32,
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_proof,
+        )
+        .expect_err("oversized proof must fail before cryptographic verification");
+        assert!(err.to_string().contains("response proof exceeds size limit"));
+
+        let oversized_error = TierMutationControlResponse {
+            response_proof: Bytes::new(),
+            error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
+            ..oversized_proof
+        };
+        let err = validate_tier_mutation_response_proof(
+            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
+            TierMutationRpcPhase::Prepare,
+            mutation_id,
+            payload,
+            &oversized_error,
+        )
+        .expect_err("oversized error detail must fail before proof construction");
+        assert!(err.to_string().contains("error response exceeds size limit"));
+    }
+
     #[test]
     fn tier_mutation_peer_state_decode_fails_closed() {
         assert_eq!(
@@ -3463,8 +3571,17 @@ mod tests {
             )
             .is_err()
         );
-        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
-        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
+        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
+            .expect("max abort payload should fit");
+        assert!(
+            validate_tier_mutation_payload_len(
+                TierMutationRpcPhase::Abort,
+                rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -3479,7 +3596,7 @@ mod tests {
                 tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
                 tonic::Status::unavailable("peer tier mutation control unavailable"),
             ] {
-                let err = tier_mutation_control_status_error(phase, status);
+                let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
                 let rendered = err.to_string();
                 assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
                 assert!(
@@ -3493,6 +3610,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let exact = format!("unsupported tier mutation peer protocol version: {version}");
+        let rejected = tier_mutation_control_status_error(
+            TierMutationRpcPhase::Prepare,
+            version,
+            tonic::Status::failed_precondition(exact.clone()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for status in [
+            tonic::Status::failed_precondition(format!("{exact}.")),
+            tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
+            tonic::Status::invalid_argument(exact.clone()),
+            tonic::Status::unimplemented(exact),
+        ] {
+            let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "near-text, wrong-code, and Unimplemented failures must remain ambiguous"
+            );
+        }
+    }
+
+    #[test]
+    fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
+        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
+        let rejected = tier_mutation_failed_response_error(
+            version,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("rejected".to_string()),
+        );
+        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
+
+        for failure_class in [
+            TierMutationFailureClass::Unspecified as i32,
+            TierMutationFailureClass::Ambiguous as i32,
+            99,
+        ] {
+            let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
+            assert!(
+                !tier_mutation_error_is_definitely_rejected(&ambiguous),
+                "missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
+            );
+        }
+
+        let v3_ignores_v4_class = tier_mutation_failed_response_error(
+            rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
+            TierMutationFailureClass::PreDispatchRejected as i32,
+            Some("legacy failure".to_string()),
+        );
+        assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
+    }
+
     #[tokio::test]
     async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -190,7 +190,7 @@ fn install_heal_bucket_pre_mutation_barrier() -> Arc<DeleteBucketEmptyScanBarrie
 }
 
 #[cfg(test)]
-async fn pause_after_delete_bucket_empty_scan() {
+pub(crate) async fn pause_after_delete_bucket_empty_scan() {
     let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
         .lock()
         .expect("empty scan barrier lock should not be poisoned")

crates/ecstore/src/core/pools_test.rs

@@ -196,7 +196,7 @@ mod decommission_lock_order_tests {
     use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING;
     use crate::core::pools::{
         DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
-        POOL_META_NAME, set_decommission_capacity_info_overrides_for_test,
+        POOL_META_NAME, decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test,
     };
     use crate::data_movement;
     use crate::disk::RUSTFS_META_BUCKET;
@@ -3047,6 +3047,198 @@ mod decommission_lock_order_tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn exact_delete_reconciles_pending_capacity_before_removing_replicas() {
+        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+        let bucket = test_bucket("exact-delete-capacity");
+        let object = "published-before-exact-delete.bin";
+        let body = vec![0x55; 64 * 1024];
+        let version_id = uuid::Uuid::new_v4().to_string();
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create the exact-delete reconciliation bucket");
+        let incarnation = store
+            .bucket_incarnation_id(&bucket)
+            .await
+            .expect("load the exact-delete bucket incarnation");
+
+        let mut source_data = PutObjReader::from_vec(body.clone());
+        let source = store.pools[0]
+            .put_object(
+                &bucket,
+                object,
+                &mut source_data,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.clone()),
+                    expected_bucket_incarnation_id: Some(incarnation),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed the exact source version");
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        let target_total = body.len().saturating_mul(4);
+        set_decommission_capacity_info_overrides_for_test(
+            store.id,
+            vec![vec![
+                DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
+                DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
+            ]],
+        );
+        store
+            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
+            .await
+            .expect("activate the exact-delete capacity reservation");
+        let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
+        let target_pool_index = store.pool_meta.read().await.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the exact-delete capacity reservation should exist")
+            .targets[0]
+            .pool_index;
+        assert_eq!(target_pool_index, 2);
+
+        let target_options = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.clone()),
+            mod_time: source.mod_time,
+            preserve_etag: source.etag.clone(),
+            user_defined: (*source.user_defined).clone(),
+            data_movement: true,
+            src_pool_idx: 0,
+            expected_bucket_incarnation_id: Some(incarnation),
+            ..Default::default()
+        };
+        let mut target_data = PutObjReader::from_vec(body.clone());
+        let target = store.pools[target_pool_index]
+            .put_object(&bucket, object, &mut target_data, &target_options)
+            .await
+            .expect("publish the target version before capacity progress");
+        assert_eq!(target.version_id, source.version_id);
+        assert_eq!(target.mod_time, source.mod_time);
+        assert_eq!(target.size, source.size);
+
+        let source_version_id = source.version_id.map(|version_id| version_id.to_string());
+        let mutation_id = decommission_capacity_mutation_id(
+            owner,
+            &source.bucket,
+            &source.name,
+            source_version_id.as_deref(),
+            source.delete_marker,
+            source.mod_time,
+        );
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            let source_pool = &mut pool_meta.pools[0];
+            let reservation = source_pool
+                .decommission
+                .as_mut()
+                .and_then(|info| info.capacity_reservation.as_mut())
+                .expect("the exact-delete capacity reservation should remain active");
+            let target = reservation
+                .targets
+                .iter_mut()
+                .find(|target| target.pool_index == target_pool_index)
+                .expect("the exact-delete target allocation should exist");
+            target.pending_physical_bytes = body.len();
+            target.pending_mutation_id = Some(mutation_id);
+            reservation.pending_target_physical_bytes = body.len();
+            source_pool.last_update = time::OffsetDateTime::now_utc();
+        }
+        store
+            .save_current_pool_meta_for_test(&[0])
+            .await
+            .expect("persist the simulated post-commit capacity intent");
+
+        let exact_delete_options = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.clone()),
+            expected_bucket_incarnation_id: Some(incarnation),
+            ..Default::default()
+        };
+        store.pools[target_pool_index]
+            .delete_object(&bucket, object, exact_delete_options.clone())
+            .await
+            .expect("remove the target evidence before the fail-closed exact delete");
+        let delete_err = store
+            .delete_object(&bucket, object, exact_delete_options.clone())
+            .await
+            .expect_err("exact delete must fail while its pending target evidence is absent");
+        assert!(matches!(delete_err, crate::error::Error::DecommissionCapacityBlocked { .. }));
+        store.pools[0]
+            .get_object_info(
+                &bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.clone()),
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("a failed reconciliation must preserve the source evidence");
+        let mut failed = crate::core::pools::PoolMeta::default();
+        failed
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("the failed exact delete must preserve readable capacity metadata");
+        let failed_reservation = failed.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the failed exact delete reservation should remain present");
+        assert_eq!(failed_reservation.pending_target_physical_bytes, body.len());
+        assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
+
+        let mut replacement_target_data = PutObjReader::from_vec(body.clone());
+        store.pools[target_pool_index]
+            .put_object(&bucket, object, &mut replacement_target_data, &target_options)
+            .await
+            .expect("restore the equivalent target evidence for the exact-delete retry");
+
+        store
+            .delete_object(&bucket, object, exact_delete_options)
+            .await
+            .expect("the exact delete should reconcile capacity before removing replicas");
+
+        let mut persisted = crate::core::pools::PoolMeta::default();
+        persisted
+            .load_no_lock_from_replicas(store.pools.clone())
+            .await
+            .expect("the exact-delete reconciliation should remain durable");
+        let reservation = persisted.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("the reconciled reservation should remain present");
+        assert_eq!(reservation.pending_target_physical_bytes, 0);
+        assert_eq!(reservation.consumed_target_physical_bytes, body.len());
+        assert_eq!(reservation.committed_data_bytes, body.len());
+
+        for pool_index in [0, target_pool_index] {
+            let err = store.pools[pool_index]
+                .get_object_info(
+                    &bucket,
+                    object,
+                    &ObjectOptions {
+                        versioned: true,
+                        version_id: Some(version_id.clone()),
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect_err("the exact version should be absent after reconciliation and delete");
+            assert!(crate::error::is_err_object_not_found(&err) || crate::error::is_err_version_not_found(&err));
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() {

crates/ecstore/src/core/sets.rs

@@ -37,8 +37,9 @@ use crate::{
     runtime::instance::{InstanceContext, bootstrap_ctx},
     runtime::sources as runtime_sources,
     set_disk::{PreparedGetObjectMetadata, SetDisks},
-    store::init_format::{
-        check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
+    store::{
+        RemoteTuplePublicationFence,
+        init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
     },
 };
 use futures::{
@@ -625,6 +626,19 @@ impl Sets {
             .put_object_with_old_current_size(bucket, object, data, opts)
             .await
     }
+
+    pub(crate) async fn put_object_with_old_current_size_for_data_movement(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut PutObjReader,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
+        self.get_disks_by_key(object)
+            .put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
+            .await
+    }
 }
 
 #[async_trait::async_trait]

crates/ecstore/src/data_movement/mod.rs

@@ -27,7 +27,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
+use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,7 +161,7 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
     flag.store(false, Ordering::Relaxed);
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 struct DataMovementMultipartAbortBarrierState {
     bucket: String,
     object: String,
@@ -169,17 +169,17 @@ struct DataMovementMultipartAbortBarrierState {
     release: tokio::sync::Notify,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) struct DataMovementMultipartAbortBarrier {
     state: Arc<DataMovementMultipartAbortBarrierState>,
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
     std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
 > = std::sync::OnceLock::new();
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl DataMovementMultipartAbortBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
         let state = Arc::new(DataMovementMultipartAbortBarrierState {
@@ -204,7 +204,7 @@ impl DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 impl Drop for DataMovementMultipartAbortBarrier {
     fn drop(&mut self) {
         self.state.release.notify_one();
@@ -218,7 +218,7 @@ impl Drop for DataMovementMultipartAbortBarrier {
     }
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
     let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
         .get_or_init(|| std::sync::Mutex::new(None))
@@ -717,8 +717,8 @@ fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPart
                 == target.checksums.as_ref().filter(|checksums| !checksums.is_empty()))
 }
 
-fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<BTreeMap<usize, &ObjectPartInfo>> {
-    let mut parts_by_number = BTreeMap::new();
+pub(crate) fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<HashMap<usize, &ObjectPartInfo>> {
+    let mut parts_by_number = HashMap::with_capacity(parts.len());
     for part in parts {
         if parts_by_number.insert(part.number, part).is_some() {
             return None;
@@ -1518,7 +1518,7 @@ pub(crate) async fn migrate_decommission_object(
     capacity_owner: Option<DecommissionCapacityOwner>,
 ) -> Result<()> {
     let source = rd.object_info.clone();
-    let _mutation_fence = store
+    let mutation_fence = store
         .acquire_decommission_object_mutation_fence(&bucket, &source.name)
         .await?;
     let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1537,7 +1537,7 @@ pub(crate) async fn migrate_decommission_object(
         op_label,
         None,
         capacity_owner,
-        Some(&_mutation_fence),
+        Some(mutation_fence),
     )
     .await
 }
@@ -1588,8 +1588,9 @@ async fn migrate_object_inner(
     op_label: &str,
     lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     capacity_owner: Option<DecommissionCapacityOwner>,
-    mutation_fence: Option<&ObjectLockDiagGuard>,
+    mutation_fence: Option<DecommissionFixedReadAnchor>,
 ) -> Result<()> {
+    let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
     let capacity_owner = capacity_owner.map(|owner| {
         let version_id = object_info.version_id.map(|version_id| version_id.to_string());
@@ -1605,6 +1606,13 @@ async fn migrate_object_inner(
         });
         owner.with_mutation_id(mutation_id)
     });
+    // Capture the exact source/tier identity before any client-paced read, but
+    // defer both the tier lease and source/target write locks to the final
+    // publication. Decommission already owns main's fixed-domain mutation
+    // fence, so reacquiring that domain as a write lock would self-deadlock.
+    let remote_tuple_publication_fence = store
+        .acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
+        .await?;
     let has_part_checksums = object_info
         .parts
         .iter()
@@ -1656,8 +1664,8 @@ async fn migrate_object_inner(
             }
             let mut cleanup_opts =
                 data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut cleanup_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut cleanup_opts);
@@ -1684,7 +1692,12 @@ async fn migrate_object_inner(
             }
         }
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
+            .handle_new_multipart_upload_with_pool_idx(
+                &bucket,
+                &object_info.name,
+                &new_multipart_opts,
+                mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
+            )
             .await
         {
             Ok(res) => res,
@@ -1797,15 +1810,20 @@ async fn migrate_object_inner(
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut complete_multipart_opts);
             }
+            let remote_tuple_publication_fence = match mutation_fence.take() {
+                Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+                None => remote_tuple_publication_fence,
+            };
             if let Err(err) = store
                 .clone()
-                .complete_multipart_upload_for_data_movement(
-                    (target_pool_idx, mutation_fence),
+                .complete_multipart_upload_for_data_movement_with_publication_fence(
+                    target_pool_idx,
                     &bucket,
                     &object_info.name,
                     &res.upload_id,
                     parts,
                     &complete_multipart_opts,
+                    remote_tuple_publication_fence,
                 )
                 .await
             {
@@ -1849,8 +1867,8 @@ async fn migrate_object_inner(
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
             let mut abort_opts =
                 data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(fence) = mutation_fence {
-                fence.add_namespace_lock_fence(&mut abort_opts);
+            if let Some(anchor) = mutation_fence.as_ref() {
+                anchor.guard().add_namespace_lock_fence(&mut abort_opts);
             }
             if let Some(fence) = multipart_mutation_fence.as_ref() {
                 fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1923,12 +1941,12 @@ async fn migrate_object_inner(
 
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
-                #[cfg(test)]
+                #[cfg(all(test, feature = "test-util"))]
                 pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
                 let mut abort_opts =
                     data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-                if let Some(fence) = mutation_fence {
-                    fence.add_namespace_lock_fence(&mut abort_opts);
+                if let Some(anchor) = mutation_fence.as_ref() {
+                    anchor.guard().add_namespace_lock_fence(&mut abort_opts);
                 }
                 if let Some(fence) = multipart_mutation_fence.as_ref() {
                     fence.add_namespace_lock_fence(&mut abort_opts);
@@ -1982,8 +2000,18 @@ async fn migrate_object_inner(
     if let Some(signal) = lock_lost_signal {
         put_opts.add_namespace_lock_lost_signal(signal);
     }
+    let remote_tuple_publication_fence = match mutation_fence.take() {
+        Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
+        None => remote_tuple_publication_fence,
+    };
     let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
+        .put_object_for_data_movement_with_publication_fence(
+            &bucket,
+            &object_info.name,
+            &mut data,
+            &put_opts,
+            remote_tuple_publication_fence,
+        )
         .await
         .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
     if let Err(err) = put_result {

crates/ecstore/src/disk/disk_store.rs

@@ -2435,6 +2435,25 @@ mod tests {
         assert_eq!(window.acc_time, 18_000);
     }
 
+    #[test]
+    fn timed_action_slot_snapshot_skips_writer_owned_slot() {
+        let slot = TimedActionSlot::default();
+        slot.unix_sec.store(70, Ordering::Relaxed);
+        slot.count.store(2, Ordering::Relaxed);
+        slot.acc_time.store(18_000, Ordering::Relaxed);
+        slot.version.store(2, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
+
+        assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
+        slot.unix_sec.store(71, Ordering::Relaxed);
+        slot.count.store(1, Ordering::Relaxed);
+        slot.acc_time.store(11_000, Ordering::Relaxed);
+        assert_eq!(slot.snapshot(), None);
+
+        slot.version.store(4, Ordering::Release);
+        assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
+    }
+
     #[test]
     fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
         let metrics = DiskHealthMetricEpoch::default();

crates/ecstore/src/disk/local.rs

@@ -6444,6 +6444,9 @@ impl LocalDisk {
                     .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
                     .await);
             }
+            if should_fail_after_delete_commit(self.root.as_path(), path) {
+                return Err(DiskError::Unexpected);
+            }
             return Ok(());
         }
 
@@ -6514,6 +6517,10 @@ impl LocalDisk {
                 .await);
         }
 
+        if should_fail_after_delete_commit(self.root.as_path(), path) {
+            return Err(DiskError::Unexpected);
+        }
+
         Ok(())
     }
 
@@ -7092,6 +7099,9 @@ impl LocalDisk {
                                 .await?
                         {
                             meta.name.push_str(SLASH_SEPARATOR);
+                            // Conservative listings verify physical prefixes. Never-versioned
+                            // buckets use the bounded fast path and reclaim residue after an
+                            // exact recursive listing proves that prefix empty.
                             if opts.recursive
                                 || opts.incl_deleted
                                 || opts.skip_hidden_prefix_check
@@ -17619,7 +17629,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_scan_dir_nonrecursive_visible_prefix_probe_cost() {
+    async fn test_scan_dir_nonrecursive_fast_path_preserves_probe_bound() {
         use rustfs_filemeta::MetacacheReader;
         use tempfile::tempdir;
 
@@ -17651,6 +17661,10 @@ mod test {
             expected_names.push(format!("{prefix}/"));
         }
 
+        fs::create_dir_all(bucket_dir.join("stale/nested/residue"))
+            .await
+            .expect("stale backing directory should be created");
+
         async fn scan_prefixes(disk: &LocalDisk, bucket: &str, skip_hidden_prefix_check: bool) -> (Vec<String>, usize) {
             let probe_count = Arc::new(AtomicUsize::new(0));
             let (reader, mut writer) = tokio::io::duplex(64 * 1024);
@@ -17693,8 +17707,11 @@ mod test {
         let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
 
         assert_eq!(conservative_names, expected_names);
-        assert_eq!(fast_path_names, expected_names);
-        assert_eq!(conservative_probes, PREFIX_COUNT * 3);
+        let mut expected_fast_path_names = expected_names.clone();
+        expected_fast_path_names.push("stale/".to_owned());
+        assert_eq!(fast_path_names, expected_fast_path_names);
+        let expected_probes = PREFIX_COUNT * 3 + 3;
+        assert_eq!(conservative_probes, expected_probes);
         assert_eq!(fast_path_probes, 0);
     }
 

crates/ecstore/src/disk/os.rs

@@ -298,6 +298,39 @@ pub(crate) mod windows_rename_test_hooks {
     }
 }
 
+/// Test-only hooks into the destination-parent walk of rename preparation.
+///
+/// The prune race lives between two syscalls inside
+/// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
+/// point reproduces it deterministically. Hooks are keyed by the absolute path
+/// of the component just opened and queued per path: a retrying preparation
+/// visits the same component again, so a test models a pruner that keeps
+/// walking upward by queueing one hook per visit.
+#[cfg(all(test, unix))]
+pub(crate) mod prepare_rename_test_hooks {
+    use super::*;
+
+    type Hook = Box<dyn FnOnce() + Send>;
+
+    static AFTER_COMPONENT_OPENED: LazyLock<Mutex<HashMap<PathBuf, VecDeque<Hook>>>> =
+        LazyLock::new(|| Mutex::new(HashMap::new()));
+
+    pub(crate) fn queue_after_component_opened(path: &Path, hook: impl FnOnce() + Send + 'static) {
+        AFTER_COMPONENT_OPENED
+            .lock()
+            .entry(path.to_path_buf())
+            .or_default()
+            .push_back(Box::new(hook));
+    }
+
+    pub(crate) fn run_after_component_opened(path: &Path) {
+        let hook = AFTER_COMPONENT_OPENED.lock().get_mut(path).and_then(VecDeque::pop_front);
+        if let Some(hook) = hook {
+            hook();
+        }
+    }
+}
+
 /// Fsync a directory so recently created or renamed entries survive power loss.
 /// No-op on non-Unix platforms where directories cannot be opened for syncing.
 pub fn fsync_dir_std(dir: impl AsRef<Path>) -> io::Result<()> {
@@ -1905,8 +1938,8 @@ pub(crate) async fn rename_all_with_prepared_source(
         let base_dir = base_dir.clone();
         move || {
             validate_prepared_rename_source(&prepared_source, &src_file_path)?;
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2008,8 +2041,8 @@ async fn reliable_rename_inner_with_lease(
         let dst_file_path = dst_file_path.clone();
         let base_dir = base_dir.clone();
         move || {
-            let (preparation, attempt) = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
-            rename_prepared(&src_file_path, &dst_file_path, &preparation, attempt)
+            let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
+            rename_prepared(&src_file_path, &dst_file_path, &preparation)
         }
     };
     let result = run_blocking_namespace_operation(lease, operation).await;
@@ -2233,12 +2266,13 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
     let mut attempt = 0;
     loop {
         match prepare_rename(src_file_path, dst_file_path, base_dir, publication_root) {
-            Ok(preparation) => return Ok((preparation, attempt)),
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Ok(preparation) => return Ok(preparation),
+            Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                 attempt += 1;
             }
             Err(err) => return Err(err),
@@ -2252,23 +2286,26 @@ fn prepare_rename_with_retry(
     dst_file_path: &Path,
     base_dir: &Path,
     publication_root: &PublicationRoot,
-) -> io::Result<(RenamePreparation, usize)> {
+) -> io::Result<RenamePreparation> {
     let source_parent = src_file_path
         .parent()
         .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
     let destination_parent = dst_file_path.parent();
-    let mut attempt = 0;
-    let prepare_destination_parent = |attempt: &mut usize| -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+    let prune_budget = prepare_prune_budget(dst_file_path, base_dir);
+    // The destination walk and the source open below keep separate counters:
+    // exhausting one must not deny the other its own retry.
+    let prepare_destination_parent = || -> io::Result<Option<ExistingBaseDirectoryGuard>> {
+        let mut attempt = 0;
         loop {
             let result = destination_parent
                 .map(|parent| mkdir_all_below_existing_base_std(parent, base_dir, publication_root))
                 .transpose();
             match result {
                 Ok(parent_guard) => break Ok(parent_guard),
-                Err(err) if should_retry_rename(&err, *attempt) => {
+                Err(err) if should_retry_prepare(&err, attempt, prune_budget) => {
                     #[cfg(test)]
                     windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                    *attempt += 1;
+                    attempt += 1;
                 }
                 Err(err) => break Err(err),
             }
@@ -2281,7 +2318,7 @@ fn prepare_rename_with_retry(
         None => false,
     };
     let (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity) = if same_parent {
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         let source_parent_guard = parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?
@@ -2293,16 +2330,17 @@ fn prepare_rename_with_retry(
         let source_parent_guard = lock_windows_directory_tree(source_parent, destination_parent, publication_root)?;
         let (source_identity_anchor, expected_source_identity) =
             open_windows_rename_source_identity(src_file_path, &source_parent_guard)?;
-        let parent_guard = prepare_destination_parent(&mut attempt)?;
+        let parent_guard = prepare_destination_parent()?;
         (source_parent_guard, parent_guard, source_identity_anchor, expected_source_identity)
     };
+    let mut source_attempt = 0;
     let source = loop {
         match open_windows_rename_source(src_file_path, &source_parent_guard) {
             Ok(source) => break source,
-            Err(err) if should_retry_rename(&err, attempt) => {
+            Err(err) if should_retry_rename(&err, source_attempt) => {
                 #[cfg(test)]
                 windows_rename_test_hooks::run_before_rename_retry(dst_file_path);
-                attempt += 1;
+                source_attempt += 1;
             }
             Err(err) => return Err(err),
         }
@@ -2315,14 +2353,11 @@ fn prepare_rename_with_retry(
     }
     drop(source_identity_anchor);
 
-    Ok((
-        RenamePreparation {
-            parent_guard,
-            _source_parent_guard: source_parent_guard,
-            source,
-        },
-        attempt,
-    ))
+    Ok(RenamePreparation {
+        parent_guard,
+        _source_parent_guard: source_parent_guard,
+        source,
+    })
 }
 
 #[cfg(not(windows))]
@@ -2339,24 +2374,22 @@ fn prepare_rename(
     Ok(RenamePreparation { parent_guard })
 }
 
-fn rename_prepared(
-    _src_file_path: &Path,
-    dst_file_path: &Path,
-    preparation: &RenamePreparation,
-    attempt: usize,
-) -> io::Result<()> {
+/// Publish a prepared rename. The retry budget starts fresh here: preparation
+/// keeps its own counter, so a chain rebuilt after a concurrent prune must not
+/// cost the rename its one retry.
+fn rename_prepared(_src_file_path: &Path, dst_file_path: &Path, preparation: &RenamePreparation) -> io::Result<()> {
     #[cfg(windows)]
     {
         let parent_guard = preparation
             .parent_guard
             .as_ref()
             .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a parent directory"))?;
-        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, attempt)
+        rename_windows_prepared(dst_file_path, parent_guard, &preparation.source, 0)
     }
 
     #[cfg(not(windows))]
     {
-        let mut attempt = attempt;
+        let mut attempt = 0;
         loop {
             let rename_result = rename_into_existing_parent(_src_file_path, dst_file_path, preparation.parent_guard.as_ref());
             match rename_result {
@@ -3756,6 +3789,8 @@ pub(crate) fn mkdir_all_below_existing_base_std(
         let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
         let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
 
+        #[cfg(test)]
+        let mut walked_path = base_dir.to_path_buf();
         for component in relative.components() {
             let Component::Normal(component) = component else {
                 continue;
@@ -3769,6 +3804,11 @@ pub(crate) fn mkdir_all_below_existing_base_std(
                 Err(err) => return Err(err.into()),
             }
             parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
+            #[cfg(test)]
+            {
+                walked_path.push(component);
+                prepare_rename_test_hooks::run_after_component_opened(&walked_path);
+            }
         }
 
         Ok(parents)
@@ -3866,11 +3906,51 @@ fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base
 /// cleanup renames (e.g. `move_to_trash` on an already-removed tmp path) a
 /// pointless second syscall. This predicate is shared by the `rename_data`
 /// commit path via `rename_all`, so any relaxation here must keep genuine
-/// transient errors retryable.
+/// transient errors retryable. The *preparation* phase deliberately uses
+/// [`should_retry_prepare`] instead — see there for why `NotFound` is
+/// recoverable while the destination parent chain is still being built.
 fn should_retry_rename(err: &io::Error, attempt: usize) -> bool {
     attempt == 0 && err.kind() != io::ErrorKind::NotFound
 }
 
+/// How many times rename preparation may retry a `NotFound`.
+///
+/// A pruning walk (`LocalDisk::delete_file`) removes empty ancestors
+/// monotonically upward and stops at the volume root, so it can invalidate
+/// each component *below* the base at most once. One attempt per such
+/// component therefore outlasts a pruning walk, and concurrent walks only
+/// steal an attempt by making that same upward progress. A destination whose
+/// parent *is* the base gets a budget of zero, keeping `NotFound` immediately
+/// terminal for speculative cleanup renames.
+fn prepare_prune_budget(dst_file_path: &Path, base_dir: &Path) -> usize {
+    dst_file_path
+        .parent()
+        .and_then(|parent| parent.strip_prefix(base_dir).ok())
+        .map(|relative| relative.components().count())
+        .unwrap_or(0)
+}
+
+/// Whether a failed rename *preparation* attempt (building the destination
+/// parent chain) should be retried.
+///
+/// Unlike [`should_retry_rename`], `NotFound` is recoverable here: a concurrent
+/// delete prunes now-empty parent directories, so it can unlink an intermediate
+/// destination component between this walk opening a directory and creating the
+/// next child inside it, which a handle-relative `mkdirat`/`openat` reports as
+/// `NotFound`. Each retry rebuilds the whole chain from the base directory,
+/// which no walk below it can remove; `prune_budget` bounds how far a pruner
+/// can push the walk back. A genuinely missing base directory fails identically
+/// on every attempt — the base is only ever opened, never created — so the
+/// missing-base contract holds at the cost of a few extra syscalls on an
+/// already-failing path.
+fn should_retry_prepare(err: &io::Error, attempt: usize, prune_budget: usize) -> bool {
+    if err.kind() == io::ErrorKind::NotFound {
+        attempt < prune_budget
+    } else {
+        attempt == 0
+    }
+}
+
 pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
     let mut i = 0;
 
@@ -4401,6 +4481,141 @@ mod tests {
         assert!(!should_retry_rename(&denied, 1));
     }
 
+    #[test]
+    fn prepare_retry_budget_covers_every_prunable_component() {
+        // A pruner can invalidate each component below the base once, so the
+        // budget must match the chain depth, not a fixed count.
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "pruned");
+        assert!(should_retry_prepare(&not_found, 0, 2));
+        assert!(should_retry_prepare(&not_found, 1, 2));
+        assert!(!should_retry_prepare(&not_found, 2, 2));
+    }
+
+    #[test]
+    fn prepare_retry_keeps_other_errors_at_a_single_retry() {
+        // Only a prune produces a recoverable NotFound; everything else keeps
+        // the historical single retry so persistent failures stay cheap.
+        let denied = io::Error::new(io::ErrorKind::PermissionDenied, "denied");
+        assert!(should_retry_prepare(&denied, 0, 3));
+        assert!(!should_retry_prepare(&denied, 1, 3));
+    }
+
+    #[test]
+    fn prepare_retry_budget_is_zero_when_the_parent_is_the_base() {
+        // Speculative cleanup renames (move_to_trash) land directly in their
+        // base, so a NotFound there is a missing base: terminal, not a prune.
+        let base = Path::new("/vol");
+        assert_eq!(prepare_prune_budget(Path::new("/vol/entry"), base), 0);
+        assert_eq!(prepare_prune_budget(Path::new("/vol/data-movement/sha/id/xl.meta"), base), 3);
+        let not_found = io::Error::new(io::ErrorKind::NotFound, "missing base");
+        assert!(!should_retry_prepare(&not_found, 0, 0));
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_concurrent_empty_parent_prune() {
+        // A multipart staging cleanup prunes the momentarily empty shared
+        // `data-movement/` prefix while a concurrent upload publishes its
+        // xl.meta below that same prefix. The writer's walk holds an fd to the
+        // pruned component, so its next handle-relative mkdirat fails
+        // NotFound; preparation must rebuild the chain and still publish.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let pruned = shared.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            // The cleanup chain's empty-parent prune lands after the writer
+            // opened the shared component but before it creates its child.
+            std::fs::remove_dir(&pruned).expect("prune the empty shared prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a concurrently pruned intermediate directory must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_survives_a_prune_walking_up_every_shared_component() {
+        // The multipart data-movement chain has TWO shared components below the
+        // volume (`data-movement/` and the per-object `<sha>/`), so one cleanup
+        // walk pruning upward can invalidate the writer twice: once at <sha>,
+        // then again at data-movement while the writer rebuilds. A budget that
+        // covers only a single component would still break write quorum here.
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let movement = base.join("data-movement");
+        let sha = movement.join("sha");
+        std::fs::create_dir_all(&sha).expect("create shared chain");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = sha.join("upload-id").join("xl.meta");
+
+        // First visit of `data-movement` is the writer's initial walk, which the
+        // pruner has not reached yet; it prunes on the writer's rebuild.
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, || {});
+        let pruned_sha = sha.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&sha, move || {
+            std::fs::remove_dir(&pruned_sha).expect("prune the empty per-object prefix");
+        });
+        let pruned_movement = movement.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&movement, move || {
+            std::fs::remove_dir(&pruned_movement).expect("prune the empty data-movement prefix");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect("a prune walking up the whole shared chain must not fail the publish");
+
+        assert_eq!(std::fs::read(&dst).expect("read published meta"), b"payload");
+        assert!(!src.exists(), "publish must consume the staged source");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_rejects_a_symlink_swapped_in_between_prepare_attempts() {
+        // The retry must not become a traversal window: replacing the pruned
+        // component with a symlink out of the volume before the rebuilt walk
+        // reopens it must fail closed, exactly as a symlink staged before the
+        // first attempt does.
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("multipart-volume");
+        let shared = base.join("data-movement");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir_all(&shared).expect("create shared prefix");
+        std::fs::create_dir_all(&outside).expect("create outside target");
+        let src = temp_dir.path().join("staged.meta");
+        std::fs::write(&src, b"payload").expect("write staged meta");
+        let dst = shared.join("sha").join("upload-id").join("xl.meta");
+
+        let swapped = shared.clone();
+        let outside_target = outside.clone();
+        prepare_rename_test_hooks::queue_after_component_opened(&shared, move || {
+            std::fs::remove_dir(&swapped).expect("prune the shared prefix");
+            symlink(&outside_target, &swapped).expect("replace the pruned component with a symlink");
+        });
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect_err("a symlink swapped in between attempts must not be followed");
+
+        assert!(src.exists(), "rejected publish must preserve the staged source");
+        assert!(
+            !outside.join("sha").exists(),
+            "the rebuilt walk must not create or publish through the replacement symlink"
+        );
+    }
+
     #[test]
     fn is_dir_not_empty_error_recognizes_directory_not_empty_kind() {
         let err = io::Error::from(io::ErrorKind::DirectoryNotEmpty);

crates/ecstore/src/error/conversion_roundtrip_tests.rs

@@ -278,3 +278,17 @@ fn reduce_errs_buckets_identical_other_messages_together() {
     assert_eq!(count, 3);
     assert_eq!(err, Some(DiskError::other("can not get client")));
 }
+
+#[test]
+fn stable_io_context_buckets_by_cause_and_preserves_diagnostic_source() {
+    let first = StorageError::other_with_context("tier mutation intent changed", "mutation-a");
+    let second = StorageError::other_with_context("tier mutation intent changed", "mutation-b");
+
+    assert_eq!(first, second, "diagnostic identity must not split quorum buckets");
+    let StorageError::Io(io_error) = first else {
+        panic!("stable context must remain an io error");
+    };
+    assert_eq!(io_error.to_string(), "tier mutation intent changed");
+    let context = io_error.get_ref().expect("stable context must remain downcastable");
+    assert_eq!(context.source().expect("diagnostic source must be retained").to_string(), "mutation-a");
+}

crates/ecstore/src/error/mod.rs

@@ -23,6 +23,36 @@ use s3s::S3ErrorCode;
 pub type Error = StorageError;
 pub type Result<T> = core::result::Result<T, Error>;
 
+/// Keeps high-cardinality diagnostic detail in the error source while making
+/// the rendered `io::Error` stable for quorum aggregation.
+#[derive(Debug)]
+struct StableIoContextError {
+    message: &'static str,
+    source: Box<dyn std::error::Error + Send + Sync>,
+}
+
+impl std::fmt::Display for StableIoContextError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(self.message)
+    }
+}
+
+impl std::error::Error for StableIoContextError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(self.source.as_ref())
+    }
+}
+
+pub(crate) fn stable_io_error<E>(message: &'static str, source: E) -> std::io::Error
+where
+    E: Into<Box<dyn std::error::Error + Send + Sync>>,
+{
+    std::io::Error::other(StableIoContextError {
+        message,
+        source: source.into(),
+    })
+}
+
 /// Storage layer error type covering disk, volume, bucket, object, multipart,
 /// erasure-coding, and operational error conditions.
 ///
@@ -264,6 +294,13 @@ impl StorageError {
         StorageError::Io(std::io::Error::other(error))
     }
 
+    pub(crate) fn other_with_context<E>(message: &'static str, source: E) -> Self
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        StorageError::Io(stable_io_error(message, source))
+    }
+
     pub fn is_not_found(&self) -> bool {
         matches!(
             self,

crates/ecstore/src/object_api/mod.rs

@@ -37,7 +37,9 @@ use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, RestoreS
 use rustfs_rio::Checksum;
 use rustfs_utils::CompressionAlgorithm;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
-use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
+use rustfs_utils::http::{
+    AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS, SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str,
+};
 use rustfs_utils::path::decode_dir_object;
 use std::collections::HashMap;
 use std::fmt::Debug;

crates/ecstore/src/object_api/types.rs

@@ -19,6 +19,7 @@ use crate::storage_api_contracts::{
         HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
     },
 };
+use std::io;
 use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
 use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard};
 use tokio_util::sync::CancellationToken;
@@ -301,36 +302,23 @@ pub enum LifecycleDeleteAllPhase {
 #[doc(hidden)]
 #[derive(Default)]
 pub struct LifecycleDeleteAllJournalState {
-    prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
     mutation_started: bool,
 }
 
 impl Debug for LifecycleDeleteAllJournalState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LifecycleDeleteAllJournalState")
-            .field("prepared_count", &self.prepared.len())
             .field("mutation_started", &self.mutation_started)
             .finish()
     }
 }
 
 impl LifecycleDeleteAllJournalState {
-    pub(crate) fn contains(&self, name: &str) -> bool {
-        self.prepared.contains_key(name)
-    }
-
-    pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
-        self.prepared.insert(name, entry);
-    }
-
-    pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
-        self.prepared.values().cloned().collect()
-    }
-
     pub(crate) fn mark_mutation_started(&mut self) {
         self.mutation_started = true;
     }
 
+    #[cfg(test)]
     pub(crate) fn mutation_started(&self) -> bool {
         self.mutation_started
     }
@@ -691,6 +679,196 @@ pub struct DecommissionCapacityOptions {
     pub(crate) mutation_id: Option<Uuid>,
 }
 
+/// Opaque storage-owned collection point for post-commit tier free-version
+/// cleanup receipts. This type is public only because workspace crates build
+/// [`ObjectOptions`] with struct literals; callers outside `ecstore` must leave
+/// the corresponding option unset.
+#[doc(hidden)]
+#[derive(Clone)]
+pub struct TierFreeVersionReceiptSink {
+    inner: Arc<parking_lot::Mutex<TierFreeVersionReceiptSinkState>>,
+}
+
+struct TierFreeVersionReceiptSinkState {
+    receipts: Option<HashMap<TierFreeVersionReceiptIdentity, TierFreeVersionReceiptPayload>>,
+}
+
+#[derive(PartialEq, Eq, Hash)]
+struct TierFreeVersionReceiptIdentity {
+    bucket: String,
+    logical_name: String,
+    tier: String,
+    remote_name: String,
+    remote_version_state: TierFreeVersionReceiptVersionState,
+    remote_version: String,
+    backend_identity: crate::services::tier::tier::TierDestinationId,
+}
+
+struct TierFreeVersionReceiptPayload {
+    local_free_version_id: Uuid,
+    mod_time: Option<OffsetDateTime>,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+enum TierFreeVersionReceiptVersionState {
+    KnownDisabled,
+    SuspendedNull,
+    Exact,
+}
+
+impl TierFreeVersionReceiptSink {
+    /// Only the delete wrapper may originate a sink. The public type exists so
+    /// workspace struct literals can carry it, but external crates cannot
+    /// create an undrainable collector accidentally.
+    pub(crate) fn new() -> Self {
+        Self {
+            inner: Arc::new(parking_lot::Mutex::new(TierFreeVersionReceiptSinkState {
+                receipts: Some(HashMap::new()),
+            })),
+        }
+    }
+}
+
+impl std::fmt::Debug for TierFreeVersionReceiptSink {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let state = self.inner.lock();
+        f.debug_struct("TierFreeVersionReceiptSink")
+            .field("drained", &state.receipts.is_none())
+            .field("receipt_count", &state.receipts.as_ref().map(HashMap::len).unwrap_or_default())
+            .finish()
+    }
+}
+
+impl TierFreeVersionReceiptVersionState {
+    fn persisted(self) -> rustfs_filemeta::TransitionVersionState {
+        match self {
+            Self::KnownDisabled => rustfs_filemeta::TransitionVersionState::KnownDisabled,
+            Self::SuspendedNull => rustfs_filemeta::TransitionVersionState::SuspendedNull,
+            Self::Exact => rustfs_filemeta::TransitionVersionState::Exact,
+        }
+    }
+}
+
+impl TierFreeVersionReceiptIdentity {
+    fn into_object_info(self, payload: TierFreeVersionReceiptPayload) -> ObjectInfo {
+        let mut metadata = HashMap::with_capacity(2);
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(self.backend_identity),
+        );
+        ObjectInfo {
+            bucket: self.bucket,
+            name: self.logical_name,
+            mod_time: payload.mod_time,
+            user_defined: Arc::new(metadata),
+            version_id: Some(payload.local_free_version_id),
+            delete_marker: true,
+            transitioned_object: TransitionedObject {
+                name: self.remote_name,
+                version_id: self.remote_version,
+                tier: self.tier,
+                free_version: true,
+                status: String::new(),
+            },
+            transition_version_state: self.remote_version_state.persisted(),
+            ..Default::default()
+        }
+    }
+}
+
+fn tier_free_version_scheduling_receipt_from_source(
+    source: &ObjectInfo,
+    local_free_version_id: Uuid,
+) -> io::Result<Option<(TierFreeVersionReceiptIdentity, TierFreeVersionReceiptPayload)>> {
+    if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
+        || source.transitioned_object.free_version
+        || source.delete_marker
+        || source.bucket.is_empty()
+        || source.name.is_empty()
+        || source.transitioned_object.tier.is_empty()
+        || source.transitioned_object.name.is_empty()
+    {
+        return Ok(None);
+    }
+    if local_free_version_id.is_nil() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "tier free-version receipt has a nil local version identity",
+        ));
+    }
+
+    let remote_version = source.transitioned_object.version_id.as_str();
+    let remote_version_state = match source.transition_version_state {
+        rustfs_filemeta::TransitionVersionState::Unknown => return Ok(None),
+        rustfs_filemeta::TransitionVersionState::KnownDisabled if remote_version.is_empty() => {
+            TierFreeVersionReceiptVersionState::KnownDisabled
+        }
+        rustfs_filemeta::TransitionVersionState::SuspendedNull if remote_version == "null" => {
+            TierFreeVersionReceiptVersionState::SuspendedNull
+        }
+        rustfs_filemeta::TransitionVersionState::Exact if !remote_version.is_empty() && remote_version != "null" => {
+            TierFreeVersionReceiptVersionState::Exact
+        }
+        _ => return Ok(None),
+    };
+
+    let Some(backend_identity) = crate::services::tier::tier::tier_destination_id_from_metadata(&source.user_defined)
+        .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?
+    else {
+        return Ok(None);
+    };
+
+    Ok(Some((
+        TierFreeVersionReceiptIdentity {
+            bucket: source.bucket.clone(),
+            logical_name: decode_dir_object(&source.name),
+            tier: source.transitioned_object.tier.clone(),
+            remote_name: source.transitioned_object.name.clone(),
+            remote_version_state,
+            remote_version: source.transitioned_object.version_id.clone(),
+            backend_identity,
+        },
+        TierFreeVersionReceiptPayload {
+            local_free_version_id,
+            mod_time: source.mod_time,
+        },
+    )))
+}
+
+impl TierFreeVersionReceiptSink {
+    /// Record one committed free-version cleanup target. Cloned options share
+    /// this sink; tuple-equivalent physical copies collapse to one worker task.
+    /// `false` means the source cannot safely identify a destructive cleanup.
+    pub(crate) fn record(&self, source: &ObjectInfo, local_free_version_id: Uuid) -> io::Result<bool> {
+        let Some((identity, payload)) = tier_free_version_scheduling_receipt_from_source(source, local_free_version_id)? else {
+            return Ok(false);
+        };
+        let mut state = self.inner.lock();
+        let receipts = state
+            .receipts
+            .as_mut()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "tier free-version receipt sink was already drained"))?;
+        receipts.entry(identity).or_insert(payload);
+        Ok(true)
+    }
+
+    /// Consume every receipt exactly once. A second drain is a caller bug: it
+    /// could otherwise make two outer wrappers believe they own the same tasks.
+    pub(crate) fn drain(&self) -> io::Result<Vec<ObjectInfo>> {
+        let mut state = self.inner.lock();
+        let receipts = state
+            .receipts
+            .take()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "tier free-version receipt sink was already drained"))?;
+        drop(state);
+        Ok(receipts
+            .into_iter()
+            .map(|(identity, payload)| identity.into_object_info(payload))
+            .collect())
+    }
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -706,6 +884,12 @@ pub struct ObjectOptions {
     pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
     #[doc(hidden)]
     pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
+    /// Whole-operation authorization created only by consuming a validated
+    /// v6 dispatch-manifest permit. Clones share the authorization, not the
+    /// one-shot permit itself.
+    #[doc(hidden)]
+    pub tier_delete_dispatch_authorization:
+        Option<crate::bucket::lifecycle::tier_delete_journal::TierDeleteDispatchAuthorization>,
     /// RustFS-only compare-and-set condition checked under the object write lock.
     pub expected_current_version_id: Option<String>,
     /// Persisted bucket incarnation observed before authorization.
@@ -723,6 +907,12 @@ pub struct ObjectOptions {
     pub skip_rebalancing: bool,
     pub skip_free_version: bool,
 
+    /// Storage-owned, per-request hand-off for committed tier free-version
+    /// cleanup work. The outer delete wrapper installs and drains it; clones
+    /// below that boundary share the same opaque sink.
+    #[doc(hidden)]
+    pub tier_free_version_receipt_sink: Option<TierFreeVersionReceiptSink>,
+
     /// Cooperative cancellation for an owned PutObject before authoritative
     /// rename begins. Storage ignores it after entering the durable commit.
     #[doc(hidden)]
@@ -847,6 +1037,7 @@ impl std::fmt::Debug for ObjectOptions {
             .field("version_id", &self.version_id.is_some())
             .field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
             .field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
+            .field("tier_delete_dispatch_authorization", &self.tier_delete_dispatch_authorization.is_some())
             .field("expected_current_version_id", &self.expected_current_version_id.is_some())
             .field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
             .field("no_lock", &self.no_lock)
@@ -857,6 +1048,7 @@ impl std::fmt::Debug for ObjectOptions {
             .field("skip_decommissioned", &self.skip_decommissioned)
             .field("skip_rebalancing", &self.skip_rebalancing)
             .field("skip_free_version", &self.skip_free_version)
+            .field("tier_free_version_receipt_sink", &self.tier_free_version_receipt_sink)
             .field("put_object_cancellation", &self.put_object_cancellation.is_some())
             .field("scanner_publication_commit_scope", &self.scanner_publication_commit_scope)
             .field("data_movement", &self.data_movement)
@@ -972,7 +1164,7 @@ impl ObjectOptions {
     }
 
     #[cfg(test)]
-    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+    pub(crate) fn add_namespace_lock_fence(&mut self, fence: &NamespaceLockFence) {
         self.namespace_lock_fence
             .get_or_insert_with(NamespaceLockFence::new)
             .extend(fence);
@@ -1777,9 +1969,10 @@ impl ObjectInfo {
         }
 
         if let Some(data) = &self.checksum {
-            if self.is_encrypted() {
+            if self.is_encrypted() && get_consistent_str(&self.user_defined, SUFFIX_PLAINTEXT_CHECKSUM) != Some("true") {
                 // Object-level encrypted checksum bytes require SSE decrypt material,
-                // so do not expose them as plaintext checksum headers here. The
+                // unless RustFS marked the stored bytes as plaintext. Do not expose
+                // unmarked bytes as checksum headers here. The
                 // `false` multipart flag feeds the response-path COMPOSITE
                 // fallback; callers that need accurate multipart routing must
                 // consult `is_multipart()` instead of this value.
@@ -2485,6 +2678,31 @@ mod tests {
         assert!(checksums.is_empty());
     }
 
+    #[test]
+    fn decrypt_checksums_reads_marked_rustfs_encrypted_object_checksum() {
+        let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")
+            .expect("test checksum should be valid");
+        let checksum_key = checksum.checksum_type.to_string();
+        let expected_checksum = checksum.encoded.clone();
+        let mut user_defined =
+            HashMap::from([(rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())]);
+        rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
+        assert_eq!(user_defined.get("x-rustfs-internal-plaintext-checksum").map(String::as_str), Some("true"));
+        assert_eq!(user_defined.get("x-minio-internal-plaintext-checksum").map(String::as_str), Some("true"));
+        let info = ObjectInfo {
+            checksum: Some(checksum.to_bytes(&[])),
+            user_defined: Arc::new(user_defined),
+            ..Default::default()
+        };
+
+        let (checksums, is_multipart) = info
+            .decrypt_checksums(0, &HeaderMap::new())
+            .expect("marked RustFS checksum should decode");
+
+        assert!(!is_multipart);
+        assert_eq!(checksums.get(&checksum_key), Some(&expected_checksum));
+    }
+
     #[test]
     fn decrypt_checksums_keeps_encrypted_multipart_flag_false_for_response_paths() {
         let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")
@@ -2602,11 +2820,381 @@ mod tests {
         assert!(default_cloned.parts.is_empty());
     }
 
+    fn transitioned_receipt_source(
+        bucket: &str,
+        object: &str,
+        remote_version: &str,
+        version_state: rustfs_filemeta::TransitionVersionState,
+        identity_hex: Option<&str>,
+    ) -> ObjectInfo {
+        let mut metadata = HashMap::new();
+        if let Some(identity_hex) = identity_hex {
+            rustfs_utils::http::metadata_compat::insert_str(
+                &mut metadata,
+                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+                identity_hex.to_string(),
+            );
+        }
+        ObjectInfo {
+            bucket: bucket.to_string(),
+            name: object.to_string(),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            user_defined: Arc::new(metadata),
+            transitioned_object: TransitionedObject {
+                name: format!("remote/{object}"),
+                version_id: remote_version.to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            transition_version_state: version_state,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn tier_free_version_receipt_matches_persisted_free_version_worker_fields() {
+        let bucket = "receipt-bucket";
+        let object = "archive/object.bin";
+        let source_version_id = Uuid::from_u128(1);
+        let local_free_version_id = Uuid::from_u128(2);
+        let remote_version_id = Uuid::from_u128(3);
+        let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::hours(4);
+        let identity_hex = "ab".repeat(32);
+        let mut source_metadata = HashMap::from([
+            ("etag".to_string(), "source-etag".to_string()),
+            ("x-amz-meta-private".to_string(), "must-not-enter-receipt".to_string()),
+        ]);
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut source_metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            identity_hex.clone(),
+        );
+        let source_file_info = FileInfo {
+            volume: bucket.to_string(),
+            name: object.to_string(),
+            version_id: Some(source_version_id),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/receipt-object".to_string(),
+            transition_tier: "WARM".to_string(),
+            transition_version_id: Some(remote_version_id),
+            transition_version: Some(remote_version_id.to_string()),
+            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+            mod_time: Some(source_mod_time),
+            size: 8192,
+            data_dir: Some(Uuid::from_u128(4)),
+            metadata: source_metadata,
+            ..Default::default()
+        };
+        let source = ObjectInfo::from_file_info(&source_file_info, bucket, object, true);
+        let mut persisted = FileMeta::new();
+        persisted
+            .add_version(source_file_info)
+            .expect("transitioned receipt source should be persisted");
+        let mut delete_file_info = FileInfo {
+            volume: bucket.to_string(),
+            name: object.to_string(),
+            version_id: Some(source_version_id),
+            mod_time: Some(source_mod_time + time::Duration::minutes(1)),
+            ..Default::default()
+        };
+        delete_file_info.set_tier_free_version_id(&local_free_version_id.to_string());
+        persisted
+            .delete_version(&delete_file_info)
+            .expect("transitioned source delete should create a free-version");
+
+        let encoded = persisted.marshal_msg().expect("free-version metadata should encode");
+        let decoded = FileMeta::load(&encoded).expect("free-version metadata should decode");
+        let persisted_free_version = decoded
+            .get_all_file_info_versions(bucket, object, true)
+            .expect("decoded free-version should produce FileInfo")
+            .versions
+            .into_iter()
+            .find(|version| version.tier_free_version())
+            .expect("decoded metadata should contain the persisted free-version");
+        let persisted_object_info = ObjectInfo::from_file_info(&persisted_free_version, bucket, object, true);
+
+        let sink = TierFreeVersionReceiptSink::new();
+        assert!(
+            sink.record(&source, local_free_version_id)
+                .expect("valid transitioned source should produce a receipt")
+        );
+        let mut receipts = sink.drain().expect("receipt owner should drain exactly once");
+        assert_eq!(receipts.len(), 1);
+        let receipt = receipts.pop().expect("one receipt should be present");
+
+        assert_eq!(receipt.bucket, persisted_object_info.bucket);
+        assert_eq!(receipt.name, persisted_object_info.name);
+        assert_eq!(receipt.version_id, persisted_object_info.version_id);
+        assert_eq!(receipt.mod_time, persisted_object_info.mod_time);
+        assert_eq!(receipt.delete_marker, persisted_object_info.delete_marker);
+        assert_eq!(receipt.transitioned_object.name, persisted_object_info.transitioned_object.name);
+        assert_eq!(
+            receipt.transitioned_object.version_id,
+            persisted_object_info.transitioned_object.version_id
+        );
+        assert_eq!(receipt.transitioned_object.tier, persisted_object_info.transitioned_object.tier);
+        assert_eq!(
+            receipt.transitioned_object.free_version,
+            persisted_object_info.transitioned_object.free_version
+        );
+        assert_eq!(receipt.transitioned_object.status, persisted_object_info.transitioned_object.status);
+        assert_eq!(receipt.transition_version_state, persisted_object_info.transition_version_state);
+        assert_eq!(
+            crate::services::tier::tier::tier_destination_id_from_metadata(&receipt.user_defined)
+                .expect("receipt identity should decode"),
+            crate::services::tier::tier::tier_destination_id_from_metadata(&persisted_object_info.user_defined)
+                .expect("persisted identity should decode")
+        );
+        assert_eq!(
+            receipt.user_defined.len(),
+            2,
+            "receipt should carry only the two compatibility identity keys"
+        );
+        assert_eq!(
+            receipt.user_defined.get("x-rustfs-internal-transition-tier-destination-id"),
+            Some(&identity_hex)
+        );
+        assert_eq!(
+            receipt.user_defined.get("x-minio-internal-transition-tier-destination-id"),
+            Some(&identity_hex)
+        );
+        assert!(!receipt.user_defined.contains_key("x-amz-meta-private"));
+        assert_eq!(receipt.size, 0);
+        assert_eq!(receipt.actual_size, 0);
+        assert!(receipt.parts.is_empty());
+        assert!(receipt.etag.is_none());
+        assert!(receipt.checksum.is_none());
+        assert!(receipt.data_dir.is_none());
+    }
+
+    #[test]
+    fn tier_free_version_receipt_sink_deduplicates_remote_target_and_drains_once() {
+        let identity_hex = "11".repeat(32);
+        let source = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "remote-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        let other_object = transitioned_receipt_source(
+            "bucket",
+            "other-object",
+            "remote-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        let sink = TierFreeVersionReceiptSink::new();
+        let clone = sink.clone();
+
+        assert!(
+            sink.record(&source, Uuid::from_u128(10))
+                .expect("first physical receipt should record")
+        );
+        assert!(
+            clone
+                .record(&source, Uuid::from_u128(11))
+                .expect("tuple-equivalent physical receipt should be represented")
+        );
+        assert!(
+            clone
+                .record(&other_object, Uuid::from_u128(12))
+                .expect("a different logical key should retain its own task")
+        );
+
+        let mut receipts = sink.drain().expect("owner should drain shared receipts");
+        receipts.sort_by(|left, right| left.name.cmp(&right.name));
+        assert_eq!(receipts.len(), 2);
+        assert_eq!(receipts[0].name, "object");
+        assert_eq!(receipts[0].version_id, Some(Uuid::from_u128(10)));
+        assert_eq!(receipts[1].name, "other-object");
+        assert_eq!(receipts[1].version_id, Some(Uuid::from_u128(12)));
+        assert_eq!(
+            clone.drain().expect_err("a shared sink must drain only once").kind(),
+            io::ErrorKind::BrokenPipe
+        );
+        assert_eq!(
+            clone
+                .record(&source, Uuid::from_u128(13))
+                .expect_err("recording after drain must fail")
+                .kind(),
+            io::ErrorKind::BrokenPipe
+        );
+    }
+
+    #[test]
+    fn tier_free_version_receipt_identity_covers_every_destructive_dimension() {
+        let identity_hex = "44".repeat(32);
+        let baseline = transitioned_receipt_source(
+            "bucket",
+            "directory/",
+            "remote-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        let mut encoded_duplicate = baseline.clone();
+        encoded_duplicate.name = "directory__XLDIR__".to_string();
+
+        let mut variants = Vec::new();
+        let mut changed = baseline.clone();
+        changed.bucket = "other-bucket".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        changed.name = "other-directory/".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        changed.transitioned_object.tier = "COLD".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        changed.transitioned_object.name = "remote/other-directory/".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        changed.transitioned_object.version_id = "other-remote-version".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        changed.transition_version_state = rustfs_filemeta::TransitionVersionState::SuspendedNull;
+        changed.transitioned_object.version_id = "null".to_string();
+        variants.push(changed);
+        let mut changed = baseline.clone();
+        rustfs_utils::http::metadata_compat::insert_str(
+            Arc::make_mut(&mut changed.user_defined),
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "55".repeat(32),
+        );
+        variants.push(changed);
+
+        let sink = TierFreeVersionReceiptSink::new();
+        assert!(
+            sink.record(&baseline, Uuid::from_u128(20))
+                .expect("baseline receipt should record")
+        );
+        assert!(
+            sink.record(&encoded_duplicate, Uuid::from_u128(21))
+                .expect("the encoded spelling of one logical key should deduplicate")
+        );
+        for (offset, variant) in variants.iter().enumerate() {
+            assert!(
+                sink.record(variant, Uuid::from_u128(30 + offset as u128))
+                    .expect("each distinct cleanup identity should record")
+            );
+        }
+
+        let receipts = sink.drain().expect("identity matrix should drain once");
+        assert_eq!(receipts.len(), 8, "every destructive identity dimension must prevent deduplication");
+        let baseline_receipt = receipts
+            .iter()
+            .find(|receipt| {
+                receipt.bucket == "bucket"
+                    && receipt.name == "directory/"
+                    && receipt.transitioned_object.tier == "WARM"
+                    && receipt.transitioned_object.name == "remote/directory/"
+                    && receipt.transitioned_object.version_id == "remote-version"
+                    && receipt.transition_version_state == rustfs_filemeta::TransitionVersionState::Exact
+                    && crate::services::tier::tier::tier_destination_id_from_metadata(&receipt.user_defined)
+                        .is_ok_and(|identity| identity == Some([0x44; 32]))
+            })
+            .expect("baseline cleanup identity should remain present");
+        assert_eq!(
+            baseline_receipt.version_id,
+            Some(Uuid::from_u128(20)),
+            "deduplication must retain the first UUID"
+        );
+    }
+
+    #[test]
+    fn tier_free_version_receipt_source_validation_fails_closed() {
+        let identity_hex = "22".repeat(32);
+        for (state, remote_version) in [
+            (rustfs_filemeta::TransitionVersionState::KnownDisabled, ""),
+            (rustfs_filemeta::TransitionVersionState::SuspendedNull, "null"),
+            (rustfs_filemeta::TransitionVersionState::Exact, "opaque-version"),
+        ] {
+            let source = transitioned_receipt_source("bucket", "object", remote_version, state, Some(&identity_hex));
+            assert!(
+                TierFreeVersionReceiptSink::new()
+                    .record(&source, Uuid::new_v4())
+                    .expect("canonical remote-version state should be eligible"),
+                "state={state:?} remote_version={remote_version:?}"
+            );
+        }
+
+        let unknown = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "opaque-version",
+            rustfs_filemeta::TransitionVersionState::Unknown,
+            Some(&identity_hex),
+        );
+        assert!(
+            !TierFreeVersionReceiptSink::new()
+                .record(&unknown, Uuid::new_v4())
+                .expect("unknown remote version state should defer to recovery")
+        );
+        let missing_identity = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "opaque-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            None,
+        );
+        assert!(
+            !TierFreeVersionReceiptSink::new()
+                .record(&missing_identity, Uuid::new_v4())
+                .expect("missing durable identity should defer to recovery")
+        );
+        let invalid_exact = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        assert!(
+            !TierFreeVersionReceiptSink::new()
+                .record(&invalid_exact, Uuid::new_v4())
+                .expect("conflicting remote state should defer to recovery")
+        );
+
+        let mut conflicting = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "opaque-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        Arc::make_mut(&mut conflicting.user_defined)
+            .insert("x-minio-internal-transition-tier-destination-id".to_string(), "33".repeat(32));
+        assert_eq!(
+            TierFreeVersionReceiptSink::new()
+                .record(&conflicting, Uuid::new_v4())
+                .expect_err("conflicting identity aliases must fail closed")
+                .kind(),
+            io::ErrorKind::InvalidData
+        );
+
+        let valid = transitioned_receipt_source(
+            "bucket",
+            "object",
+            "opaque-version",
+            rustfs_filemeta::TransitionVersionState::Exact,
+            Some(&identity_hex),
+        );
+        assert_eq!(
+            TierFreeVersionReceiptSink::new()
+                .record(&valid, Uuid::nil())
+                .expect_err("nil local free-version identity must be rejected")
+                .kind(),
+            io::ErrorKind::InvalidInput
+        );
+    }
+
     #[test]
     fn object_options_default_does_not_allocate_lifecycle_delete_all_journal() {
         let mut opts = ObjectOptions::default();
 
         assert!(opts.lifecycle_delete_all_journal().is_none());
+        assert!(opts.tier_free_version_receipt_sink.is_none());
         opts.ensure_lifecycle_delete_all_journal();
         assert!(opts.lifecycle_delete_all_journal().is_some());
     }

crates/ecstore/src/services/notification_sys.rs

@@ -29,10 +29,14 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
+use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
-use std::sync::{Arc, Mutex, OnceLock};
+use std::sync::{
+    Arc, Mutex, OnceLock,
+    atomic::{AtomicBool, AtomicUsize, Ordering},
+};
 use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
@@ -52,6 +56,20 @@ const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
 const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
+const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
+type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
+
+fn cross_pool_fence_policy_results(
+    peer_epochs: BTreeMap<String, Uuid>,
+    minimum_version: u32,
+) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
+    let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
+        Ok(peer_epochs.clone())
+    } else {
+        Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
+    };
+    (Ok(peer_epochs), journal_result)
+}
 
 #[derive(Clone, Debug)]
 pub struct ScannerPublicationLeaseGrant {
@@ -107,15 +125,91 @@ struct FleetCapabilityProof {
     topology_fingerprint: String,
     peer_epochs: Arc<BTreeMap<String, Uuid>>,
     expires_at: Instant,
+    generation: Arc<FleetCapabilityProofGeneration>,
 }
 
 impl FleetCapabilityProof {
+    fn new(topology_fingerprint: String, peer_epochs: Arc<BTreeMap<String, Uuid>>, expires_at: Instant) -> Self {
+        Self {
+            topology_fingerprint,
+            peer_epochs,
+            expires_at,
+            generation: FleetCapabilityProofGeneration::fresh(),
+        }
+    }
+
     fn token(&self) -> FleetCapabilityProofToken {
         FleetCapabilityProofToken {
             topology_fingerprint: self.topology_fingerprint.clone(),
             peer_epochs: self.peer_epochs.clone(),
         }
     }
+
+    #[cfg(any(test, feature = "test-util"))]
+    fn with_fresh_generation(&self) -> Self {
+        Self::new(self.topology_fingerprint.clone(), Arc::clone(&self.peer_epochs), self.expires_at)
+    }
+}
+
+/// Admission generation for effects that must not straddle a fleet-proof
+/// replacement. Revocation is deliberately non-blocking: it closes admission
+/// immediately, while the proof slot withholds the successor generation until
+/// every admitted operation has drained.
+#[derive(Default)]
+struct FleetCapabilityProofGeneration {
+    accepting: AtomicBool,
+    active: AtomicUsize,
+}
+
+impl FleetCapabilityProofGeneration {
+    fn fresh() -> Arc<Self> {
+        Arc::new(Self {
+            accepting: AtomicBool::new(true),
+            active: AtomicUsize::new(0),
+        })
+    }
+
+    fn try_acquire(self: &Arc<Self>) -> Option<FleetCapabilityProofPermit> {
+        if !self.accepting.load(Ordering::Acquire) {
+            return None;
+        }
+        self.active.fetch_add(1, Ordering::AcqRel);
+        if self.accepting.load(Ordering::Acquire) {
+            Some(FleetCapabilityProofPermit {
+                generation: Arc::clone(self),
+            })
+        } else {
+            self.release();
+            None
+        }
+    }
+
+    fn revoke(&self) {
+        self.accepting.store(false, Ordering::Release);
+    }
+
+    fn is_accepting(&self) -> bool {
+        self.accepting.load(Ordering::Acquire)
+    }
+
+    fn is_drained(&self) -> bool {
+        self.active.load(Ordering::Acquire) == 0
+    }
+
+    fn release(&self) {
+        let previous = self.active.fetch_sub(1, Ordering::AcqRel);
+        debug_assert!(previous > 0, "fleet capability permit count underflow");
+    }
+}
+
+struct FleetCapabilityProofPermit {
+    generation: Arc<FleetCapabilityProofGeneration>,
+}
+
+impl Drop for FleetCapabilityProofPermit {
+    fn drop(&mut self) {
+        self.generation.release();
+    }
 }
 
 #[derive(Clone, PartialEq, Eq)]
@@ -127,6 +221,7 @@ struct FleetCapabilityProofToken {
 #[derive(Default)]
 struct FleetCapabilityProofState {
     proof: Option<FleetCapabilityProof>,
+    draining_generation: Option<Arc<FleetCapabilityProofGeneration>>,
     topology_conflict: bool,
 }
 
@@ -136,8 +231,17 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
 #[derive(Clone, PartialEq, Eq)]
 pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
 
+/// A point-in-time proof that every current storage member implements the v6
+/// dispatch-manifest policy. It intentionally has no `Clone` implementation:
+/// one acquisition authorizes one manifest construction attempt.
+pub(crate) struct TierDeleteJournalFleetProofToken {
+    token: FleetCapabilityProofToken,
+    _permit: FleetCapabilityProofPermit,
+}
+
 static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
+static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -148,8 +252,35 @@ fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCa
     REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
-fn replace_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>, proof: Option<FleetCapabilityProof>) {
-    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
+    TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
+}
+
+fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
+    if state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        state.draining_generation = None;
+    }
+}
+
+fn revoke_fleet_capability_proof(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    revoke_fleet_capability_proof_state(&mut state);
+}
+
+fn mark_fleet_capability_topology_conflict(slot: &std::sync::RwLock<FleetCapabilityProofState>) {
+    let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.topology_conflict = true;
+    revoke_fleet_capability_proof_state(&mut state);
 }
 
 fn publish_fleet_capability_probe_result(
@@ -161,21 +292,42 @@ fn publish_fleet_capability_probe_result(
     match result {
         Ok(peer_epochs) => {
             let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-            let peer_epochs = state
+            if let Some(current) = state
                 .proof
-                .as_ref()
+                .as_mut()
                 .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
-                .map(|proof| Arc::clone(&proof.peer_epochs))
-                .unwrap_or_else(|| Arc::new(peer_epochs));
-            state.proof = Some(FleetCapabilityProof {
-                topology_fingerprint: topology_fingerprint.to_string(),
-                peer_epochs,
-                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
-            });
+            {
+                current.expires_at = observed_at + REMOTE_VERSION_STATE_PROOF_TTL;
+                return None;
+            }
+
+            if let Some(previous) = state.proof.take() {
+                previous.generation.revoke();
+                if !previous.generation.is_drained() {
+                    state.draining_generation = Some(previous.generation);
+                }
+            }
+            if state
+                .draining_generation
+                .as_ref()
+                .is_some_and(|generation| generation.is_drained())
+            {
+                state.draining_generation = None;
+            }
+            if state.draining_generation.is_some() {
+                return Some(Error::other(
+                    "fleet capability proof successor waits for the previous generation to drain",
+                ));
+            }
+            state.proof = Some(FleetCapabilityProof::new(
+                topology_fingerprint.to_string(),
+                Arc::new(peer_epochs),
+                observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            ));
             None
         }
         Err(err) => {
-            replace_fleet_capability_proof(slot, None);
+            revoke_fleet_capability_proof(slot);
             Some(err)
         }
     }
@@ -216,7 +368,72 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
     fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
 }
 
-#[cfg(test)]
+pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_tier_delete_journal_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_tier_delete_journal_fleet_proof_from(
+    state: &FleetCapabilityProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<TierDeleteJournalFleetProofToken> {
+    let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
+    let permit = state.proof.as_ref()?.generation.try_acquire()?;
+    Some(TierDeleteJournalFleetProofToken { token, _permit: permit })
+}
+
+pub(crate) fn tier_delete_journal_fleet_proof_matches(proof: &TierDeleteJournalFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    tier_delete_journal_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn tier_delete_journal_fleet_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &TierDeleteJournalFleetProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof._permit.generation.is_accepting()
+        && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
+        && state
+            .proof
+            .as_ref()
+            .is_some_and(|current| Arc::ptr_eq(&current.generation, &proof._permit.generation))
+}
+
+pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalFleetProofToken) -> String {
+    stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
+}
+
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
+    let state = tier_delete_journal_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    state.proof.as_ref().is_some_and(|proof| !proof.generation.is_drained())
+        || state
+            .draining_generation
+            .as_ref()
+            .is_some_and(|generation| !generation.is_drained())
+}
+
+fn stable_tier_delete_journal_topology_generation(topology_fingerprint: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(b"rustfs-tier-delete-journal-topology-v1\0");
+    hasher.update(topology_fingerprint.as_bytes());
+    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
+}
+
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
         .get()
@@ -226,18 +443,39 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let now = Instant::now();
+    let proof = if !state.topology_conflict && fleet_capability_proof_valid_at(state.proof.as_ref(), &topology, now) {
+        state.proof.clone()
+    } else {
+        Some(FleetCapabilityProof::new(
+            topology,
+            Arc::new(BTreeMap::new()),
+            now + Duration::from_secs(60 * 60),
+        ))
+    };
     state.topology_conflict = false;
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: topology,
-        peer_epochs: Arc::new(BTreeMap::new()),
-        expires_at: Instant::now() + Duration::from_secs(60 * 60),
-    });
+    state.proof = proof.clone();
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    debug_assert!(
+        journal_state
+            .proof
+            .as_ref()
+            .is_none_or(|current| current.generation.is_drained())
+    );
+    journal_state.topology_conflict = false;
+    journal_state.draining_generation = None;
+    journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
 }
 
 #[cfg(test)]
 pub(crate) struct CrossPoolFenceFleetProofGuard {
     previous_proof: Option<FleetCapabilityProof>,
     previous_topology_conflict: bool,
+    previous_journal_proof: Option<FleetCapabilityProof>,
+    previous_journal_topology_conflict: bool,
 }
 
 #[cfg(test)]
@@ -246,8 +484,24 @@ impl Drop for CrossPoolFenceFleetProofGuard {
         let mut state = cross_pool_fence_fleet_proof_slot()
             .write()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
-        state.proof = self.previous_proof.take();
+        state.proof = self
+            .previous_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        state.draining_generation = None;
         state.topology_conflict = self.previous_topology_conflict;
+        drop(state);
+        let mut journal_state = tier_delete_journal_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        journal_state.proof = self
+            .previous_journal_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        journal_state.draining_generation = None;
+        journal_state.topology_conflict = self.previous_journal_topology_conflict;
     }
 }
 
@@ -258,12 +512,29 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
     let mut state = cross_pool_fence_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
     let guard = CrossPoolFenceFleetProofGuard {
         previous_proof: state.proof.clone(),
         previous_topology_conflict: state.topology_conflict,
+        previous_journal_proof: journal_state.proof.clone(),
+        previous_journal_topology_conflict: journal_state.topology_conflict,
     };
-    state.proof = None;
+    if let Some(proof) = state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            state.draining_generation = Some(proof.generation);
+        }
+    }
     state.topology_conflict = true;
+    if let Some(proof) = journal_state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            journal_state.draining_generation = Some(proof.generation);
+        }
+    }
+    journal_state.topology_conflict = true;
     guard
 }
 
@@ -275,11 +546,33 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
     let Some(current) = state.proof.as_ref() else {
         return false;
     };
-    state.proof = Some(FleetCapabilityProof {
-        topology_fingerprint: current.topology_fingerprint.clone(),
-        peer_epochs: Arc::new(current.peer_epochs.as_ref().clone()),
-        expires_at: current.expires_at,
-    });
+    let proof = FleetCapabilityProof::new(
+        current.topology_fingerprint.clone(),
+        Arc::new(current.peer_epochs.as_ref().clone()),
+        current.expires_at,
+    );
+    state.proof = Some(proof.clone());
+    drop(state);
+    let mut journal_state = tier_delete_journal_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    journal_state.topology_conflict = false;
+    if let Some(previous) = journal_state.proof.take() {
+        previous.generation.revoke();
+        if !previous.generation.is_drained() {
+            journal_state.draining_generation = Some(previous.generation);
+        }
+    }
+    if journal_state
+        .draining_generation
+        .as_ref()
+        .is_some_and(|generation| generation.is_drained())
+    {
+        journal_state.draining_generation = None;
+    }
+    if journal_state.draining_generation.is_none() {
+        journal_state.proof = Some(proof.with_fresh_generation());
+    }
     true
 }
 
@@ -291,15 +584,22 @@ fn fleet_capability_proof_matches(
         return false;
     };
     let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
-    if state.topology_conflict {
-        return false;
-    }
-    state.proof.as_ref().is_some_and(|current| {
-        current.topology_fingerprint == *expected_topology
-            && current.topology_fingerprint == proof.topology_fingerprint
-            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
-            && Instant::now() < current.expires_at
-    })
+    fleet_capability_proof_matches_at(&state, proof, expected_topology, Instant::now())
+}
+
+fn fleet_capability_proof_matches_at(
+    state: &FleetCapabilityProofState,
+    proof: &FleetCapabilityProofToken,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    !state.topology_conflict
+        && state.proof.as_ref().is_some_and(|current| {
+            current.topology_fingerprint == expected_topology
+                && current.topology_fingerprint == proof.topology_fingerprint
+                && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+                && now < current.expires_at
+        })
 }
 
 fn fleet_capability_proof_valid_at(proof: Option<&FleetCapabilityProof>, expected_topology: &str, now: Instant) -> bool {
@@ -312,7 +612,7 @@ pub(crate) struct RemoteVersionStateFleetProofGuard;
 #[cfg(test)]
 impl Drop for RemoteVersionStateFleetProofGuard {
     fn drop(&mut self) {
-        replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
+        revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
     }
 }
 
@@ -348,10 +648,12 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
 pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
     if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
         if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
-            for slot in [remote_version_state_fleet_proof_slot(), cross_pool_fence_fleet_proof_slot()] {
-                let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
-                state.topology_conflict = true;
-                state.proof = None;
+            for slot in [
+                remote_version_state_fleet_proof_slot(),
+                cross_pool_fence_fleet_proof_slot(),
+                tier_delete_journal_fleet_proof_slot(),
+            ] {
+                mark_fleet_capability_topology_conflict(slot);
             }
         }
         return;
@@ -373,7 +675,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 }
                 None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
             };
-            let fence_result = match get_global_notification_sys() {
+            let fence_probe = match get_global_notification_sys() {
                 Some(notification_sys) => timeout(
                     REMOTE_VERSION_STATE_PROBE_TIMEOUT,
                     notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
@@ -382,13 +684,21 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
                 None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
             };
+            let (fence_result, journal_result) = match fence_probe {
+                Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
+                Err(err) => {
+                    let message = err.to_string();
+                    (Err(Error::other(message.clone())), Err(Error::other(message)))
+                }
+            };
             let topology_conflict = remote_version_state_fleet_proof_slot()
                 .read()
                 .unwrap_or_else(std::sync::PoisonError::into_inner)
                 .topology_conflict;
             if topology_conflict {
-                replace_fleet_capability_proof(remote_version_state_fleet_proof_slot(), None);
-                replace_fleet_capability_proof(cross_pool_fence_fleet_proof_slot(), None);
+                revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
+                revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
+                revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -409,7 +719,25 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                     event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_NOTIFICATION,
-                    capability = "cross_pool_fence_v2",
+                    capability = "cross_pool_fence",
+                    state = "failed_closed",
+                    error = %err,
+                    "notification capability probe"
+                );
+            }
+            if !topology_conflict
+                && let Some(err) = publish_fleet_capability_probe_result(
+                    tier_delete_journal_fleet_proof_slot(),
+                    &topology_fingerprint,
+                    journal_result,
+                    Instant::now(),
+                )
+            {
+                debug!(
+                    event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    capability = "tier_delete_journal_v6_policy",
                     state = "failed_closed",
                     error = %err,
                     "notification capability probe"
@@ -483,7 +811,7 @@ impl NotificationSys {
         Ok(peer_epochs)
     }
 
-    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+    async fn probe_cross_pool_fence_fleet(&self, topology_fingerprint: &str) -> Result<(BTreeMap<String, Uuid>, u32)> {
         if self.peer_clients.len() != self.peer_topology_hosts.len() {
             return Err(Error::other("cross-pool fence capability fleet membership is incomplete"));
         }
@@ -494,14 +822,21 @@ impl NotificationSys {
             client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
         });
         let mut peer_epochs = BTreeMap::new();
+        let mut minimum_version = u32::MAX;
         for result in join_all(probes).await {
             let (peer, version, epoch) = result?;
             if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
                 return Err(Error::other("cross-pool fence capability version is unsupported"));
             }
+            minimum_version = minimum_version.min(version);
             insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
         }
-        Ok(peer_epochs)
+        // A single-node deployment has no remote member to lower the local
+        // policy version advertised by this binary.
+        if minimum_version == u32::MAX {
+            minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
+        }
+        Ok((peer_epochs, minimum_version))
     }
 }
 
@@ -1827,12 +2162,13 @@ impl NotificationSys {
         join_all(futures).await
     }
 
-    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
+    pub async fn abort_tier_mutation(&self, mutation_id: Uuid, canonical_prepare_payload: Bytes) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter().cloned() {
+            let payload = canonical_prepare_payload.clone();
             futures.push(async move {
                 if let Some(client) = client {
-                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
+                    notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id, payload).await)
                 } else {
                     unreachable_notification_peer_err()
                 }
@@ -2467,16 +2803,24 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
+        let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
+        let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
+        assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
+        assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
+
+        let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
+        assert!(generic_v3.is_ok());
+        assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
         let now = Instant::now();
         let mut peer_epochs = BTreeMap::new();
         peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(peer_epochs),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(peer_epochs), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
         assert!(!fleet_capability_proof_valid_at(Some(&proof), "topology-b", now));
@@ -2495,11 +2839,7 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_accepts_single_node_membership() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::new()),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new("topology-a".to_string(), Arc::new(BTreeMap::new()), now + Duration::from_secs(1));
 
         assert!(fleet_capability_proof_valid_at(Some(&proof), "topology-a", now));
     }
@@ -2507,21 +2847,97 @@ mod tests {
     #[test]
     fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
         let now = Instant::now();
-        let proof = FleetCapabilityProof {
-            topology_fingerprint: "topology-a".to_string(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: now + Duration::from_secs(1),
-        };
+        let proof = FleetCapabilityProof::new(
+            "topology-a".to_string(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
         let captured = proof.token();
-        let restarted = FleetCapabilityProof {
-            topology_fingerprint: proof.topology_fingerprint.clone(),
-            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
-            expires_at: proof.expires_at,
-        };
+        let restarted = FleetCapabilityProof::new(
+            proof.topology_fingerprint.clone(),
+            Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            proof.expires_at,
+        );
 
         assert!(captured != restarted.token());
     }
 
+    #[test]
+    fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() {
+        let topology = "topology-a";
+        let now = Instant::now();
+        let node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let node_b_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-a".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+        let restarted_node_a_view = FleetCapabilityProof::new(
+            topology.to_string(),
+            Arc::new(BTreeMap::from([("node-b".to_string(), Uuid::new_v4())])),
+            now + Duration::from_secs(1),
+        );
+
+        let generations = [&node_a_view, &node_b_view, &restarted_node_a_view]
+            .map(|proof| stable_tier_delete_journal_topology_generation(&proof.token().topology_fingerprint));
+        assert_eq!(generations[0], generations[1]);
+        assert_eq!(generations[0], generations[2]);
+        assert_ne!(
+            generations[0],
+            stable_tier_delete_journal_topology_generation("topology-b"),
+            "a real topology change must produce a different durable generation"
+        );
+    }
+
+    #[test]
+    fn tier_delete_journal_restart_revokes_old_token_but_fresh_token_recovers_same_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+        let original_generation = stable_tier_delete_journal_topology_generation(&original.topology_fingerprint);
+
+        let restarted_peers = BTreeMap::from([("node-b".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(1))
+                .is_none()
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        let fresh = state
+            .proof
+            .as_ref()
+            .expect("restart probe should publish a fresh proof")
+            .token();
+
+        assert!(!fleet_capability_proof_matches_at(
+            &state,
+            &original,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert!(fleet_capability_proof_matches_at(
+            &state,
+            &fresh,
+            "topology-a",
+            now + Duration::from_millis(2)
+        ));
+        assert_eq!(
+            original_generation,
+            stable_tier_delete_journal_topology_generation(&fresh.topology_fingerprint)
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
         let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
@@ -2561,15 +2977,106 @@ mod tests {
         assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
     }
 
+    #[test]
+    fn tier_delete_journal_successor_waits_for_inflight_generation_to_drain() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original_peers), now).is_none());
+
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(
+                tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+                "a freshly admitted journal proof must remain current"
+            );
+            assert!(
+                acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now + REMOTE_VERSION_STATE_PROOF_TTL,)
+                    .is_none(),
+                "TTL expiry must stop new admission"
+            );
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(
+                    &state,
+                    &admitted,
+                    "topology-a",
+                    now + REMOTE_VERSION_STATE_PROOF_TTL,
+                ),
+                "TTL expiry must also stop an admitted proof at its next durable fence"
+            );
+            assert!(!admitted._permit.generation.is_drained());
+        }
+
+        let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        let blocked = publish_fleet_capability_probe_result(
+            &slot,
+            "topology-a",
+            Ok(restarted_peers.clone()),
+            now + Duration::from_millis(1),
+        )
+        .expect("a successor proof must wait for the admitted generation");
+        assert!(blocked.to_string().contains("previous generation to drain"));
+        {
+            let state = slot.read().expect("proof slot should not poison");
+            assert!(state.proof.is_none(), "new operations must remain closed while the predecessor drains");
+            assert!(state.draining_generation.is_some());
+            assert!(
+                !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now + Duration::from_millis(1),),
+                "a restarted peer must revoke an admitted proof before its next durable fence"
+            );
+        }
+
+        drop(admitted);
+        assert!(
+            publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted_peers), now + Duration::from_millis(2),)
+                .is_none(),
+            "the successor may publish after the in-flight operation releases its permit"
+        );
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.proof.is_some());
+        assert!(state.draining_generation.is_none());
+    }
+
+    #[test]
+    fn tier_delete_journal_topology_conflict_revokes_admitted_generation() {
+        let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
+        let now = Instant::now();
+        let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
+        let admitted = {
+            let state = slot.read().expect("proof slot should not poison");
+            acquire_tier_delete_journal_fleet_proof_from(&state, "topology-a", now)
+                .expect("a fresh proof should admit one journal operation")
+        };
+
+        mark_fleet_capability_topology_conflict(&slot);
+
+        let state = slot.read().expect("proof slot should not poison");
+        assert!(state.topology_conflict);
+        assert!(state.proof.is_none());
+        assert!(state.draining_generation.is_some());
+        assert!(!admitted._permit.generation.is_accepting());
+        assert!(
+            !tier_delete_journal_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
+            "topology conflict must revoke an already admitted journal proof"
+        );
+    }
+
     #[test]
     fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
         let now = Instant::now();
         let mut state = FleetCapabilityProofState {
-            proof: Some(FleetCapabilityProof {
-                topology_fingerprint: "topology-a".to_string(),
-                peer_epochs: Arc::new(BTreeMap::new()),
-                expires_at: now + Duration::from_secs(1),
-            }),
+            proof: Some(FleetCapabilityProof::new(
+                "topology-a".to_string(),
+                Arc::new(BTreeMap::new()),
+                now + Duration::from_secs(1),
+            )),
+            draining_generation: None,
             topology_conflict: false,
         };
         assert!(acquire_fleet_capability_proof_from(&state, "topology-a", now).is_some());
@@ -3279,7 +3786,7 @@ mod tests {
         assert_eq!(commit.len(), 1);
         assert!(commit[0].err.is_some());
 
-        let abort = sys.abort_tier_mutation(mutation_id).await;
+        let abort = sys.abort_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
         assert_eq!(abort.len(), 1);
         assert!(abort[0].err.is_some());
     }

crates/ecstore/src/services/rebalance/control.rs

@@ -1329,8 +1329,9 @@ mod tests {
     use super::*;
     use crate::config::com::delete_config;
     use crate::core::pools::{
-        POOL_META_NAME, PoolActivationDurableSaveBarrier, PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState,
-        persist_pool_meta_identity_for_startup,
+        DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
+        PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
+        set_decommission_capacity_info_overrides_for_test,
     };
     use crate::object_api::NamespaceLockFence;
     use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
@@ -1751,26 +1752,15 @@ mod tests {
         ];
         set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
         set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
-        crate::core::pools::set_decommission_space_info_override_for_test(
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        let capacity_snapshot = vec![
+            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
+            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
+        ];
+        // Decommission start samples capacity before and inside its durable activation fence.
+        set_decommission_capacity_info_overrides_for_test(
             decommission_store.id,
-            vec![
-                (
-                    0,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 0,
-                        total: 100,
-                        used: 100,
-                    },
-                ),
-                (
-                    1,
-                    crate::core::pools::PoolSpaceInfo {
-                        free: 200,
-                        total: 200,
-                        used: 0,
-                    },
-                ),
-            ],
+            vec![capacity_snapshot.clone(), capacity_snapshot],
         );
         let (first_object, competing_object, competing_kind) = match paused_kind {
             PoolActivationStartKind::Rebalance => {

crates/ecstore/src/services/rebalance/entry.rs

@@ -356,7 +356,7 @@ impl ECStore {
             };
             run_guard.ensure_held("rebalance version migration")?;
             let result = migrate_entry_version(
-                &RebalanceMigrationBackend::new(set.as_ref(), self.as_ref(), lock_lost_signal.clone()),
+                &RebalanceMigrationBackend::new(set.as_ref(), self.clone(), lock_lost_signal.clone()),
                 bucket.clone(),
                 pool_index,
                 version,
@@ -1478,6 +1478,7 @@ mod tests {
         let (_temp_dirs, store, _unused_store) =
             crate::services::rebalance::test_two_pool_stores(Some(active_rebalance_meta(REBALANCE_ID))).await;
         prepare_rebalance_test_volumes(store.as_ref()).await;
+        crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), "WARM").await;
         let source_set = store.pools[0].get_disks_by_key(object);
         let target_set = store.pools[1].get_disks_by_key(object);
         let version_id = uuid::Uuid::new_v4();
@@ -1520,14 +1521,17 @@ mod tests {
         let entry = metacache_entry_from_source(source_set.as_ref(), bucket, object).await;
         let run_signal_fence = RebalanceRunSignalTestFence::install(REBALANCE_ID);
         let barrier = TieredMetadataCommitBarrier::install(bucket, object);
-        let task = spawn_real_rebalance_entry(
+        let mut task = spawn_real_rebalance_entry(
             Arc::clone(&store),
             Arc::clone(&source_set),
             entry,
             REBALANCE_ID,
             Arc::new(RebalanceBucketConfigs::default()),
         );
-        barrier.wait_until_paused().await;
+        tokio::select! {
+            _ = barrier.wait_until_paused() => {}
+            result = &mut task => panic!("rebalance exited before the tiered commit barrier: {result:?}"),
+        }
         run_signal_fence.mark_lost();
         barrier.release();
         drop(barrier);

crates/ecstore/src/services/rebalance/migration.rs

@@ -101,14 +101,14 @@ pub(crate) trait MigrationBackend: Send + Sync {
 
 pub(crate) struct RebalanceMigrationBackend<'a> {
     source: &'a SetDisks,
-    store: &'a ECStore,
+    store: std::sync::Arc<ECStore>,
     lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
 }
 
 impl<'a> RebalanceMigrationBackend<'a> {
     pub(crate) fn new(
         source: &'a SetDisks,
-        store: &'a ECStore,
+        store: std::sync::Arc<ECStore>,
         lock_lost_signal: Option<std::sync::Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
     ) -> Self {
         Self {

crates/ecstore/src/services/rebalance/mod.rs

@@ -134,7 +134,7 @@ pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
     test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
 }
 
-#[cfg(test)]
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
     rebalance_meta: Option<RebalanceMeta>,
 ) -> (

crates/ecstore/src/services/tier/mod.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub mod test_util;
 pub mod tier;
 pub mod tier_admin;

crates/ecstore/src/services/tier/tier_mutation_intent.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
 use serde::{Deserialize, Serialize};
@@ -32,9 +32,34 @@ pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-inten
 pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
 pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
 pub(crate) const TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/coordinators";
+pub(crate) const TIER_MUTATION_MUTEX_SHARDS: usize = 64;
 const TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS: usize = 3;
 pub(crate) type TierMutationDigest = [u8; 32];
 
+static TIER_MUTATION_MUTEXES: LazyLock<[tokio::sync::Mutex<()>; TIER_MUTATION_MUTEX_SHARDS]> =
+    LazyLock::new(|| std::array::from_fn(|_| tokio::sync::Mutex::new(())));
+
+/// Serializes every local phase and recovery action for one mutation id while
+/// retaining bounded parallelism for unrelated mutations.
+pub(crate) async fn acquire_tier_mutation_mutex(mutation_id: Uuid) -> tokio::sync::MutexGuard<'static, ()> {
+    TIER_MUTATION_MUTEXES[tier_mutation_mutex_shard_index(mutation_id)]
+        .lock()
+        .await
+}
+
+fn tier_mutation_mutex_shard_index(mutation_id: Uuid) -> usize {
+    let raw = mutation_id.as_u128();
+    let mut mixed = (raw as u64) ^ ((raw >> 64) as u64);
+    // MurmurHash3's 64-bit finalizer gives stable diffusion without allocating
+    // or relying on RandomState, whose seed differs between processes.
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xff51_afd7_ed55_8ccd);
+    mixed ^= mixed >> 33;
+    mixed = mixed.wrapping_mul(0xc4ce_b9fe_1a85_ec53);
+    mixed ^= mixed >> 33;
+    (mixed as usize) & (TIER_MUTATION_MUTEX_SHARDS - 1)
+}
+
 pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
 
 #[derive(Debug, thiserror::Error)]
@@ -265,6 +290,30 @@ impl TierMutationIntent {
             && self.expires_at_unix_nanos == other.expires_at_unix_nanos
     }
 
+    /// Reconstruct the exact Prepared record that originally produced this
+    /// intent. Abort RPCs are identity-bound to that payload; serializing an
+    /// Aborted terminal record would both violate the wire contract and use a
+    /// different revision if a missing peer has to persist a tombstone.
+    pub(crate) fn original_prepared(&self) -> Result<Self> {
+        if self.state == TierMutationIntentState::Prepared {
+            self.validate()?;
+            return Ok(self.clone());
+        }
+        let mut prepared = self.clone();
+        prepared.revision =
+            prepared
+                .revision
+                .checked_sub(1)
+                .filter(|revision| *revision != 0)
+                .ok_or(TierMutationIntentError::Corrupt(
+                    "terminal intent cannot reconstruct its prepared revision",
+                ))?;
+        prepared.state = TierMutationIntentState::Prepared;
+        prepared.committed_config_etag = None;
+        prepared.validate()?;
+        Ok(prepared)
+    }
+
     pub(crate) fn encode(&self) -> Result<Vec<u8>> {
         self.validate()?;
         let intent_bytes = serde_json::to_vec(self)?;
@@ -439,6 +488,16 @@ where
     load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
+pub(crate) async fn load_tier_coordinator_mutation_intent_record_with_etag<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+) -> EcstoreResult<(TierMutationIntent, String)>
+where
+    S: EcstoreObjectIO,
+{
+    load_tier_mutation_intent_record_with_etag_at_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+}
+
 async fn load_tier_mutation_intent_record_with_etag_at_prefix<S>(
     api: Arc<S>,
     prefix: &str,
@@ -507,6 +566,7 @@ where
     .await
 }
 
+#[cfg(test)]
 pub(crate) async fn delete_tier_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -514,13 +574,36 @@ where
     delete_tier_mutation_intent_record_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
 }
 
-pub(crate) async fn delete_tier_coordinator_mutation_intent_record<S>(api: Arc<S>, mutation_id: Uuid) -> EcstoreResult<()>
+pub(crate) async fn delete_tier_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
 {
-    delete_tier_mutation_intent_record_with_prefix(api, TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX, mutation_id).await
+    delete_tier_mutation_intent_record_if_current_with_prefix(api, TIER_MUTATION_INTENT_RECORD_PREFIX, mutation_id, current_etag)
+        .await
 }
 
+pub(crate) async fn delete_tier_coordinator_mutation_intent_record_if_current<S>(
+    api: Arc<S>,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    delete_tier_mutation_intent_record_if_current_with_prefix(
+        api,
+        TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX,
+        mutation_id,
+        current_etag,
+    )
+    .await
+}
+
+#[cfg(test)]
 async fn delete_tier_mutation_intent_record_with_prefix<S>(api: Arc<S>, prefix: &str, mutation_id: Uuid) -> EcstoreResult<()>
 where
     S: EcstoreObjectOperations,
@@ -533,6 +616,40 @@ where
     }
 }
 
+async fn delete_tier_mutation_intent_record_if_current_with_prefix<S>(
+    api: Arc<S>,
+    prefix: &str,
+    mutation_id: Uuid,
+    current_etag: &str,
+) -> EcstoreResult<()>
+where
+    S: EcstoreObjectOperations,
+{
+    if current_etag.trim().is_empty() {
+        return Err(Error::other("tier mutation intent current ETag is empty"));
+    }
+    let object =
+        tier_mutation_intent_record_object_name_with_prefix(prefix, mutation_id).map_err(tier_mutation_intent_store_error)?;
+    match api
+        .delete_object(
+            RUSTFS_META_BUCKET,
+            &object,
+            ObjectOptions {
+                http_preconditions: Some(HTTPPreconditions {
+                    if_match: Some(current_etag.to_string()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(err) if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) => Err(Error::ConfigNotFound),
+        Err(err) => Err(err),
+    }
+}
+
 pub(crate) async fn advance_tier_mutation_intent_record_idempotent<S>(
     api: Arc<S>,
     mutation_id: Uuid,
@@ -713,6 +830,7 @@ fn digest_is_empty(digest: &TierMutationDigest) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::time::Duration;
 
     const OLD_IDENTITY: TierDestinationId = [1; 32];
     const NEW_IDENTITY: TierDestinationId = [2; 32];
@@ -736,6 +854,61 @@ mod tests {
         }
     }
 
+    #[test]
+    fn mutation_mutex_uses_exactly_64_stable_shards() {
+        assert_eq!(TIER_MUTATION_MUTEX_SHARDS, 64);
+        assert_eq!(TIER_MUTATION_MUTEXES.len(), TIER_MUTATION_MUTEX_SHARDS);
+
+        let mutation_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
+        let shard = tier_mutation_mutex_shard_index(mutation_id);
+        assert!(shard < TIER_MUTATION_MUTEX_SHARDS);
+        assert_eq!(shard, tier_mutation_mutex_shard_index(mutation_id));
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_serializes_the_same_id() {
+        let mutation_id = Uuid::new_v4();
+        let first = acquire_tier_mutation_mutex(mutation_id).await;
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            started_tx.send(()).expect("test receiver should remain alive");
+            let _second = acquire_tier_mutation_mutex(mutation_id).await;
+            acquired_tx.send(()).expect("test receiver should remain alive");
+        });
+        started_rx.await.expect("waiter should start");
+        assert!(
+            tokio::time::timeout(Duration::from_millis(25), &mut acquired_rx)
+                .await
+                .is_err(),
+            "the same mutation id must not enter concurrently"
+        );
+
+        drop(first);
+        tokio::time::timeout(Duration::from_secs(1), &mut acquired_rx)
+            .await
+            .expect("waiter should acquire after release")
+            .expect("waiter should report acquisition");
+        waiter.await.expect("waiter task should finish");
+    }
+
+    #[tokio::test]
+    async fn mutation_mutex_allows_different_shards_to_progress() {
+        let first_id = Uuid::new_v4();
+        let first_shard = tier_mutation_mutex_shard_index(first_id);
+        let second_id = (0..1024)
+            .map(|_| Uuid::new_v4())
+            .find(|candidate| tier_mutation_mutex_shard_index(*candidate) != first_shard)
+            .expect("a distinct shard should be easy to find");
+        let first = acquire_tier_mutation_mutex(first_id).await;
+
+        let _second = tokio::time::timeout(Duration::from_secs(1), acquire_tier_mutation_mutex(second_id))
+            .await
+            .expect("a different shard must not wait for the first mutation");
+        drop(first);
+    }
+
     #[test]
     fn intent_round_trip_preserves_committed_state() {
         let mut intent = prepared_intent();
@@ -873,6 +1046,37 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn terminal_intent_reconstructs_original_prepared_abort_payload() {
+        for terminal in [TierMutationIntentState::Aborted, TierMutationIntentState::Committed] {
+            let original = prepared_intent();
+            let mut intent = original.clone();
+            let committed_etag = (terminal == TierMutationIntentState::Committed).then(|| "new-etag".to_string());
+            intent
+                .advance(terminal, committed_etag)
+                .expect("terminal transition should succeed");
+
+            let reconstructed = intent
+                .original_prepared()
+                .expect("terminal record should recover prepared payload");
+            assert_eq!(reconstructed, original);
+            assert!(intent.same_identity_as(&reconstructed));
+        }
+    }
+
+    #[test]
+    fn terminal_intent_with_initial_revision_fails_prepared_reconstruction() {
+        let mut corrupt = prepared_intent();
+        corrupt.state = TierMutationIntentState::Aborted;
+
+        assert!(matches!(
+            corrupt.original_prepared(),
+            Err(TierMutationIntentError::Corrupt(
+                "terminal intent cannot reconstruct its prepared revision"
+            ))
+        ));
+    }
+
     #[test]
     fn intent_validation_rejects_placeholder_identity() {
         let mut intent = prepared_intent();

crates/ecstore/src/services/tier/tier_mutation_peer.rs

@@ -15,12 +15,13 @@
 use std::sync::Arc;
 
 use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
+use time::OffsetDateTime;
 use uuid::Uuid;
 
 use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
 use super::tier_mutation_intent::{
-    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
-    load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
+    MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, acquire_tier_mutation_mutex,
+    advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
 };
 use crate::error::{Error, StorageError};
 use crate::store::ECStore;
@@ -57,6 +58,8 @@ pub enum TierMutationPeerError {
     CommitProofMismatch,
     #[error("tier mutation peer abort proof does not match the persisted tier configuration")]
     AbortProofMismatch,
+    #[error("tier mutation peer prepared intent has expired")]
+    ExpiredIntent,
     #[error("tier mutation peer runtime error: {0}")]
     Runtime(#[source] AdminError),
     #[error("tier mutation peer store error: {0}")]
@@ -79,6 +82,7 @@ pub async fn handle_tier_mutation_peer_request(
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
     validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
+    let _mutation_guard = acquire_tier_mutation_mutex(mutation_id).await;
     match phase {
         TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
         TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
@@ -103,43 +107,57 @@ async fn handle_prepare(
     }
     let tier_config_mgr = api.tier_config_mgr();
 
-    match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
-        Ok(()) => {
-            TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
-                .await
-                .map_err(TierMutationPeerError::Runtime)?;
-            Ok(TierMutationPeerOutcome {
-                state: TierMutationPeerState::Prepared,
-                applied: true,
-            })
-        }
-        Err(Error::PreconditionFailed) => {
-            let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
-            if !existing.same_identity_as(&intent) {
-                return Err(TierMutationPeerError::ConflictingIntent);
+    for _ in 0..3 {
+        let (stored, applied) = match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&intent) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                (existing, false)
             }
-            match existing.state {
-                TierMutationIntentState::Prepared => {
-                    TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
+            Err(Error::ConfigNotFound) => {
+                let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).unwrap_or(i64::MAX);
+                if intent.expires_at_unix_nanos <= now {
+                    return Err(TierMutationPeerError::ExpiredIntent);
                 }
-                TierMutationIntentState::Committed => {
-                    TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
-                        .await
-                        .map_err(TierMutationPeerError::Runtime)?;
-                }
-                TierMutationIntentState::Aborted => {
-                    TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
+                    Ok(()) => (intent.clone(), true),
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
                 }
             }
-            Ok(TierMutationPeerOutcome {
-                state: peer_state_from_intent(existing.state),
-                applied: false,
-            })
+            Err(err) => return Err(err.into()),
+        };
+
+        match stored.state {
+            TierMutationIntentState::Prepared => {
+                TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::wait_for_blocked_tier_operation_leases(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Committed => {
+                TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &stored)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+            }
+            TierMutationIntentState::Aborted => {
+                TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
+                    .await
+                    .map_err(TierMutationPeerError::Runtime)?;
+                TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
+            }
         }
-        Err(err) => Err(err.into()),
+        return Ok(TierMutationPeerOutcome {
+            state: peer_state_from_intent(stored.state),
+            applied,
+        });
     }
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation prepare raced repeatedly with another decision",
+    )))
 }
 
 async fn handle_commit(
@@ -201,26 +219,106 @@ async fn handle_abort(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<TierMutationPeerOutcome> {
-    if !canonical_payload.is_empty() {
-        return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
+    let prepared = TierMutationIntent::decode(mutation_id, canonical_payload)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+    if prepared.state != TierMutationIntentState::Prepared {
+        return Err(TierMutationPeerError::InvalidPayload(
+            "abort payload must carry the original prepared intent".to_string(),
+        ));
     }
-    let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
-    if existing.state == TierMutationIntentState::Prepared
-        && !tier_config_abort_matches(api.clone(), &existing)
-            .await
-            .map_err(Error::other)?
-    {
-        return Err(TierMutationPeerError::AbortProofMismatch);
+    let mut tombstone = prepared.clone();
+    tombstone
+        .advance(TierMutationIntentState::Aborted, None)
+        .map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
+
+    for _ in 0..3 {
+        match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+            Ok(existing) => {
+                if !existing.same_identity_as(&prepared) {
+                    return Err(TierMutationPeerError::ConflictingIntent);
+                }
+                match existing.state {
+                    TierMutationIntentState::Committed => {
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Committed,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Aborted => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: false,
+                        });
+                    }
+                    TierMutationIntentState::Prepared => {}
+                }
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                let advanced = advance_tier_mutation_intent_record_idempotent(
+                    api.clone(),
+                    mutation_id,
+                    TierMutationIntentState::Aborted,
+                    None,
+                )
+                .await;
+                let (intent, applied) = match advanced {
+                    Ok(result) => result,
+                    Err(err) => match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
+                        Ok(current)
+                            if current.same_identity_as(&prepared) && current.state != TierMutationIntentState::Prepared =>
+                        {
+                            (current, false)
+                        }
+                        _ => return Err(err.into()),
+                    },
+                };
+                if intent.state == TierMutationIntentState::Aborted {
+                    TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                    TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                        .await
+                        .map_err(TierMutationPeerError::Runtime)?;
+                }
+                return Ok(TierMutationPeerOutcome {
+                    state: peer_state_from_intent(intent.state),
+                    applied,
+                });
+            }
+            Err(Error::ConfigNotFound) => {
+                if !tier_config_abort_matches(api.clone(), &prepared)
+                    .await
+                    .map_err(Error::other)?
+                {
+                    return Err(TierMutationPeerError::AbortProofMismatch);
+                }
+                match save_tier_mutation_intent_record_if_absent(api.clone(), &tombstone).await {
+                    Ok(()) => {
+                        TierConfigMgr::clear_prepared_mutation_intent_block(&api.tier_config_mgr(), mutation_id)
+                            .await
+                            .map_err(TierMutationPeerError::Runtime)?;
+                        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
+                        return Ok(TierMutationPeerOutcome {
+                            state: TierMutationPeerState::Aborted,
+                            applied: true,
+                        });
+                    }
+                    Err(Error::PreconditionFailed) => continue,
+                    Err(err) => return Err(err.into()),
+                }
+            }
+            Err(err) => return Err(err.into()),
+        }
     }
-    let (intent, applied) =
-        advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
-    if intent.state == TierMutationIntentState::Aborted {
-        TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
-    }
-    Ok(TierMutationPeerOutcome {
-        state: peer_state_from_intent(intent.state),
-        applied,
-    })
+    Err(TierMutationPeerError::Store(Error::other(
+        "tier mutation abort raced repeatedly with prepare",
+    )))
 }
 
 fn validate_peer_request_envelope(
@@ -228,7 +326,10 @@ fn validate_peer_request_envelope(
     mutation_id: Uuid,
     canonical_payload: &[u8],
 ) -> TierMutationPeerResult<()> {
-    if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
+    if !matches!(
+        protocol_version,
+        rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION | TIER_MUTATION_RPC_PROTOCOL_VERSION
+    ) {
         return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
     }
     if mutation_id.is_nil() {
@@ -276,6 +377,8 @@ mod tests {
     #[test]
     fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
         let mutation_id = Uuid::new_v4();
+        validate_peer_request_envelope(rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION, mutation_id, b"payload")
+            .expect("v3 must remain accepted during the v4 rollout");
         assert!(matches!(
             validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
             Err(TierMutationPeerError::UnsupportedProtocolVersion(_))

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -41,22 +41,22 @@ use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
 #[cfg(test)]
 use super::super::get_metadata_slowtail_fault_delay;
 use super::super::{
-    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
-    EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion, GetCodecStreamingFallbackReason,
-    GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource,
-    LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions,
-    ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq,
-    ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError,
-    UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
-    capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason,
-    current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info,
-    inline_erasure_shard_file_offset, inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found,
-    is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
-    is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
-    is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
-    merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
-    reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
-    try_read_inline_data_shards_direct, warn,
+    Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+    EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion,
+    GetCodecStreamingFallbackReason, GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult,
+    HealChannelPriority, HealRequestSource, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext,
+    OBJECT_OP_IGNORED_ERRS, ObjectOptions, ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET,
+    RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks,
+    SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs,
+    can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
+    collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
+    file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
+    inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
+    is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
+    is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
+    issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
+    object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
+    send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
 };
 #[cfg(test)]
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
@@ -3733,16 +3733,34 @@ fn dangling_delete_grace() -> time::Duration {
 /// Result of scanning one disk's copy of a directory prefix while deciding
 /// whether an orphan (metadata-less) directory tree can be safely purged.
 enum OrphanDirScan {
-    /// The subtree holds at least one regular file (object metadata or data), so
-    /// it is a real object and must not be purged.
+    /// The subtree holds object metadata or uncommitted data, so it must not be
+    /// purged.
     HasData,
-    /// The prefix exists on this disk and contains only nested empty directories.
-    /// Carries every directory path in pre-order (parents before children).
-    Empty(Vec<String>),
+    /// The prefix contains only empty directories and/or UUID data directories
+    /// carrying a committed delete marker.
+    Purgeable {
+        empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    },
     /// The prefix does not exist on this disk.
     Missing,
 }
 
+fn is_safe_orphan_dir_entry(entry: &str) -> bool {
+    let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
+    !component.is_empty()
+        && component != "."
+        && component != ".."
+        && !component.contains(SLASH_SEPARATOR)
+        && !component.contains('\\')
+}
+
+fn is_committed_delete_marker(entry: &str) -> bool {
+    entry
+        .strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
+        .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil()))
+}
+
 /// Outcome of a *post-quorum* `rename_data` commit, classifying whether the
 /// committed replicas converged so the caller can decide heal admission
 /// WITHOUT conflating "a version signature exists" with "this write needs
@@ -6125,52 +6143,151 @@ impl SetDisks {
     }
 
     /// Scan a single disk's copy of `prefix` and decide whether it is an orphan
-    /// (metadata-less) directory subtree. Walks the tree iteratively and returns
-    /// [`OrphanDirScan::HasData`] as soon as any regular file is found.
+    /// directory subtree. Only empty directories and UUID data directories with
+    /// valid committed delete markers are purgeable; every child is still scanned.
     async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
         let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
         let mut stack = vec![root.clone()];
         // Pre-order list of directories (a parent always precedes its descendants),
         // so reversing it yields a safe children-first removal order.
         let mut dirs: Vec<String> = Vec::new();
+        let mut committed_files: Vec<String> = Vec::new();
         let mut existed = false;
 
         while let Some(dir) = stack.pop() {
             let entries = match disk.list_dir("", bucket, &dir, 0).await {
                 Ok(entries) => entries,
-                Err(_) => {
-                    // The root missing (or never existing) means there is nothing to
-                    // purge on this disk. A nested directory vanishing mid-scan is a
-                    // benign race, so skip it and keep walking.
+                Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {
                     if dir == root {
                         return OrphanDirScan::Missing;
                     }
+                    // A nested directory vanishing mid-scan is a benign race.
                     continue;
                 }
+                // Classification must fail closed: committed residue is safe to
+                // remove only after every reachable child was inspected.
+                Err(_) => return OrphanDirScan::HasData,
             };
 
             existed = true;
-            dirs.push(dir.clone());
+            let mut child_dirs = Vec::new();
+            let mut files = Vec::new();
 
             for entry in entries {
+                if !is_safe_orphan_dir_entry(&entry) {
+                    return OrphanDirScan::HasData;
+                }
                 match entry.strip_suffix(SLASH_SEPARATOR) {
-                    // `read_dir` marks directories with a trailing slash; anything else
-                    // is a regular file, which means real object data lives here.
-                    Some(child) => stack.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
-                    None => return OrphanDirScan::HasData,
+                    Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
+                    None => files.push(entry),
                 }
             }
+
+            if !files.is_empty() {
+                let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
+                let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
+                let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
+
+                if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
+                    return OrphanDirScan::HasData;
+                }
+
+                committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
+                dirs.push(dir);
+                stack.extend(child_dirs);
+                continue;
+            }
+
+            dirs.push(dir);
+            stack.extend(child_dirs);
         }
 
         if existed {
-            OrphanDirScan::Empty(dirs)
+            OrphanDirScan::Purgeable {
+                empty_dirs: dirs,
+                committed_files,
+            }
         } else {
             OrphanDirScan::Missing
         }
     }
 
+    async fn delete_purgeable_orphan_entries(
+        disk: &DiskStore,
+        bucket: &str,
+        object: &str,
+        mut empty_dirs: Vec<String>,
+        committed_files: Vec<String>,
+    ) {
+        // Keep every committed marker until all ordinary residue files are gone.
+        // If any delete fails, a later request can still recognize and retry the
+        // committed cleanup instead of stranding an unmarked partial residue.
+        for delete_markers in [false, true] {
+            for file in &committed_files {
+                let is_marker = file.rsplit(SLASH_SEPARATOR).next().is_some_and(is_committed_delete_marker);
+                if is_marker != delete_markers {
+                    continue;
+                }
+                if let Err(err) = disk
+                    .delete(
+                        bucket,
+                        file,
+                        DeleteOptions {
+                            recursive: false,
+                            immediate: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                {
+                    debug!(
+                        event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_SET_DISK,
+                        bucket,
+                        object,
+                        path = file,
+                        error = ?err,
+                        "Orphan prefix purge skipped"
+                    );
+                    return;
+                }
+            }
+        }
+        empty_dirs.reverse();
+        for dir in empty_dirs {
+            if let Err(err) = disk
+                .delete(
+                    bucket,
+                    &dir,
+                    DeleteOptions {
+                        recursive: false,
+                        immediate: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                // Best effort: a sibling removal may have already cleared a shared
+                // parent, or a concurrent writer repopulated the directory. Neither
+                // is fatal to purging the orphan tree.
+                debug!(
+                    event = EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    bucket,
+                    object,
+                    path = dir,
+                    error = ?err,
+                    "Orphan prefix purge skipped"
+                );
+            }
+        }
+    }
+
     /// Purge an orphan directory prefix — a trailing-slash key that exists on disk
-    /// as an empty directory tree with no object metadata on any disk of this set.
+    /// as empty directories or committed delete residue, with no object metadata
+    /// or uncommitted data on any disk of this set.
     /// Such prefixes are listable (see `scan_dir`) yet are not real objects, so the
     /// normal delete path returns NotFound and leaves them stranded (issue #4189).
     ///
@@ -6187,15 +6304,18 @@ impl SetDisks {
         // Phase 1: classify every online disk. Refuse to purge if ANY disk holds
         // object data under the prefix, so a degraded/healable object is never
         // destroyed.
-        let mut per_disk_dirs: Vec<(usize, Vec<String>)> = Vec::new();
+        let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
         let mut existed = false;
         for (i, disk) in disks.iter().enumerate() {
             let Some(disk) = disk else { continue };
             match Self::scan_orphan_dir(disk, bucket, object).await {
                 OrphanDirScan::HasData => return Ok(false),
-                OrphanDirScan::Empty(dirs) => {
+                OrphanDirScan::Purgeable {
+                    empty_dirs,
+                    committed_files,
+                } => {
                     existed = true;
-                    per_disk_dirs.push((i, dirs));
+                    per_disk_dirs.push((i, empty_dirs, committed_files));
                 }
                 OrphanDirScan::Missing => {}
             }
@@ -6205,32 +6325,14 @@ impl SetDisks {
             return Ok(false);
         }
 
-        // Phase 2: remove the empty directories children-first on each disk. A
-        // non-recursive delete performs an empty-only `rmdir`, so a directory that
-        // concurrently gained an object fails with DirectoryNotEmpty and is skipped —
-        // a racing PutObject is never clobbered.
-        for (i, mut dirs) in per_disk_dirs {
+        // Phase 2: remove only the files classified as committed residue, then
+        // remove directories children-first. Every directory delete is
+        // non-recursive, so a directory that concurrently gained an object fails
+        // with DirectoryNotEmpty and is skipped — a racing PutObject is never
+        // clobbered.
+        for (i, empty_dirs, committed_files) in per_disk_dirs {
             let Some(disk) = disks[i].as_ref() else { continue };
-            dirs.reverse();
-            for dir in dirs {
-                if let Err(err) = disk
-                    .delete(
-                        bucket,
-                        &dir,
-                        DeleteOptions {
-                            recursive: false,
-                            immediate: true,
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                {
-                    // Best effort: a sibling removal may have already cleared a shared
-                    // parent, or a concurrent writer repopulated the directory. Neither
-                    // is fatal to purging the orphan tree.
-                    debug!(bucket, object, dir, error = ?err, "purge_orphan_dir_object: skipped non-empty/absent directory");
-                }
-            }
+            Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
         }
 
         Ok(true)
@@ -7049,6 +7151,16 @@ mod tests {
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
 
+    #[test]
+    fn orphan_dir_entries_must_be_single_relative_components() {
+        for entry in ["part.1", "child/", "delete-data.00000000-0000-0000-0000-000000000001"] {
+            assert!(is_safe_orphan_dir_entry(entry), "{entry:?} should be accepted");
+        }
+        for entry in ["", "/", ".", "..", "../", "child//", "a/b", r"a\b", "./"] {
+            assert!(!is_safe_orphan_dir_entry(entry), "{entry:?} should be rejected");
+        }
+    }
+
     #[test]
     #[serial_test::serial(codec_streaming_env)]
     fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() {
@@ -7227,6 +7339,96 @@ mod tests {
         (dir, disk)
     }
 
+    #[tokio::test]
+    async fn orphan_cleanup_preserves_object_published_after_scan() {
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let transaction = Uuid::new_v4();
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}")), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+
+        let nested_object = residue.join("nested");
+        tokio::fs::create_dir_all(&nested_object)
+            .await
+            .expect("concurrent object directory should be created");
+        tokio::fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"new metadata")
+            .await
+            .expect("concurrent object metadata should be written");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        assert!(
+            nested_object.join(STORAGE_FORMAT_FILE).exists(),
+            "an object published after classification must survive cleanup"
+        );
+        assert!(!residue.join("part.1").exists(), "classified stale data should be reclaimed");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn orphan_cleanup_keeps_commit_marker_when_residue_delete_fails() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let (dir, disk) = read_multiple_test_disk("bucket", &[]).await;
+        let marker_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4());
+        let residue = dir
+            .path()
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        tokio::fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        tokio::fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        tokio::fs::write(residue.join(&marker_name), [])
+            .await
+            .expect("committed delete marker should be written");
+
+        let OrphanDirScan::Purgeable {
+            empty_dirs,
+            committed_files,
+        } = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
+        else {
+            panic!("committed residue should be classified as purgeable");
+        };
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
+            .await
+            .expect("residue directory should become read-only");
+
+        SetDisks::delete_purgeable_orphan_entries(&disk, "bucket", "pfx/", empty_dirs, committed_files).await;
+
+        let part_remains = residue.join("part.1").exists();
+        let marker_remains = residue.join(marker_name).exists();
+        tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o755))
+            .await
+            .expect("residue directory permissions should be restored");
+        assert!(part_remains, "the injected residue delete failure should retain the part");
+        assert!(marker_remains, "the commit marker must remain so a later cleanup can retry");
+    }
+
     async fn io_primitives_test_set(disks: Vec<Option<DiskStore>>, default_parity_count: usize) -> Arc<SetDisks> {
         let set_drive_count = disks.len();
         SetDisks::new(

crates/ecstore/src/set_disk/mod.rs

@@ -127,7 +127,7 @@ use rustfs_filemeta::{
 };
 use rustfs_heal_contracts::heal_channel::{
     DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
-    send_heal_disk, send_heal_request_with_admission,
+    send_heal_replacement_disk, send_heal_request_with_admission,
 };
 use rustfs_io_metrics::{
     record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
@@ -329,6 +329,7 @@ const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
 const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
 const EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED: &str = "set_disk_rename_tail_drain_failed";
 const EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY: &str = "set_disk_put_object_stage_summary";
+const EVENT_SET_DISK_ORPHAN_PURGE_SKIPPED: &str = "set_disk_orphan_purge_skipped";
 const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
 const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
 const DEFAULT_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 64 * 1024 * 1024;
@@ -858,7 +859,9 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
 
 mod core;
 #[cfg(test)]
-pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
+pub(crate) use core::io_primitives::disk_call_counters;
+#[cfg(all(test, feature = "test-util"))]
+pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
 mod ctx;
 mod metadata;
 mod ops;
@@ -871,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
 pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
 #[cfg(test)]
 pub(crate) use ops::object::DeleteObjectCommitBarrier;
-#[cfg(feature = "test-util")]
+#[cfg(any(test, feature = "test-util"))]
 pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
 pub(crate) use ops::object::body_cache_plaintext_len;
 #[cfg(all(test, feature = "test-util"))]
@@ -1638,10 +1641,13 @@ mod prepared_get_object_metadata_tests {
         .await;
     }
 
-    #[tokio::test]
+    #[test]
     #[serial_test::serial(body_cache_hook)]
-    async fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
-        let (_dirs, set_disks) = make_local_set_disks(6, 2).await;
+    fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("current-thread runtime should build");
         let bucket = "inline-read-plan-unequal";
         let object = object_with_initial_data_shards_for_geometry(bucket, "inline-object", 6, 2);
         let payload = b"inline quorum payload".repeat(256);
@@ -1650,55 +1656,65 @@ mod prepared_get_object_metadata_tests {
             ..Default::default()
         };
 
-        set_disks
-            .make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("bucket should be created");
-        let mut put_reader = PutObjReader::from_vec(payload.clone());
-        set_disks
-            .put_object(bucket, &object, &mut put_reader, &opts)
-            .await
-            .expect("inline object should be written");
-
-        let read_once = |enabled: bool| {
-            let set_disks = Arc::clone(&set_disks);
-            let bucket = bucket.to_string();
-            let object = object.clone();
-            let payload = payload.clone();
-            let opts = opts.clone();
-            async move {
-                temp_env::async_with_vars(
-                    [
-                        (
-                            "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
-                            Some(if enabled { "true" } else { "false" }),
-                        ),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
-                        ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
-                    ],
-                    async {
-                        let calls = disk_call_counters::observe(&object);
-                        let mut reader = set_disks
-                            .get_object_reader(&bucket, &object, None, HeaderMap::new(), &opts)
-                            .await
-                            .expect("inline GET reader should open");
-                        let mut restored = Vec::new();
-                        reader
-                            .stream
-                            .read_to_end(&mut restored)
-                            .await
-                            .expect("inline GET body should stream");
-                        assert_eq!(restored, payload);
-                        calls.total(disk_call_counters::KIND_READ_VERSION)
-                    },
-                )
+        let (_dirs, set_disks) = runtime.block_on(async {
+            let (dirs, set_disks) = make_local_set_disks(6, 2).await;
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
                 .await
-            }
-        };
+                .expect("bucket should be created");
+            let mut put_reader = PutObjReader::from_vec(payload.clone());
+            set_disks
+                .put_object(bucket, &object, &mut put_reader, &opts)
+                .await
+                .expect("inline object should be written");
+            (dirs, set_disks)
+        });
 
-        let gate_off_calls = read_once(false).await;
-        let gate_on_calls = read_once(true).await;
-        assert_eq!(gate_on_calls, gate_off_calls, "inline gate must not add reserve fanout");
+        // disk_call_counters counts tasks that started running, so an
+        // early-stop abort races the single-pending inline hedge into a ±1
+        // count per read. The fanout lifecycle histogram records the
+        // scheduling decision itself and stays deterministic under load.
+        let recorder = CapturingRecorder::default();
+        let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
+        rustfs_io_metrics::set_get_stage_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            runtime.block_on(async {
+                for enabled in [false, true] {
+                    temp_env::async_with_vars(
+                        [
+                            (
+                                "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
+                                Some(if enabled { "true" } else { "false" }),
+                            ),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
+                            ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
+                        ],
+                        async {
+                            let mut reader = set_disks
+                                .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
+                                .await
+                                .expect("inline GET reader should open");
+                            let mut restored = Vec::new();
+                            reader
+                                .stream
+                                .read_to_end(&mut restored)
+                                .await
+                                .expect("inline GET body should stream");
+                            assert_eq!(restored, payload);
+                        },
+                    )
+                    .await;
+                }
+            })
+        });
+        rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
+
+        let scheduled = recorder.histogram_values(
+            "rustfs_io_get_object_metadata_fanout_scheduled",
+            &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
+        );
+        assert_eq!(scheduled.len(), 2, "each GET should run exactly one metadata fanout");
+        assert_eq!(scheduled[1], scheduled[0], "inline gate must not add reserve fanout");
     }
 
     #[test]
@@ -8775,6 +8791,111 @@ mod tests {
         assert!(root.join("bucket").exists(), "bucket volume should remain");
     }
 
+    #[tokio::test]
+    async fn purge_orphan_dir_object_removes_committed_delete_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let data_dir = Uuid::new_v4();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("nested")
+            .join("object")
+            .join(data_dir.to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed delete residue should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("purge should succeed");
+
+        assert!(purged, "committed delete residue should be purgeable");
+        assert!(!root.join("bucket").join("pfx").exists(), "prefix directory should be gone");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_uncommitted_data_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("uncommitted data residue should be created");
+        fs::write(residue.join("part.1"), b"possibly live")
+            .await
+            .expect("data part should be written");
+        fs::write(residue.join("delete-data.not-a-uuid"), [])
+            .await
+            .expect("malformed marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "data without a valid committed marker must be preserved");
+        assert!(residue.join("part.1").exists(), "possibly live data must remain");
+    }
+
+    #[tokio::test]
+    async fn purge_orphan_dir_object_preserves_nested_object_below_committed_residue() {
+        let (dir, disk) = make_single_local_disk().await;
+        let root = dir.path();
+        let transaction = Uuid::new_v4();
+        let residue = root
+            .join("bucket")
+            .join("pfx")
+            .join("object")
+            .join(Uuid::new_v4().to_string());
+        fs::create_dir_all(&residue)
+            .await
+            .expect("committed data directory should be created");
+        fs::write(residue.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+        let nested_object = residue.join("nested");
+        fs::create_dir_all(&nested_object)
+            .await
+            .expect("nested object directory should be created");
+        fs::write(nested_object.join(STORAGE_FORMAT_FILE), b"meta")
+            .await
+            .expect("nested object metadata should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let purged = set
+            .purge_orphan_dir_object("bucket", "pfx/")
+            .await
+            .expect("scan should succeed");
+
+        assert!(!purged, "nested object metadata must veto committed-residue cleanup");
+        assert!(nested_object.join(STORAGE_FORMAT_FILE).exists(), "nested object metadata must remain");
+        assert!(residue.join("part.1").exists(), "committed residue must remain when cleanup is vetoed");
+    }
+
     // issue #4189: a prefix that still anchors a real object must be left intact.
     #[tokio::test]
     async fn purge_orphan_dir_object_preserves_prefix_with_object() {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -2812,9 +2812,20 @@ mod heal_result_report_tests {
             }
 
             let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-            set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-                .expect("source object should be written");
+            // This fixture reads and removes physical shards immediately after
+            // PUT. A lock-owning PUT may quorum-ack before its rename tail
+            // drains, so keep the setup on the full-fanout commit path.
+            set.put_object(
+                &bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("source object should be written");
             let source = disks[2]
                 .read_version("", &bucket, object, "", &ReadOptions::default())
                 .await
@@ -3230,9 +3241,20 @@ mod heal_result_report_tests {
         }
 
         let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
-        set.put_object(bucket, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("source object should be written");
+        // The target-evidence readback below asserts per-disk state right
+        // after PUT. A lock-owning PUT may quorum-ack before its rename tail
+        // drains, so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source object should be written");
         let source = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
             .await
@@ -3279,6 +3301,9 @@ mod heal_result_report_tests {
         .await
         .expect("versioned bucket should be created");
 
+        // The per-version target-evidence readback below asserts per-disk
+        // state right after PUT. A lock-owning PUT may quorum-ack before its
+        // rename tail drains, so keep the setup on the full-fanout commit path.
         let mut old_reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
         let old_info = set
             .put_object(
@@ -3287,6 +3312,7 @@ mod heal_result_report_tests {
                 &mut old_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -3304,6 +3330,7 @@ mod heal_result_report_tests {
                 &mut latest_reader,
                 &ObjectOptions {
                     versioned: true,
+                    no_lock: true,
                     ..Default::default()
                 },
             )
@@ -4325,9 +4352,20 @@ mod heal_result_report_tests {
 
         const PAYLOAD_SIZE: usize = 1024 * 1024;
         let mut initial_reader = PutObjReader::from_vec(vec![0x11; PAYLOAD_SIZE]);
-        set.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
-            .await
-            .expect("initial object should be written");
+        // This fixture reads and removes physical shards immediately after
+        // PUT. A lock-owning PUT may quorum-ack before its rename tail drains,
+        // so keep the setup on the full-fanout commit path.
+        set.put_object(
+            bucket,
+            object,
+            &mut initial_reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("initial object should be written");
 
         let current = disks[2]
             .read_version("", bucket, object, "", &ReadOptions::default())
@@ -4415,11 +4453,12 @@ mod heal_result_report_tests {
                 // Give the heal something to rebuild on alternating rounds: remove a
                 // shard of the current data dir right before the race.
                 if round % 2 == 1 {
-                    let current = disks[2]
-                        .read_version("", bucket, object, "", &ReadOptions::default())
-                        .await
-                        .expect("current metadata should be readable");
-                    if let Some(data_dir) = current.data_dir {
+                    // The previous round's lock-owning PUT may still be
+                    // draining its rename tail on this disk; shard damage is
+                    // best-effort here, so skip injection when it lags.
+                    if let Ok(current) = disks[2].read_version("", bucket, object, "", &ReadOptions::default()).await
+                        && let Some(data_dir) = current.data_dir
+                    {
                         let shard = temp_dirs[3]
                             .path()
                             .join(bucket)

crates/ecstore/src/set_disk/ops/locking.rs

@@ -22,12 +22,10 @@
 use super::super::{
     Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
     NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
-    send_heal_disk, warn,
+    send_heal_replacement_disk, warn,
 };
-use crate::disk::DiskAPI;
 use crate::disk::health_state::DriveMembershipSnapshot;
-#[cfg(test)]
-use crate::disk::new_disk;
+use crate::disk::{DiskAPI, new_disk};
 use crate::runtime::sources as runtime_sources;
 use rand::prelude::SliceRandom;
 #[cfg(test)]
@@ -356,11 +354,28 @@ impl SetDisks {
             Ok(res) => res,
             Err(e) => {
                 warn!("renew_disk: connect_endpoint err {:?}", &e);
-                if ep.is_local && e == DiskError::UnformattedDisk {
-                    info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep);
-                    let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx);
-                    let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await;
+                if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
+                    return;
                 }
+
+                let attached = match self.attach_unformatted_replacement_disk(ep).await {
+                    Ok(attached) => attached,
+                    Err(err) => {
+                        warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
+                        return;
+                    }
+                };
+                if !attached {
+                    return;
+                }
+
+                info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
+                let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
+                    warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
+                    return;
+                };
+                let _ =
+                    send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
                 return;
             }
         };
@@ -412,6 +427,60 @@ impl SetDisks {
         disk_lock[disk_idx] = Some(new_disk);
     }
 
+    /// Attach a replacement target only after proving that the exact local slot
+    /// is present and unformatted. A health-checked reconnect may reject a
+    /// blank target before it reaches the format-heal path; that target still
+    /// has to be visible in this set for the formatter to claim it safely.
+    async fn attach_unformatted_replacement_disk(&self, ep: &Endpoint) -> disk::error::Result<bool> {
+        if !ep.is_local
+            || usize::try_from(ep.pool_idx).ok() != Some(self.pool_index)
+            || usize::try_from(ep.set_idx).ok() != Some(self.set_index)
+        {
+            return Ok(false);
+        }
+
+        let Some(disk_idx) = self.set_endpoints.iter().position(|candidate| candidate == ep) else {
+            return Ok(false);
+        };
+
+        let replacement = new_disk(
+            ep,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await?;
+        match load_format_erasure(&replacement, false).await {
+            Err(DiskError::UnformattedDisk) => {}
+            Ok(_) => return Ok(false),
+            Err(err) => return Err(err),
+        }
+
+        {
+            let mut disks = self.disks.write().await;
+            if disks[disk_idx].as_ref().is_some_and(|existing| existing.endpoint() != *ep) {
+                warn!(endpoint = %ep, disk_idx, "renew_disk rejected unformatted replacement for an occupied foreign slot");
+                return Ok(false);
+            }
+            disks[disk_idx] = Some(replacement.clone());
+        }
+
+        let local_disk_map = runtime_sources::local_disk_map_handle();
+        local_disk_map
+            .write()
+            .await
+            .insert(replacement.endpoint().to_string(), Some(replacement.clone()));
+
+        if runtime_sources::setup_is_dist_erasure().await {
+            let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
+            let mut local_set_drives = local_disk_set_drives.write().await;
+            local_set_drives[self.pool_index][self.set_index][disk_idx] = Some(replacement);
+        }
+
+        Ok(true)
+    }
+
     pub(in crate::set_disk) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> {
         self.format.check_other(fm)?;
 
@@ -779,6 +848,75 @@ mod tests {
         drop(temp_dirs);
     }
 
+    #[tokio::test]
+    async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
+        let disk_count = 4;
+        let format = FormatV3::new(1, disk_count);
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count - 1 {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let replacement_dir = tempfile::tempdir().expect("replacement tempdir should be created");
+        let mut replacement_endpoint =
+            Endpoint::try_from(replacement_dir.path().to_str().expect("replacement path should be utf8"))
+                .expect("replacement endpoint should parse");
+        replacement_endpoint.set_pool_index(0);
+        replacement_endpoint.set_set_index(0);
+        replacement_endpoint.set_disk_index(disk_count - 1);
+        temp_dirs.push(replacement_dir);
+        endpoints.push(replacement_endpoint.clone());
+        disks.push(None);
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        assert!(
+            set_disks
+                .attach_unformatted_replacement_disk(&replacement_endpoint)
+                .await
+                .expect("a blank local replacement should be admitted")
+        );
+
+        let attached = set_disks.get_disks_internal().await;
+        let replacement = attached[disk_count - 1]
+            .as_ref()
+            .expect("the verified replacement must occupy its exact set slot");
+        assert_eq!(replacement.endpoint(), replacement_endpoint);
+        assert!(
+            !replacement.health_check_enabled_for_test(),
+            "the blank replacement must not start health checks before it receives a format"
+        );
+        assert_eq!(
+            load_format_erasure(replacement, false).await.unwrap_err(),
+            DiskError::UnformattedDisk,
+            "only a still-unformatted replacement may be attached by the fallback"
+        );
+
+        runtime_sources::local_disk_map_handle()
+            .write()
+            .await
+            .remove(&replacement_endpoint.to_string());
+        drop(set_disks);
+        drop(temp_dirs);
+    }
+
     #[tokio::test]
     async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
         let disk_count = 3;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -244,7 +244,7 @@ impl StaleMultipartCleanupGuard {
 }
 
 #[cfg(any(test, feature = "test-util"))]
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum MultipartCommitPause {
     NewUploadBeforeLockLost,
     PutPartBeforeLockAcquire,
@@ -5459,19 +5459,42 @@ mod tests {
             disk.make_volume(bucket).await.expect("bucket volume should be created");
         }
         let mut initial_reader = PutObjReader::from_vec(b"old multipart body".to_vec());
+        // A lock-owning PUT may quorum-ack before its rename tail drains, and
+        // cache priming refuses to publish while a straggler disk still reads
+        // as an error; keep the setup on the full-fanout commit path.
         set_disks
-            .put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
+            .put_object(
+                bucket,
+                object,
+                &mut initial_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
             .await
             .expect("initial object should be written");
-        set_disks
-            .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
-            .await
-            .expect("initial metadata should resolve");
-        let generation = set_disks
-            .get_object_metadata_cache_generation(bucket, object)
-            .expect("metadata cache generation should be active");
-        let retired_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
-        assert!(set_disks.get_object_metadata_cache.get(&retired_key).await.is_some());
+        // The publish is also bounded by the cache TTL, so re-prime until the
+        // current generation is observably cached instead of asserting on a
+        // single read that a loaded host can stall past expiry.
+        let retired_key = tokio::time::timeout(std::time::Duration::from_secs(30), async {
+            loop {
+                set_disks
+                    .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+                    .await
+                    .expect("initial metadata should resolve");
+                let generation = set_disks
+                    .get_object_metadata_cache_generation(bucket, object)
+                    .expect("metadata cache generation should be active");
+                let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+                if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
+                    return key;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("metadata priming should publish the current generation");
 
         let upload = set_disks
             .new_multipart_upload(bucket, object, &ObjectOptions::default())

crates/ecstore/src/store/heal.rs

@@ -36,6 +36,10 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
     )
 }
 
+fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
+    bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
+}
+
 #[derive(Debug, Clone, Copy)]
 enum HealFormatPoolSkip {
     Completed,
@@ -492,7 +496,7 @@ impl ECStore {
         #[cfg(test)]
         let store_id = self.id;
 
-        let mut futures = Vec::with_capacity(pools.len());
+        let mut heal_pools = Vec::with_capacity(pools.len());
         for pool in pools.iter() {
             let suspended_complete = {
                 let pool_meta = self.pool_meta.read().await;
@@ -520,19 +524,58 @@ impl ECStore {
                 }
                 continue;
             }
-            let pool_idx = pool.pool_idx;
-            let pool = Arc::clone(pool);
-            let pool_object = object.clone();
-            let opts = *opts;
-            futures.push(
-                self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move {
-                    #[cfg(test)]
-                    crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
-                    pool.heal_object(bucket, &pool_object, version_id, &opts).await
-                }),
-            );
+            heal_pools.push(Arc::clone(pool));
         }
-        let results = join_all(futures).await;
+        let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
+            let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
+            match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
+                Ok((pool_meta_guard, admissions)) => {
+                    let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
+                    let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
+                        let pool = Arc::clone(pool);
+                        let pool_object = object.clone();
+                        let fixed_set = fixed_set.clone();
+                        let mut opts = *opts;
+                        async move {
+                            admission?;
+                            let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
+                            let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
+                            opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
+                            #[cfg(test)]
+                            if !opts.no_lock {
+                                crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
+                            }
+                            #[cfg(test)]
+                            crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                            pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                        }
+                    });
+                    let results = join_all(futures).await;
+                    drop(pool_meta_guard);
+                    results
+                }
+                Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
+            }
+        } else {
+            let mut futures = Vec::with_capacity(heal_pools.len());
+            for pool in heal_pools {
+                let pool_idx = pool.pool_idx;
+                let pool_object = object.clone();
+                let opts = *opts;
+                futures.push(self.run_external_decommission_capacity_heal(
+                    pool_idx,
+                    bucket,
+                    &object,
+                    opts,
+                    move |opts| async move {
+                        #[cfg(test)]
+                        crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
+                        pool.heal_object(bucket, &pool_object, version_id, &opts).await
+                    },
+                ));
+            }
+            join_all(futures).await
+        };
 
         let mut errs = Vec::with_capacity(self.pools.len());
         let mut ress = Vec::with_capacity(self.pools.len());
@@ -631,7 +674,7 @@ mod tests {
     };
     use crate::core::sets::HealFormatAfterSaveBarrier;
     use crate::disk::error::Result as DiskResult;
-    use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
+    use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::services::rebalance::{
@@ -784,6 +827,30 @@ mod tests {
         }
     }
 
+    async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
+        let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
+        let missing_disk = target_set.disks.read().await[0]
+            .clone()
+            .expect("pool metadata fixture disk should be online");
+        missing_disk
+            .delete(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("one pool metadata shard should be removable");
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
+            "pool metadata fixture must start with one missing shard"
+        );
+        missing_disk
+    }
+
     #[tokio::test]
     async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
         let store = minimal_heal_store().await;
@@ -1243,6 +1310,121 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_read_repair_reuses_its_write_fence() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let missing_disk = remove_pool_meta_shard(&store, 0).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    read_repair: true,
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata read repair must not wait on its own capacity fence")
+        .expect("pool metadata read repair should complete");
+
+        assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "pool metadata read repair should restore the missing shard"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_preserves_typed_lock_timeout() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let lock = store.pools[0]
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("pool metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("pool metadata lock should be acquired");
+
+        let (_, err) = temp_env::async_with_vars(
+            [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(0),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata lock timeout should be mapped into the heal result");
+
+        assert!(
+            matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
+            "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
+        );
+        drop(guard);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
+        let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
+        let object = "pool.bin.backup";
+        save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
+            .await
+            .expect("neighbor metadata fixture should be written");
+
+        let target_set = store.pools[0].get_disks_by_key(object);
+        let lock = target_set
+            .new_ns_lock(RUSTFS_META_BUCKET, object)
+            .await
+            .expect("neighbor metadata lock should be created");
+        let guard = lock
+            .get_write_lock(get_lock_acquire_timeout())
+            .await
+            .expect("neighbor metadata lock should be acquired");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    object,
+                    "",
+                    &HealOpts {
+                        pool: Some(0),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        barrier.wait_until_external_heal_target_lock_attempted().await;
+        tokio::task::yield_now().await;
+        assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
+        drop(guard);
+
+        let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
+            .await
+            .expect("neighbor metadata heal should finish after the object lock is released")
+            .expect("neighbor metadata heal task should not panic")
+            .expect("neighbor metadata heal should complete");
+        assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
+        assert_eq!(result.object, object);
+        drop(barrier);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
@@ -1322,6 +1504,72 @@ mod tests {
             missing_disk.read_xl(&bucket, object, false).await.is_err(),
             "capacity-blocked targeted heal must not rewrite the missing shard"
         );
+
+        let (_, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("pool metadata admission should not recurse on its namespace lock")
+        .expect("pool metadata capacity rejection should be mapped");
+        assert!(
+            matches!(err, Some(Error::SlowDown)),
+            "pool metadata heal must preserve target reservation admission: {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_per_target_capacity_admission() {
+        let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
+        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
+        set_decommission_capacity_info_overrides_for_test(
+            store.id,
+            vec![vec![
+                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
+                DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
+                DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
+            ]],
+        );
+        store
+            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
+            .await
+            .expect("pool metadata reservation should activate");
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let reservation = pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .expect("pool metadata reservation should be durable");
+            assert_eq!(reservation.targets.len(), 1);
+            assert_eq!(reservation.targets[0].pool_index, 1);
+        }
+
+        let missing_disk = remove_pool_meta_shard(&store, 2).await;
+
+        let (result, err) = tokio::time::timeout(
+            std::time::Duration::from_secs(30),
+            store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
+        )
+        .await
+        .expect("unscoped pool metadata heal should complete")
+        .expect("unscoped pool metadata heal should return a mapped result");
+
+        assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
+        assert_eq!(result.object, POOL_META_NAME);
+        assert!(
+            missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "unscoped metadata heal should repair the admitted target while another target is reserved"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/store/init_format.rs

@@ -14,6 +14,7 @@
 
 use crate::cluster::rpc::client::is_network_like_disk_error;
 use crate::config::storageclass;
+use crate::core::pools::PoolMetaBootstrapAuthority;
 use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs};
 use crate::disk::{self, DiskAPI};
 use crate::error::{Error, Result};
@@ -84,7 +85,7 @@ pub async fn connect_load_init_formats(
 
 pub(crate) struct LoadedFormat {
     pub(crate) format: FormatV3,
-    pub(crate) fresh_bootstrap_proven: bool,
+    pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
 }
 
 pub(crate) async fn connect_load_init_formats_with_instance_ctx(
@@ -133,7 +134,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
                 retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
                 return Ok(LoadedFormat {
                     format: *format,
-                    fresh_bootstrap_proven: false,
+                    pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
                 });
             }
             Ok(LegacyFormatOutcome::Incompatible) => {
@@ -153,7 +154,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
             let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
             return Ok(LoadedFormat {
                 format: fm,
-                fresh_bootstrap_proven: true,
+                pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
             });
         }
     }
@@ -182,10 +183,29 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
 
     Ok(LoadedFormat {
         format: fm,
-        fresh_bootstrap_proven: false,
+        pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
     })
 }
 
+async fn verified_legacy_adoption_source(
+    disks: &[Option<DiskStore>],
+    rustfs_formats: &[Option<FormatV3>],
+    set_count: usize,
+    set_drive_count: usize,
+) -> Result<PoolMetaBootstrapAuthority> {
+    match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
+        Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
+        Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
+        Err(err) => {
+            debug!(
+                error = %err,
+                "legacy adoption proof skipped because legacy format verification failed"
+            );
+            Ok(PoolMetaBootstrapAuthority::None)
+        }
+    }
+}
+
 async fn retain_format_quorum_members(
     instance_ctx: &Arc<InstanceContext>,
     disks: &mut [Option<DiskStore>],
@@ -1311,10 +1331,7 @@ mod tests {
         let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 3, None)
             .await
             .expect("fresh disks should receive a storage format");
-        assert!(
-            loaded.fresh_bootstrap_proven,
-            "every configured disk explicitly reporting unformatted should establish fresh topology proof"
-        );
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
         let format = loaded.format;
 
         let (formats, errors) = load_format_erasure_all(&disks, false).await;
@@ -1358,12 +1375,11 @@ mod tests {
 
         let mut expected = legacy;
         expected.erasure.this = Uuid::nil();
-        assert_eq!(
-            connect_load_init_formats(true, &mut disks, 1, 3, None)
-                .await
-                .expect("compatible legacy format should migrate"),
-            expected
-        );
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 3, None)
+            .await
+            .expect("compatible legacy format should migrate");
+        assert_eq!(loaded.format, expected);
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
         let (formats, errors) = load_format_erasure_all(&disks, false).await;
         assert!(
             errors.iter().all(Option::is_none),
@@ -1378,6 +1394,51 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn compatible_single_drive_legacy_format_marks_adoption_proof() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let legacy = FormatV3::new(1, 1);
+        write_legacy_majority(&disks, &legacy).await;
+
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("single-drive MinIO format should migrate");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
+    }
+
+    #[tokio::test]
+    async fn existing_migrated_format_keeps_legacy_adoption_proof() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let legacy = FormatV3::new(1, 1);
+        write_legacy_majority(&disks, &legacy).await;
+
+        connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("first run should migrate the MinIO format");
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("retry after a partial adoption should reload the migrated RustFS format");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
+    }
+
+    #[tokio::test]
+    async fn existing_rustfs_format_without_legacy_source_is_not_legacy_adoption() {
+        let (_temp_dir, mut disks) = local_disks(1).await;
+        let mut format = FormatV3::new(1, 1);
+        format.erasure.this = format.erasure.sets[0][0];
+        save_format_file(&disks[0], &Some(format))
+            .await
+            .expect("existing RustFS format should be written");
+
+        let loaded = connect_load_init_formats_with_instance_ctx(&current_ctx(), true, &mut disks, 1, 1, None)
+            .await
+            .expect("existing RustFS format should load");
+
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+
     #[tokio::test]
     async fn compatible_legacy_format_migrates_when_the_file_is_missing() {
         let (_temp_dir, mut disks) = local_disks(3).await;

crates/ecstore/src/store/list_objects.rs

@@ -20,7 +20,9 @@ fn to_filemeta_err(err: Error) -> rustfs_filemeta::Error {
     err.narrow_to_filemeta().unwrap_or_else(rustfs_filemeta::Error::other)
 }
 
-use crate::bucket::metadata_sys::{get_versioning_config, has_authoritative_never_versioned_state};
+use crate::bucket::metadata_sys::{
+    get_versioning_config, has_authoritative_never_versioned_state, has_authoritative_never_versioned_state_in,
+};
 use crate::bucket::utils::check_list_objs_args;
 use crate::bucket::versioning::VersioningApi;
 use crate::cache_value::metacache_set::{FallbackClaimTracker, ListPathRawOptions, list_path_raw_with_claim_tracker};
@@ -314,6 +316,25 @@ async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool {
         .unwrap_or(false)
 }
 
+fn should_purge_empty_directory_listing(
+    prefix: &str,
+    marker: Option<&str>,
+    delimiter: Option<&str>,
+    max_keys: i32,
+    incl_deleted: bool,
+    result: &ListObjectsInfo,
+) -> bool {
+    !prefix.is_empty()
+        && prefix.ends_with(SLASH_SEPARATOR)
+        && marker.is_none()
+        && delimiter.is_none_or(str::is_empty)
+        && max_keys == 1
+        && !incl_deleted
+        && !result.is_truncated
+        && result.objects.is_empty()
+        && result.prefixes.is_empty()
+}
+
 const MARKER_TAG_VERSION: &str = "v2";
 const LEGACY_MARKER_TAG_VERSIONS: &[&str] = &["v1", MARKER_TAG_VERSION];
 const LIST_CACHE_MARKER_PREFIX: &str = "[rustfs_cache:";
@@ -2501,6 +2522,7 @@ fn list_metadata_resolution_params(
     listing_quorum: usize,
     latest_object_quorum: usize,
     versioned: bool,
+    write_quorum_slack: usize,
 ) -> MetadataResolutionParams {
     let quorum = if versioned {
         listing_quorum
@@ -2511,6 +2533,7 @@ fn list_metadata_resolution_params(
         dir_quorum: quorum,
         obj_quorum: quorum,
         bucket,
+        write_quorum_slack,
         ..Default::default()
     };
 
@@ -2811,7 +2834,10 @@ fn cached_entry_needs_supplement(
 
     let mut selected_object_versions = 0;
     for version in cached.versions.iter() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions) {
             return true;
         }
@@ -2836,8 +2862,15 @@ fn listing_entries_supplement_target(
         return None;
     }
 
-    for entry in entries.0.iter().flatten() {
-        if entry.is_dir() {
+    for (idx, entry) in entries.0.iter().enumerate() {
+        let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
+            continue;
+        };
+        if entries.0[..idx]
+            .iter()
+            .flatten()
+            .any(|previous| previous.name == entry.name && previous.is_object())
+        {
             continue;
         }
 
@@ -2850,6 +2883,15 @@ fn listing_entries_supplement_target(
                     .is_some_and(|candidate| candidate.name == entry.name && candidate.is_object())
             })
             .count();
+        let entries_disagree = entries.0.iter().flatten().any(|candidate| {
+            candidate.name == entry.name && candidate.is_object() && !entry.matches(Some(candidate), resolver.strict).1
+        });
+        // A split primary sample can fall back to an older version even when a
+        // newer version reaches write quorum only after the fallback disks join.
+        if entries_disagree {
+            return Some(entry.name.clone());
+        }
+
         let mut entry = entry.clone();
         if let Ok(cached) = entry.xl_meta()
             && cached_entry_needs_supplement(&cached, reader_disks, resolver, enforce_write_quorum)
@@ -2946,7 +2988,10 @@ fn resolve_agreed_listing_entry(
     let mut needs_supplement = false;
 
     for (idx, version) in cached.versions.iter().enumerate() {
-        let required_quorum = version.write_quorum(resolver.obj_quorum).max(resolver.obj_quorum);
+        let required_quorum = version
+            .write_quorum(resolver.obj_quorum)
+            .saturating_sub(resolver.write_quorum_slack)
+            .max(resolver.obj_quorum);
         if reader_disks < required_quorum {
             needs_supplement |=
                 version_requires_supplement(required_quorum, reader_disks, selected_object_versions, resolver.requested_versions);
@@ -3000,21 +3045,51 @@ fn latest_listing_object_quorum(
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
-    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum)
+    latest_listing_required_object_quorum(listing_quorum, drive_count, parity_count, enforce_write_quorum, unreachable_disks)
 }
 
+/// Object quorum a listed "latest" version must reach among the drives the
+/// listing can actually consult.
+///
+/// An object legally committed at write quorum can have up to
+/// `unreachable_disks` of its metadata copies on drives that are offline for
+/// this listing, so the write-quorum requirement is relaxed by that amount.
+/// The result is floored at the erasure read quorum (data drives): below that
+/// the object could not be read back either, and a quorum-deleted object
+/// leaves at most `drive_count - write_quorum < read_quorum` stale copies, so
+/// the floor also keeps deleted objects from resurfacing.
+///
+/// Trade-off: while a drive is offline, a torn overwrite that reached only
+/// `write_quorum - 1` drives becomes indistinguishable from a committed write
+/// whose missing copy sits on the offline drive, so it can be listed as
+/// latest. GET at read quorum serves that same version in that state, so the
+/// listing stays consistent with reads instead of hiding readable objects.
 fn latest_listing_required_object_quorum(
     listing_quorum: usize,
     drive_count: usize,
     parity_count: usize,
     enforce_write_quorum: bool,
+    unreachable_disks: usize,
 ) -> usize {
     if !enforce_write_quorum {
         return listing_quorum;
     }
 
-    write_quorum_for_drive_count(drive_count, parity_count).max(listing_quorum)
+    let read_quorum = drive_count.saturating_sub(parity_count);
+    write_quorum_for_drive_count(drive_count, parity_count)
+        .saturating_sub(unreachable_disks)
+        .max(read_quorum)
+        .max(listing_quorum)
+}
+
+fn latest_listing_write_quorum_slack(enforce_write_quorum: bool, drive_count: usize, online_disks: usize) -> usize {
+    if !enforce_write_quorum {
+        return 0;
+    }
+
+    drive_count.saturating_sub(online_disks)
 }
 
 fn enforce_latest_listing_write_quorum(strict_latest: bool, ask_disks: &str) -> bool {
@@ -3772,6 +3847,19 @@ impl ECStore {
                 .list_objects_from_opt_in_key_only_provider(&opts, mode, max_keys, incl_deleted)
                 .await?
         {
+            if should_purge_empty_directory_listing(
+                prefix,
+                opts.marker.as_deref(),
+                delimiter.as_deref(),
+                max_keys,
+                incl_deleted,
+                &result,
+            ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+                .await
+                .unwrap_or(false)
+            {
+                self.purge_orphan_dir_object(bucket, prefix).await;
+            }
             return Ok(result);
         }
 
@@ -3802,6 +3890,7 @@ impl ECStore {
         };
 
         let mut list_result = self
+            .clone()
             .list_path(&opts)
             .await
             .unwrap_or_else(|err| MetaCacheEntriesSortedResult {
@@ -3820,7 +3909,7 @@ impl ECStore {
         }
 
         if let Some(result) = list_result.entries.as_mut() {
-            result.forward_past(opts.marker);
+            result.forward_past(opts.marker.clone());
         }
 
         // contextCanceled
@@ -3848,12 +3937,26 @@ impl ECStore {
         );
         let _ = next_version_idmarker;
 
-        Ok(ListObjectsInfo {
+        let result = ListObjectsInfo {
             is_truncated,
             next_marker,
             objects,
             prefixes,
-        })
+        };
+        if should_purge_empty_directory_listing(
+            prefix,
+            opts.marker.as_deref(),
+            delimiter.as_deref(),
+            max_keys,
+            incl_deleted,
+            &result,
+        ) && has_authoritative_never_versioned_state_in(&self.ctx, bucket)
+            .await
+            .unwrap_or(false)
+        {
+            self.purge_orphan_dir_object(bucket, prefix).await;
+        }
+        Ok(result)
     }
 
     pub async fn inner_list_object_versions(
@@ -4306,6 +4409,9 @@ impl ECStore {
         for eset in self.pools.iter() {
             for set in eset.disk_set.iter() {
                 let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+                // Captured before any quorum-based filtering: only genuinely
+                // unreachable drives may relax the write-quorum requirement.
+                let online_disks = disks.len();
                 let opts = opts.clone();
 
                 let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
@@ -4331,11 +4437,14 @@ impl ECStore {
                     let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                     let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                     let write_quorum_parity = set.default_parity_count;
+                    let write_quorum_slack =
+                        latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                     let required_obj_quorum = latest_listing_required_object_quorum(
                         listing_quorum,
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                     let fallback_disks = {
@@ -4357,11 +4466,17 @@ impl ECStore {
                         set.set_drive_count,
                         write_quorum_parity,
                         enforce_write_quorum,
+                        write_quorum_slack,
                     );
                     let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
 
-                    let resolver =
-                        list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                    let resolver = list_metadata_resolution_params(
+                        bucket.to_owned(),
+                        listing_quorum,
+                        obj_quorum,
+                        !opts.latest_only,
+                        write_quorum_slack,
+                    );
                     let agreed_resolver = resolver.clone();
                     let partial_resolver = resolver.clone();
                     let reader_disks = disks.len();
@@ -5548,6 +5663,9 @@ impl Sets {
 
         for set in &self.disk_set {
             let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
+            // Captured before any quorum-based filtering: only genuinely
+            // unreachable drives may relax the write-quorum requirement.
+            let online_disks = disks.len();
             let opts = opts.clone();
             let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
             inputs.push(list_out_rx);
@@ -5573,11 +5691,14 @@ impl Sets {
                 let listing_quorum = listing_quorum_from_ask_disks(ask_disks);
                 let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks);
                 let write_quorum_parity = set.default_parity_count;
+                let write_quorum_slack =
+                    latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks);
                 let required_obj_quorum = latest_listing_required_object_quorum(
                     listing_quorum,
                     set.set_drive_count,
                     write_quorum_parity,
                     enforce_write_quorum,
+                    write_quorum_slack,
                 );
                 ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
                 let fallback_disks = if let Some(asked_disks) = positive_ask_disks(ask_disks)
@@ -5592,10 +5713,21 @@ impl Sets {
                 let fallback_disks = Arc::new(fallback_disks);
                 let claim_tracker = FallbackClaimTracker::default();
 
-                let obj_quorum =
-                    latest_listing_object_quorum(listing_quorum, set.set_drive_count, write_quorum_parity, enforce_write_quorum);
+                let obj_quorum = latest_listing_object_quorum(
+                    listing_quorum,
+                    set.set_drive_count,
+                    write_quorum_parity,
+                    enforce_write_quorum,
+                    write_quorum_slack,
+                );
                 let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum);
-                let resolver = list_metadata_resolution_params(bucket.to_owned(), listing_quorum, obj_quorum, !opts.latest_only);
+                let resolver = list_metadata_resolution_params(
+                    bucket.to_owned(),
+                    listing_quorum,
+                    obj_quorum,
+                    !opts.latest_only,
+                    write_quorum_slack,
+                );
                 let agreed_resolver = resolver.clone();
                 let partial_resolver = resolver.clone();
                 let reader_disks = disks.len();
@@ -6517,6 +6649,9 @@ impl SetDisks {
         let list_path_started = std::time::Instant::now();
 
         let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await;
+        // Captured before any quorum-based filtering: only genuinely
+        // unreachable drives may relax the write-quorum requirement.
+        let online_disks = disks.len();
 
         let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
         if ask_disks == -1 {
@@ -6540,11 +6675,13 @@ impl SetDisks {
 
         let enforce_write_quorum = enforce_latest_listing_write_quorum(!opts.versioned, &opts.ask_disks);
         let write_quorum_parity = self.default_parity_count;
+        let write_quorum_slack = latest_listing_write_quorum_slack(enforce_write_quorum, self.set_drive_count, online_disks);
         let required_obj_quorum = latest_listing_required_object_quorum(
             listing_quorum,
             self.set_drive_count,
             write_quorum_parity,
             enforce_write_quorum,
+            write_quorum_slack,
         );
         ask_disks = expand_ask_disks_for_object_quorum(ask_disks, disks.len(), required_obj_quorum);
         let mut fallback_disks = Vec::new();
@@ -6562,10 +6699,21 @@ impl SetDisks {
 
         let bucket = opts.bucket.clone();
         let base_dir = opts.base_dir.clone();
-        let latest_object_quorum =
-            latest_listing_object_quorum(listing_quorum, self.set_drive_count, write_quorum_parity, enforce_write_quorum);
+        let latest_object_quorum = latest_listing_object_quorum(
+            listing_quorum,
+            self.set_drive_count,
+            write_quorum_parity,
+            enforce_write_quorum,
+            write_quorum_slack,
+        );
         let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, latest_object_quorum, enforce_write_quorum);
-        let resolver = list_metadata_resolution_params(bucket.clone(), listing_quorum, latest_object_quorum, opts.versioned);
+        let resolver = list_metadata_resolution_params(
+            bucket.clone(),
+            listing_quorum,
+            latest_object_quorum,
+            opts.versioned,
+            write_quorum_slack,
+        );
         let agreed_resolver = resolver.clone();
         let partial_resolver = resolver.clone();
         let reader_disks = disks.len();
@@ -6577,6 +6725,7 @@ impl SetDisks {
             asked_disks = ask_disks,
             listing_quorum = listing_quorum,
             latest_object_quorum = latest_object_quorum,
+            write_quorum_slack = write_quorum_slack,
             raw_min_disks = raw_min_disks,
             fallback_disks = fallback_disks.len(),
             limit = opts.limit,
@@ -6818,34 +6967,36 @@ mod test {
         LIST_CURSOR_GENERATION_LIVE, LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY,
         LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, ListIndexFallbackReason, ListIndexLifecycle, ListIndexLifecycleState,
         ListIndexSourceDecision, ListMetadataAuthority, ListMetadataIndexHealth, ListObjectsIndexProviderKind,
-        ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution,
-        ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
+        ListObjectsIndexProviderState, ListObjectsInfo, ListPathOptions, ListPathRawOptions, ListSourceMode,
+        ListingEntryResolution, ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
         NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
         PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
         PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex,
         PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker,
-        current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum,
-        expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects,
-        latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key,
-        list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
+        cached_entry_needs_supplement, current_list_objects_mutation_sequence, encode_persistent_list_metadata_object,
+        enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results,
+        latest_listing_allow_agreed_objects, latest_listing_object_quorum, latest_listing_raw_min_disks,
+        latest_listing_required_object_quorum, latest_listing_write_quorum_slack, list_marker_key, list_merged_entry_channel,
+        list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
         list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
         list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
         list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
         list_objects_metadata_fast_guardrails_from_env, list_objects_paginate, list_objects_quorum_from_env,
-        load_namespace_mutation_journal_state, load_persistent_key_only_index, max_keys_plus_one, merge_entry_channels,
-        namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env,
-        namespace_mutation_journal_chaos_enabled_from_env, namespace_mutation_journal_chaos_sequence_from_env,
-        namespace_mutation_journal_chaos_status_from_env, normalize_list_quorum, observe_list_objects_mutations_with_store,
-        parse_namespace_mutation_journal_state, parse_persistent_key_only_index, parse_persistent_list_metadata_object,
-        parse_version_marker, persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
+        listing_entries_supplement_target, load_namespace_mutation_journal_state, load_persistent_key_only_index,
+        max_keys_plus_one, merge_entry_channels, namespace_mutation_journal_chaos_bucket_from_env,
+        namespace_mutation_journal_chaos_config_from_env, namespace_mutation_journal_chaos_enabled_from_env,
+        namespace_mutation_journal_chaos_sequence_from_env, namespace_mutation_journal_chaos_status_from_env,
+        normalize_list_quorum, observe_list_objects_mutations_with_store, parse_namespace_mutation_journal_state,
+        parse_persistent_key_only_index, parse_persistent_list_metadata_object, parse_version_marker,
+        persist_observed_list_objects_mutation, persistent_key_only_index_has_complete_metadata_snapshot,
         persistent_key_only_index_health, persistent_key_only_index_matches_provider,
         reset_list_objects_mutation_sequences_for_test, resolve_agreed_listing_entry, resolve_listing_entries,
-        scanner_namespace_mutation_generation, select_list_index_provider_source_mode, select_list_index_source_mode,
-        send_or_cancel, version_marker_for_entries, walk_result_from_set_errors, write_namespace_mutation_journal_state,
-        write_persistent_key_only_index_with_metadata,
+        resolve_listing_entries_with_supplement, scanner_namespace_mutation_generation, select_list_index_provider_source_mode,
+        select_list_index_source_mode, send_or_cancel, should_purge_empty_directory_listing, version_marker_for_entries,
+        walk_result_from_set_errors, write_namespace_mutation_journal_state, write_persistent_key_only_index_with_metadata,
     };
     use crate::cache_value::metacache_set::{FallbackClaimTracker, TestReaderBehavior, list_path_raw};
-    use crate::disk::{DiskAPI, DiskOption, endpoint::Endpoint, error::DiskError, new_disk};
+    use crate::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, error::DiskError, new_disk};
     use crate::error::StorageError;
     use crate::object_api::ObjectInfo;
     use rustfs_filemeta::{
@@ -7168,6 +7319,32 @@ mod test {
         }
     }
 
+    fn test_object_with_delete_marker_meta_entry(
+        name: &str,
+        object_mod_time: time::OffsetDateTime,
+        delete_mod_time: time::OffsetDateTime,
+    ) -> MetaCacheEntry {
+        let mut object = test_object_meta_entry_with_erasure_versions(name, &[(object_mod_time, "object-etag", 4, 2)]);
+        let delete = test_delete_marker_meta_entry(name, delete_mod_time);
+        let mut metadata = object.cached.take().expect("test object metadata should be cached");
+        let delete_version = delete
+            .cached
+            .expect("test delete marker metadata should be cached")
+            .versions
+            .into_iter()
+            .next()
+            .expect("test delete marker should contain one version");
+        metadata.versions.insert(0, delete_version);
+        let encoded = metadata.marshal_msg().expect("test metadata should marshal");
+
+        MetaCacheEntry {
+            name: name.to_owned(),
+            metadata: encoded,
+            cached: Some(metadata),
+            reusable: false,
+        }
+    }
+
     fn test_dir_meta_entry(name: &str) -> MetaCacheEntry {
         MetaCacheEntry {
             name: name.to_owned(),
@@ -8665,6 +8842,80 @@ mod test {
         assert_eq!(scanner_namespace_mutation_generation(), 2);
     }
 
+    #[test]
+    fn empty_directory_listing_purge_requires_complete_exact_recursive_request() {
+        let empty = ListObjectsInfo::default();
+        assert!(should_purge_empty_directory_listing("ghost/", None, None, 1, false, &empty));
+        assert!(should_purge_empty_directory_listing("ghost/", None, Some(""), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost", None, None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", Some("marker"), None, 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, Some("/"), 1, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 0, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 2, false, &empty));
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, true, &empty));
+
+        let mut live = ListObjectsInfo::default();
+        live.objects.push(ObjectInfo::default());
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &live));
+
+        let truncated = ListObjectsInfo {
+            is_truncated: true,
+            ..Default::default()
+        };
+        assert!(!should_purge_empty_directory_listing("ghost/", None, None, 1, false, &truncated));
+    }
+
+    #[tokio::test]
+    async fn empty_recursive_listing_purges_committed_delete_residue() {
+        use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+        use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        let bucket = "listing-purge-bucket";
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created with authoritative metadata");
+        let data_dir = uuid::Uuid::new_v4();
+        let transaction = uuid::Uuid::new_v4();
+        for dir in &dirs {
+            let residue = dir
+                .path()
+                .join(bucket)
+                .join("ghost")
+                .join("nested")
+                .join("object")
+                .join(data_dir.to_string());
+            tokio::fs::create_dir_all(&residue)
+                .await
+                .expect("committed delete residue should be created");
+            tokio::fs::write(residue.join("part.1"), b"stale")
+                .await
+                .expect("stale part should be written");
+            tokio::fs::write(
+                residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+                [],
+            )
+            .await
+            .expect("committed delete marker should be written");
+        }
+
+        let result = store
+            .list_objects_generic(bucket, "ghost/", None, None, 1, false)
+            .await
+            .expect("empty recursive listing should succeed");
+
+        assert!(result.objects.is_empty());
+        assert!(result.prefixes.is_empty());
+        for dir in &dirs {
+            assert!(
+                !dir.path().join(bucket).join("ghost").exists(),
+                "the empty listing should reclaim its committed delete residue"
+            );
+        }
+    }
+
     #[test]
     fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
         let provider = ListObjectsIndexProviderState::walker_key_only();
@@ -8947,7 +9198,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_limits_plain_listing_to_latest_version() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8957,7 +9208,7 @@ mod test {
 
     #[test]
     fn list_metadata_resolution_params_keeps_all_versions_for_version_listing() {
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true, 0);
 
         assert_eq!(resolver.dir_quorum, 3);
         assert_eq!(resolver.obj_quorum, 3);
@@ -8967,22 +9218,22 @@ mod test {
 
     #[test]
     fn latest_listing_object_quorum_uses_write_quorum_for_strict_latest_listing() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 5);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
     }
 
     #[test]
     fn latest_listing_object_quorum_calculates_low_parity_write_quorum() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 1, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum);
 
         assert_eq!(required_quorum, 7);
         assert_eq!(ask_disks, 7);
-        assert_eq!(latest_listing_object_quorum(2, 8, 1, true), 7);
+        assert_eq!(latest_listing_object_quorum(2, 8, 1, true, 0), 7);
     }
 
     #[test]
@@ -8991,21 +9242,196 @@ mod test {
         assert!(!enforce_latest_listing_write_quorum(true, "disk"));
         assert!(enforce_latest_listing_write_quorum(true, "optimal"));
         assert!(!enforce_latest_listing_write_quorum(false, "optimal"));
-        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_object_quorum(1, 4, 2, false), 1);
-        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false), 2);
-        assert_eq!(latest_listing_object_quorum(2, 4, 2, false), 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_object_quorum(1, 4, 2, false, 0), 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, false, 0), 2);
+        assert_eq!(latest_listing_object_quorum(2, 4, 2, false, 0), 2);
         assert_eq!(expand_ask_disks_for_object_quorum(2, 4, 2), 2);
     }
 
+    #[test]
+    fn latest_listing_object_quorum_relaxes_write_quorum_by_unreachable_drives() {
+        // 4-drive set, EC 2+2: with one drive offline an object committed at
+        // write quorum 3 can only ever show 2 metadata copies to the listing.
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        assert_eq!(slack, 1);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, slack), 2);
+        assert_eq!(latest_listing_required_object_quorum(2, 4, 2, true, 0), 3);
+
+        // The relaxed quorum never drops below the erasure read quorum, so a
+        // quorum-deleted object (at most one stale copy) stays hidden even
+        // with half the set unreachable.
+        let slack = latest_listing_write_quorum_slack(true, 4, 2);
+        assert_eq!(slack, 2);
+        assert_eq!(latest_listing_required_object_quorum(1, 4, 2, true, slack), 2);
+
+        assert_eq!(latest_listing_write_quorum_slack(false, 4, 2), 0);
+    }
+
+    #[test]
+    fn latest_listing_resolves_degraded_object_when_a_set_drive_is_unreachable() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]);
+
+        let slack = latest_listing_write_quorum_slack(true, 4, 3);
+        let obj_quorum = latest_listing_object_quorum(2, 4, 2, true, slack);
+        assert_eq!(obj_quorum, 2);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, obj_quorum, false, slack);
+        let resolved = resolve_listing_entries(entries(), resolver, true)
+            .expect("object committed at write quorum should stay listable with one holder drive offline");
+        assert_eq!(resolved.name, "object");
+
+        // Without the unreachable-drive slack the same sample is dropped even
+        // though the object still satisfies read quorum for GET.
+        let strict_quorum = latest_listing_object_quorum(2, 4, 2, true, 0);
+        let strict_resolver = list_metadata_resolution_params("bucket".to_string(), 2, strict_quorum, false, 0);
+        assert!(resolve_listing_entries(entries(), strict_resolver, true).is_none());
+    }
+
+    #[test]
+    fn agreed_listing_entry_relaxes_write_quorum_by_unreachable_drives() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry.clone(), 2, resolver.clone(), true),
+            ListingEntryResolution::Resolved(_)
+        ));
+
+        resolver.write_quorum_slack = 0;
+        assert!(matches!(
+            resolve_agreed_listing_entry(entry, 2, resolver, true),
+            ListingEntryResolution::NeedsSupplement(_, _)
+        ));
+    }
+
+    #[test]
+    fn cached_entry_supplement_check_honors_write_quorum_slack() {
+        let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let mut entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]);
+        let cached = entry.xl_meta().expect("test entry should decode");
+        let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1);
+
+        assert!(!cached_entry_needs_supplement(&cached, 2, &resolver, true));
+
+        resolver.write_quorum_slack = 0;
+        assert!(cached_entry_needs_supplement(&cached, 2, &resolver, true));
+    }
+
     #[test]
     fn latest_listing_object_quorum_requires_write_quorum_when_degraded_cannot_satisfy_it() {
-        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true);
+        let required_quorum = latest_listing_required_object_quorum(2, 8, 4, true, 0);
         let ask_disks = expand_ask_disks_for_object_quorum(4, 4, required_quorum);
 
         assert_eq!(required_quorum, 5);
         assert_eq!(ask_disks, 4);
-        assert_eq!(latest_listing_object_quorum(2, 8, 4, true), 5);
+        assert_eq!(latest_listing_object_quorum(2, 8, 4, true, 0), 5);
+    }
+
+    #[tokio::test]
+    async fn latest_listing_supplements_a_split_delete_marker_sample() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+            Some(deleted.clone()),
+        ]);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, false), None);
+
+        let mut primary = resolve_listing_entries(MetaCacheEntries(entries.0.clone()), resolver.clone(), true)
+            .expect("the partial sample should fall back to the stale object version");
+        assert!(!primary.is_latest_delete_marker());
+
+        let mut fallback_disks = Vec::new();
+        let mut fallback_tempdirs = Vec::new();
+        for _ in 0..2 {
+            let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
+            let endpoint =
+                Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")).expect("valid endpoint");
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("fallback disk should be created");
+            disk.make_volume("bucket").await.expect("fallback bucket should be created");
+            disk.write_all(
+                "bucket",
+                &format!("object/{STORAGE_FORMAT_FILE}"),
+                bytes::Bytes::copy_from_slice(&deleted.metadata),
+            )
+            .await
+            .expect("fallback metadata should be written");
+            fallback_disks.push(disk);
+            fallback_tempdirs.push(tempdir);
+        }
+        let supplement = ListingSupplement::new(
+            ListingSupplementOptions {
+                bucket: "bucket".to_string(),
+                path: String::new(),
+                recursive: true,
+                incl_deleted: false,
+                skip_hidden_prefix_check: false,
+                filter_prefix: None,
+                forward_to: None,
+                per_disk_limit: 100,
+                skip_total_timeout: true,
+                walkdir_timeout: None,
+                walkdir_stall_timeout: None,
+            },
+            Arc::new(fallback_disks),
+            FallbackClaimTracker::default(),
+        );
+
+        let mut supplemented = resolve_listing_entries_with_supplement(entries, resolver, true, supplement)
+            .await
+            .expect("the supplemented sample should resolve the committed delete marker");
+        assert!(supplemented.is_latest_delete_marker());
+    }
+
+    #[test]
+    fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
+        let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
+        let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
+        let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]);
+        let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0);
+        let entries = MetaCacheEntries(vec![
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(stale.clone()),
+            Some(deleted.clone()),
+        ]);
+
+        assert_eq!(listing_entries_supplement_target(&entries, &resolver, true).as_deref(), Some("object"));
+
+        let mut resolved = resolve_listing_entries(
+            MetaCacheEntries(vec![
+                Some(stale.clone()),
+                Some(stale.clone()),
+                Some(stale),
+                Some(deleted.clone()),
+                Some(deleted.clone()),
+                Some(deleted),
+            ]),
+            resolver,
+            true,
+        )
+        .expect("the previous object version should remain visible below delete-marker write quorum");
+
+        assert!(!resolved.is_latest_delete_marker());
     }
 
     #[tokio::test]
@@ -9082,7 +9508,7 @@ mod test {
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
         let fallback_old_entry = test_object_meta_entry_with_erasure_versions("object", &[(old_mod_time, "old-etag", 4, 4)]);
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9137,7 +9563,7 @@ mod test {
             "object",
             &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)],
         );
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 
@@ -9181,7 +9607,7 @@ mod test {
         let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
         let entry = test_object_meta_entry_with_erasure_versions("object", &[(new_mod_time, "new-etag", 7, 1)]);
         let fallback_entry = entry.clone();
-        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false);
+        let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0);
         let seen = Arc::new(Mutex::new(Vec::new()));
         let seen_clone = seen.clone();
 

crates/ecstore/src/store/mod.rs

@@ -65,7 +65,7 @@ use http::HeaderMap;
 use lazy_static::lazy_static;
 use rand::RngExt as _;
 use rustfs_config::server_config::Config;
-use rustfs_filemeta::FileInfo;
+use rustfs_filemeta::{FileInfo, FileMeta};
 use rustfs_heal_contracts::heal_channel::{HealItemType, HealOpts};
 use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
 use rustfs_madmin::heal_commands::HealResultItem;
@@ -88,25 +88,144 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
 pub const SCANNER_PUBLICATION_LEASE_TTL_MS: u64 = 60_000;
+pub(crate) const BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES: u64 = 1024 * 1024;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES: usize = 4_096;
+pub(crate) const BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED: Duration = Duration::from_millis(100);
+
+#[derive(Debug)]
+pub(crate) struct BucketDeleteDiagnosticBudget {
+    deadline: Option<tokio::time::Instant>,
+    max_elapsed: Duration,
+    entries_remaining: usize,
+    #[cfg(test)]
+    first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
+}
+
+impl BucketDeleteDiagnosticBudget {
+    pub(crate) fn new() -> Self {
+        Self::with_limits(BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED)
+    }
+
+    fn with_limits(entries: usize, elapsed: Duration) -> Self {
+        Self {
+            deadline: None,
+            max_elapsed: elapsed,
+            entries_remaining: entries,
+            #[cfg(test)]
+            first_io_delay: None,
+        }
+    }
+
+    #[cfg(test)]
+    fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
+        self.first_io_delay = Some((delay, started));
+        self
+    }
+
+    fn deadline(&mut self) -> tokio::time::Instant {
+        let max_elapsed = self.max_elapsed;
+        *self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
+    }
+
+    fn claim_entry(&mut self) -> bool {
+        if self.entries_remaining == 0 {
+            return false;
+        }
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+        self.entries_remaining -= 1;
+        true
+    }
+
+    async fn run_io<T, F>(&mut self, future: F) -> std::io::Result<Option<T>>
+    where
+        F: std::future::Future<Output = std::io::Result<T>>,
+    {
+        let deadline = self.deadline();
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(None);
+        }
+        #[cfg(test)]
+        let first_io_delay = self.first_io_delay.take();
+        #[cfg(test)]
+        let timeout_result = tokio::time::timeout_at(deadline, async move {
+            if let Some((delay, started)) = first_io_delay {
+                started.store(true, std::sync::atomic::Ordering::SeqCst);
+                tokio::time::sleep(delay).await;
+            }
+            future.await
+        })
+        .await;
+        #[cfg(not(test))]
+        let timeout_result = tokio::time::timeout_at(deadline, future).await;
+        match timeout_result {
+            Ok(result) => result.map(Some),
+            Err(_) => Ok(None),
+        }
+    }
+}
 
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub(crate) struct BucketMetadataLessResidue {
     pub(crate) xlmeta_found: bool,
+    pub(crate) xlmeta_blocker: Option<BucketDeleteBlockerKind>,
     pub(crate) files: usize,
     pub(crate) uuid_data_dirs: usize,
+    pub(crate) entries_scanned: usize,
+    pub(crate) diagnostic_bytes_read: u64,
+    pub(crate) diagnostic_truncated: bool,
     pub(crate) sample: Option<String>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BucketDeleteBlockerKind {
+    VisibleVersion,
+    TierFreeVersion,
+    UnknownXlMeta,
+    OrphanDirectory,
+    DiagnosticBudgetExceeded,
+}
+
+impl BucketDeleteBlockerKind {
+    pub(crate) const fn as_str(self) -> &'static str {
+        match self {
+            Self::VisibleVersion => "visible_version",
+            Self::TierFreeVersion => "tier_free_version",
+            Self::UnknownXlMeta => "unknown_xlmeta",
+            Self::OrphanDirectory => "orphan_directory",
+            Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
+        }
+    }
+
+    /// Whether the blocking residue is something the caller can still see and
+    /// remove through the S3 API.
+    ///
+    /// A live version or a tier free-version is ordinary: the bucket really is
+    /// not empty, the client can list and delete what is left, and the 409 it
+    /// receives is a complete answer.
+    ///
+    /// The remaining kinds are not. They are on-disk state that no S3 request
+    /// can reach: the caller has drained every version the API will show and
+    /// `DeleteBucket` still refuses, with no way to find out why. That is a
+    /// server-side integrity problem, and it is the reason this classification
+    /// exists — see [`bucket_delete_blocker_level`].
+    pub(crate) const fn is_client_visible(self) -> bool {
+        matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
+    }
+}
+
 impl BucketMetadataLessResidue {
     pub(crate) fn has_residue_without_xlmeta(&self) -> bool {
-        !self.xlmeta_found && self.files > 0
+        !self.xlmeta_found && (self.files > 0 || self.diagnostic_truncated)
     }
 
     pub(crate) fn describe(&self) -> String {
         let sample = self.sample.as_deref().unwrap_or("<none>");
         format!(
-            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, sample={sample}",
-            self.files, self.uuid_data_dirs
+            "metadata-less on-disk residue remains after empty-bucket verification: files={}, uuid_data_dirs={}, entries_scanned={}, diagnostic_bytes_read={}, diagnostic_truncated={}, sample={sample}",
+            self.files, self.uuid_data_dirs, self.entries_scanned, self.diagnostic_bytes_read, self.diagnostic_truncated,
         )
     }
 }
@@ -145,28 +264,112 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
     Ok(false)
 }
 
+#[cfg(test)]
 pub(crate) async fn scan_metadata_less_residue(path: &std::path::Path) -> std::io::Result<BucketMetadataLessResidue> {
+    let mut budget = BucketDeleteDiagnosticBudget::new();
+    scan_metadata_less_residue_with_budget(path, &mut budget).await
+}
+
+async fn scan_metadata_less_residue_with_budget(
+    path: &std::path::Path,
+    budget: &mut BucketDeleteDiagnosticBudget,
+) -> std::io::Result<BucketMetadataLessResidue> {
     use crate::disk::STORAGE_FORMAT_FILE;
     use tokio::fs;
+    use tokio::io::AsyncReadExt as _;
 
     let mut scan = BucketMetadataLessResidue::default();
     let mut stack = vec![path.to_path_buf()];
 
+    let mark_budget_exhausted = |scan: &mut BucketMetadataLessResidue| {
+        scan.diagnostic_truncated = true;
+        scan.sample.get_or_insert_with(|| "<diagnostic-budget-exceeded>".to_string());
+    };
+
     while let Some(current_path) = stack.pop() {
-        let mut entries = match fs::read_dir(&current_path).await {
-            Ok(entries) => entries,
+        let mut entries = match budget.run_io(fs::read_dir(&current_path)).await {
+            Ok(Some(entries)) => entries,
+            Ok(None) => {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
             Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
             Err(err) => return Err(err),
         };
 
-        while let Some(entry) = entries.next_entry().await? {
-            let file_type = entry.file_type().await?;
+        loop {
+            let entry = match budget.run_io(entries.next_entry()).await? {
+                Some(Some(entry)) => entry,
+                Some(None) => break,
+                None => {
+                    mark_budget_exhausted(&mut scan);
+                    return Ok(scan);
+                }
+            };
+            if !budget.claim_entry() {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            }
+            scan.entries_scanned = scan.entries_scanned.saturating_add(1);
+            let Some(file_type) = budget.run_io(entry.file_type()).await? else {
+                mark_budget_exhausted(&mut scan);
+                return Ok(scan);
+            };
             let file_name = entry.file_name();
             let file_name_str = file_name.to_string_lossy();
 
             if file_name_str == STORAGE_FORMAT_FILE {
                 scan.xlmeta_found = true;
-                continue;
+                if scan.xlmeta_blocker.is_none() {
+                    let entry_path = entry.path();
+                    let Some(metadata) = budget.run_io(fs::metadata(&entry_path)).await? else {
+                        mark_budget_exhausted(&mut scan);
+                        scan.xlmeta_blocker = Some(BucketDeleteBlockerKind::DiagnosticBudgetExceeded);
+                        return Ok(scan);
+                    };
+                    scan.xlmeta_blocker = Some(if metadata.len() > BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES {
+                        BucketDeleteBlockerKind::UnknownXlMeta
+                    } else {
+                        match budget.run_io(fs::File::open(&entry_path)).await {
+                            Ok(Some(file)) => {
+                                let mut data = Vec::new();
+                                let read = budget
+                                    .run_io(file.take(BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES).read_to_end(&mut data))
+                                    .await;
+                                scan.diagnostic_bytes_read = data.len() as u64;
+                                match read {
+                                    Ok(Some(_)) => match FileMeta::load(&data) {
+                                        Ok(meta)
+                                            if !meta.versions.is_empty()
+                                                && meta.versions.iter().all(|version| version.header.free_version()) =>
+                                        {
+                                            BucketDeleteBlockerKind::TierFreeVersion
+                                        }
+                                        Ok(meta) if !meta.versions.is_empty() => BucketDeleteBlockerKind::VisibleVersion,
+                                        Ok(_) | Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                    },
+                                    Ok(None) => {
+                                        mark_budget_exhausted(&mut scan);
+                                        BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                                    }
+                                    Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                                }
+                            }
+                            Ok(None) => {
+                                mark_budget_exhausted(&mut scan);
+                                BucketDeleteBlockerKind::DiagnosticBudgetExceeded
+                            }
+                            Err(_) => BucketDeleteBlockerKind::UnknownXlMeta,
+                        }
+                    });
+                    let sample = entry_path
+                        .strip_prefix(path)
+                        .unwrap_or(entry_path.as_path())
+                        .to_string_lossy()
+                        .replace(std::path::MAIN_SEPARATOR, "/");
+                    scan.sample = Some(sample);
+                }
+                return Ok(scan);
             }
 
             if file_type.is_dir() {
@@ -222,7 +425,12 @@ pub(crate) mod init_format;
 pub(crate) mod list_objects;
 mod multipart;
 mod object;
-pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence, tiered_data_movement_source_matches};
+#[cfg(any(test, feature = "test-util"))]
+pub use object::DeleteAfterObjectLockSnapshotBarrier;
+pub(crate) use object::{
+    DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
+    SourceCleanupMutationFence, tiered_data_movement_source_matches,
+};
 pub use object::{
     PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
     SnapshotConsistencyError,

crates/ecstore/src/store/multipart.rs

@@ -846,6 +846,7 @@ impl ECStore {
             .await
     }
 
+    #[cfg(all(test, feature = "test-util"))]
     pub(crate) async fn complete_multipart_upload_for_data_movement(
         self: Arc<Self>,
         target: (usize, Option<&ObjectLockDiagGuard>),
@@ -854,6 +855,44 @@ impl ECStore {
         upload_id: &str,
         uploaded_parts: Vec<CompletePart>,
         opts: &ObjectOptions,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(target, bucket, object, upload_id, uploaded_parts, opts, None)
+            .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) async fn complete_multipart_upload_for_data_movement_with_publication_fence(
+        self: Arc<Self>,
+        target_pool_idx: usize,
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: RemoteTuplePublicationFence,
+    ) -> Result<ObjectInfo> {
+        self.complete_multipart_upload_for_data_movement_inner(
+            (target_pool_idx, None),
+            bucket,
+            object,
+            upload_id,
+            uploaded_parts,
+            opts,
+            Some(publication_fence),
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn complete_multipart_upload_for_data_movement_inner(
+        self: Arc<Self>,
+        target: (usize, Option<&ObjectLockDiagGuard>),
+        bucket: &str,
+        object: &str,
+        upload_id: &str,
+        uploaded_parts: Vec<CompletePart>,
+        opts: &ObjectOptions,
+        publication_fence: Option<RemoteTuplePublicationFence>,
     ) -> Result<ObjectInfo> {
         let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
@@ -885,8 +924,36 @@ impl ECStore {
             snapshot.add_lock_fences(&mut opts);
             opts.object_lock_config_snapshot = Some(snapshot);
         }
-        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+        let fixed_read_anchor = publication_fence
+            .as_ref()
+            .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence.or(fixed_read_anchor))
             .await;
+        // NewMultipart/UploadPart are staging only. Acquire and consume the
+        // non-cloneable publication capability immediately before Complete,
+        // then retain its guards until Complete has drained the commit path.
+        let publication_object = encode_dir_object(object);
+        let publication_guard = match publication_fence {
+            Some(publication_fence) => {
+                let guard = publication_fence
+                    .into_commit_guard(target_pool_idx, bucket, &publication_object)
+                    .await?;
+                guard.add_namespace_lock_fence(&mut opts);
+                opts.no_lock = true;
+                Some(guard)
+            }
+            None => {
+                if rustfs_utils::http::metadata_compat::contains_key_str(
+                    &opts.user_defined,
+                    rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
+                ) {
+                    return Err(Error::other(
+                        "data movement multipart completion cannot publish transition ownership without a publication capability",
+                    ));
+                }
+                None
+            }
+        };
         #[cfg(test)]
         pause_data_movement_multipart_before_selected_completion(bucket).await;
         let pool = self
@@ -911,6 +978,7 @@ impl ECStore {
                 },
             )
             .await;
+        drop(publication_guard);
         let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
         if result.is_ok() {
             list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;

crates/ecstore/src/store/rebalance.rs

@@ -309,6 +309,17 @@ impl ECStore {
     }
 
     pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
+            let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+            let authorization = opts
+                .tier_delete_dispatch_authorization
+                .as_ref()
+                .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
+            authorization.ensure_current(bucket, incarnation, object)?;
+            Some((authorization, incarnation))
+        } else {
+            None
+        };
         if opts.lifecycle_delete_all.is_some() {
             let mut preflight_opts = opts.clone();
             preflight_opts
@@ -317,6 +328,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
                 pool.delete_object(bucket, object, preflight_opts.clone()).await?;
@@ -326,6 +340,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .lock()
                 .mark_mutation_started();
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.mark_mutation_started(bucket, incarnation, object)?;
+            }
             let mut non_trigger_opts = opts.clone();
             non_trigger_opts
                 .lifecycle_delete_all
@@ -333,6 +350,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::History;
             for pool in &self.pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
                 let mut pool_opts = non_trigger_opts.clone();
@@ -348,6 +368,9 @@ impl ECStore {
                 .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
             let mut trigger_pools = Vec::new();
             for (pool_index, pool) in self.pools.iter().enumerate() {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
                 let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
@@ -366,6 +389,9 @@ impl ECStore {
                 .ok_or(StorageError::PreconditionFailed)?
                 .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
             for pool_index in trigger_pools {
+                if let Some((authorization, incarnation)) = dispatch_scope {
+                    authorization.ensure_current(bucket, incarnation, object)?;
+                }
                 #[cfg(test)]
                 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
                 let mut pool_opts = trigger_opts.clone();
@@ -378,7 +404,13 @@ impl ECStore {
         let mut first_error = None;
         let mut first_volume_error = None;
         let mut has_success = false;
+        if let Some((authorization, incarnation)) = dispatch_scope {
+            authorization.mark_mutation_started(bucket, incarnation, object)?;
+        }
         for pool in &self.pools {
+            if let Some((authorization, incarnation)) = dispatch_scope {
+                authorization.ensure_current(bucket, incarnation, object)?;
+            }
             let mut opts = opts.clone();
             opts.delete_prefix = true;
             match pool.delete_object(bucket, object, opts).await {
@@ -907,8 +939,12 @@ impl ECStore {
         bucket: &str,
         object: &str,
         opts: &ObjectOptions,
+        exact: &ObjectInfo,
         errs: Vec<PoolErr>,
     ) -> Result<ObjectInfo> {
+        self.reconcile_decommission_capacity_before_exact_delete(bucket, object, opts, exact)
+            .await?;
+
         let mut results = Vec::with_capacity(errs.len());
 
         for pe in errs.iter() {